feat: mock non-speech audio generation (ElevenLabs, fal.ai, Gemini)

CHANGELOG.md

@@ -10,6 +10,10 @@
 - **`./agui` subpath export** — `agui-stub.ts` wired into `tsdown.config.ts` and `package.json` exports, matching `./a2a`, `./mcp`, `./vector` pattern.
 - **Complete AG-UI type exports** — All event types (reasoning, step, thinking, raw, custom, chunk), `AGUIBuildOpts`, `matchesAGUIFixture`, `AGUIReasoningEncryptedValueSubtype`, `AGUIMessageRole` now exported from package root.
 - **`"warn"` log level** — New log level between `"silent"` and `"info"` with proper hierarchy. `AGUIMockOptions.logLevel` option added; AG-UI mock defaults to `"warn"` instead of `"silent"`.
+- **Non-speech audio generation** — Mock support for ElevenLabs sound effects (`/v1/sound-generation`) and music (`/v1/music/*`), fal.ai queue-based audio (`/fal/queue/submit/*`, `/fal/queue/requests/*`, `/fal/run/*`), Gemini HTTP audio via `generateContent`/`streamGenerateContent` with `inlineData` audio parts, and Gemini Live WebSocket audio. Convenience methods: `onAudio()`, `onSoundEffect()`, `onMusic()`, `onFalAudio()`. (PR #140, closes #118)
+- **AudioResponse broadened** — `audio` field now supports both `string` (base64) and `{ b64Json, contentType }` object form
+- **Gemini audio recording** — Record and replay Gemini audio responses (both streaming SSE and non-streaming JSON)
+- **Router audio-gen filtering** — Bidirectional endpoint filtering for `audio-gen` and `fal-audio` endpoint types
 
 ### Fixed
 

docs/index.html

@@ -1547,8 +1547,8 @@ <h3>Chaos Testing</h3>
             <div class="feature-icon">&#127912;</div>
             <h3>Multimedia APIs</h3>
             <p>
-              Image generation, text-to-speech, audio transcription, and video generation &mdash;
-              mock every multimedia endpoint with fixtures.
+              Image generation, text-to-speech, audio transcription, non-speech audio generation,
+              and video generation &mdash; mock every multimedia endpoint with fixtures.
             </p>
           </div>
 
@@ -1680,7 +1680,7 @@ <h2 class="fade-in">How aimock compares</h2>
               </tr>
               <tr>
                 <td>Multi-provider support</td>
-                <td class="col-aimock"><span class="yes">12 providers &#10003;</span></td>
+                <td class="col-aimock"><span class="yes">13 providers &#10003;</span></td>
                 <td><span class="manual">manual</span></td>
                 <td>12 providers</td>
                 <td>OpenAI only</td>
@@ -1732,6 +1732,15 @@ <h2 class="fade-in">How aimock compares</h2>
                 <td><span class="no">&#10007;</span></td>
                 <td><span class="no">&#10007;</span></td>
               </tr>
+              <tr>
+                <td>Non-speech audio</td>
+                <td class="col-aimock"><span class="yes">Built-in &#10003;</span></td>
+                <td><span class="no">&#10007;</span></td>
+                <td><span class="no">&#10007;</span></td>
+                <td><span class="no">&#10007;</span></td>
+                <td><span class="no">&#10007;</span></td>
+                <td><span class="no">&#10007;</span></td>
+              </tr>
               <tr>
                 <td>Video generation</td>
                 <td class="col-aimock"><span class="yes">Built-in &#10003;</span></td>

scripts/update-competitive-matrix.ts

@@ -91,6 +91,18 @@ const FEATURE_RULES: FeatureRule[] = [
       "transcription api",
     ],
   },
+  {
+    rowLabel: "Non-speech audio",
+    keywords: [
+      "sound-generation",
+      "sound effect",
+      "music generation",
+      "elevenlabs",
+      "fal.ai",
+      "audio generation",
+      "non-speech audio",
+    ],
+  },
   {
     rowLabel: "Video generation",
     keywords: ["sora", "/v1/videos", "video generation", "generate.*video"],
@@ -236,6 +248,7 @@ function countProviders(text: string): number {
     ["openai"],
     ["claude", "anthropic"],
     ["gemini", "google.*ai"],
+    ["gemini.*interactions"],
     ["bedrock", "aws"],
     ["azure"],
     ["vertex"],

src/__tests__/competitive-matrix.test.ts

@@ -56,6 +56,18 @@ const FEATURE_RULES: FeatureRule[] = [
     rowLabel: "Image generation",
     keywords: ["dall-e", "dalle", "/v1/images", "image generation", "imagen", "generate.*image"],
   },
+  {
+    rowLabel: "Non-speech audio",
+    keywords: [
+      "sound-generation",
+      "sound effect",
+      "music generation",
+      "elevenlabs",
+      "fal.ai",
+      "audio generation",
+      "non-speech audio",
+    ],
+  },
   {
     rowLabel: "Video generation",
     keywords: ["sora", "/v1/videos", "video generation", "generate.*video"],

src/__tests__/elevenlabs-audio.test.ts

@@ -0,0 +1,229 @@
+import { describe, test, expect, afterEach } from "vitest";
+import { LLMock } from "../llmock.js";
+
+describe("ElevenLabs sound generation", () => {
+  let mock: LLMock;
+
+  afterEach(async () => {
+    await mock?.stop();
+  });
+
+  test("sound generation with string-form audio returns binary", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.addFixture({
+      match: { userMessage: "castle door opening", endpoint: "audio-gen" },
+      response: { audio: "SGVsbG8=", format: "mp3" },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/sound-generation`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ text: "castle door opening" }),
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toBe("audio/mpeg");
+    const buffer = await res.arrayBuffer();
+    expect(buffer.byteLength).toBeGreaterThan(0);
+    // "SGVsbG8=" decodes to "Hello" (5 bytes)
+    expect(buffer.byteLength).toBe(5);
+  });
+
+  test("sound generation with object-form audio", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.addFixture({
+      match: { userMessage: "explosion", endpoint: "audio-gen" },
+      response: { audio: { b64Json: "SGVsbG8=", contentType: "audio/wav" } },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/sound-generation`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ text: "explosion" }),
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toBe("audio/wav");
+    const buffer = await res.arrayBuffer();
+    expect(buffer.byteLength).toBe(5);
+  });
+
+  test("missing text field returns 400", async () => {
+    mock = new LLMock({ port: 0 });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/sound-generation`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({}),
+    });
+
+    expect(res.status).toBe(400);
+    const data = await res.json();
+    expect(data.error.message).toContain("text");
+  });
+
+  test("no matching fixture returns 404", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.addFixture({
+      match: { userMessage: "specific sound", endpoint: "audio-gen" },
+      response: { audio: "SGVsbG8=" },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/sound-generation`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ text: "completely different sound" }),
+    });
+
+    expect(res.status).toBe(404);
+  });
+
+  test("error fixture returns error status", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.addFixture({
+      match: { userMessage: "rate limited", endpoint: "audio-gen" },
+      response: { error: { message: "rate limit", type: "rate_limit_error" }, status: 429 },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/sound-generation`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ text: "rate limited" }),
+    });
+
+    expect(res.status).toBe(429);
+    const data = await res.json();
+    expect(data.error.message).toBe("rate limit");
+  });
+});
+
+describe("ElevenLabs music", () => {
+  let mock: LLMock;
+
+  afterEach(async () => {
+    await mock?.stop();
+  });
+
+  test("music compose returns binary audio with song-id header", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.addFixture({
+      match: { userMessage: "upbeat piano", endpoint: "audio-gen" },
+      response: { audio: "SGVsbG8=", format: "mp3" },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/music`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ prompt: "upbeat piano" }),
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toBe("audio/mpeg");
+    expect(res.headers.get("song-id")).toBeTruthy();
+    expect(res.headers.get("song-id")).toMatch(/^mock-song-/);
+    const buffer = await res.arrayBuffer();
+    expect(buffer.byteLength).toBe(5);
+  });
+
+  test("music stream returns binary audio", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.addFixture({
+      match: { userMessage: "ambient drone", endpoint: "audio-gen" },
+      response: { audio: "SGVsbG8=" },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/music/stream`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ prompt: "ambient drone" }),
+    });
+
+    expect(res.status).toBe(200);
+    const buffer = await res.arrayBuffer();
+    expect(buffer.byteLength).toBe(5);
+  });
+
+  test("music plan returns JSON text", async () => {
+    mock = new LLMock({ port: 0 });
+    const compositionPlan = JSON.stringify({ sections: ["intro", "verse", "chorus"] });
+    mock.addFixture({
+      match: { userMessage: "jazz song", endpoint: "audio-gen" },
+      response: { content: compositionPlan },
+    });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/music/plan`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ prompt: "jazz song" }),
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toBe("application/json");
+    const data = await res.json();
+    expect(data.sections).toEqual(["intro", "verse", "chorus"]);
+  });
+
+  test("missing prompt returns 400 for music", async () => {
+    mock = new LLMock({ port: 0 });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/music`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({}),
+    });
+
+    expect(res.status).toBe(400);
+    const data = await res.json();
+    expect(data.error.message).toContain("prompt");
+  });
+});
+
+describe("ElevenLabs convenience methods", () => {
+  let mock: LLMock;
+
+  afterEach(async () => {
+    await mock?.stop();
+  });
+
+  test("onSoundEffect creates fixture with correct endpoint", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.onSoundEffect("door", { audio: "SGVsbG8=" });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/sound-generation`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ text: "door" }),
+    });
+
+    expect(res.status).toBe(200);
+    const buffer = await res.arrayBuffer();
+    expect(buffer.byteLength).toBe(5);
+  });
+
+  test("onMusic creates fixture with correct endpoint", async () => {
+    mock = new LLMock({ port: 0 });
+    mock.onMusic("piano", { audio: "SGVsbG8=" });
+    await mock.start();
+
+    const res = await fetch(`${mock.url}/v1/music`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: "Bearer test" },
+      body: JSON.stringify({ prompt: "piano" }),
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.headers.get("song-id")).toBeTruthy();
+    const buffer = await res.arrayBuffer();
+    expect(buffer.byteLength).toBe(5);
+  });
+});