feat: add grok plugin

.changeset/shiny-foxes-jump.md

@@ -0,0 +1,6 @@
+---
+"@livekit/agents-plugin-openai": patch
+"@livekit/agents-plugin-xai": patch
+---
+
+Add xai plugin and fix openai plugin for xai compatibility

examples/package.json

@@ -37,6 +37,7 @@
     "@livekit/agents-plugin-openai": "workspace:*",
     "@livekit/agents-plugin-resemble": "workspace:*",
     "@livekit/agents-plugin-silero": "workspace:*",
+    "@livekit/agents-plugin-xai": "workspace:*",
     "@livekit/noise-cancellation-node": "^0.1.9",
     "@livekit/rtc-node": "^0.13.22",
     "@opentelemetry/api": "^1.9.0",

examples/src/xai-realtime.ts

@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents';
+import * as xai from '@livekit/agents-plugin-xai';
+import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+
+export default defineAgent({
+  entry: async (ctx: JobContext) => {
+    const agent = new voice.Agent({
+      instructions: 'You are a helpful assistant. Keep your responses short and concise.',
+      tools: {
+        getWeather: llm.tool({
+          description: 'Get the weather for a given location.',
+          parameters: z.object({
+            location: z.string().describe('The location to get the weather for'),
+          }),
+          execute: async ({ location }) => {
+            return `The weather in ${location} is sunny.`;
+          },
+        }),
+      },
+    });
+
+    const session = new voice.AgentSession({
+      llm: new xai.realtime.RealtimeModel(),
+    });
+
+    await session.start({
+      agent,
+      room: ctx.room,
+    });
+  },
+});
+
+cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/openai/src/realtime/realtime_model.ts

@@ -1225,6 +1225,9 @@ export class RealtimeSession extends llm.RealtimeSession {
   }
 
   private handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {
+    if (!event.part) {
+      return;
+    }
     if (event.part.type !== 'text') {
       return;
     }

plugins/openai/tsconfig.json

@@ -1,6 +1,8 @@
 {
   "extends": "../../tsconfig.json",
-  "include": ["./src"],
+  "include": [
+    "./src"
+  ],
   "compilerOptions": {
     // match output dir to input dir. e.g. dist/index instead of dist/src/index
     "rootDir": "./src",
@@ -10,6 +12,8 @@
   "typedocOptions": {
     "name": "plugins/agents-plugin-openai",
     "entryPointStrategy": "resolve",
-    "entryPoints": ["src/index.ts"]
+    "entryPoints": [
+      "src/index.ts"
+    ]
   }
-}
+}

plugins/xai/README.md

@@ -0,0 +1,19 @@
+<!--
+SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+-->
+# xAI plugin for LiveKit Agents
+
+The Agents Framework is designed for building realtime, programmable
+participants that run on servers. Use it to create conversational, multi-modal
+voice agents that can see, hear, and understand.
+
+This package contains the xAI plugin, which provides access to xAI's Grok models
+via the Realtime API. Refer to the
+[documentation](https://docs.livekit.io/agents/overview/) for information on how
+to use it.
+
+See the [repository](https://github.com/livekit/agents-js) for more information
+about the framework as a whole.
+

plugins/xai/api-extractor.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}
+

plugins/xai/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@livekit/agents-plugin-xai",
+  "version": "1.0.0",
+  "description": "xAI plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "import": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "require": {
+      "types": "./dist/index.d.cts",
+      "default": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "git@github.com:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup --onSuccess \"pnpm build:types\"",
+    "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugin-openai": "workspace:*",
+    "@livekit/rtc-node": "^0.13.22",
+    "@microsoft/api-extractor": "^7.35.0",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "openai": "^6.8.1"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugin-openai": "workspace:*",
+    "@livekit/rtc-node": "^0.13.22"
+  }
+}

plugins/xai/src/index.ts

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { Plugin } from '@livekit/agents';
+
+export * as realtime from './realtime/index.js';
+
+class XAIPlugin extends Plugin {
+  constructor() {
+    super({
+      title: 'xai',
+      version: '0.1.0',
+      package: '@livekit/agents-plugin-xai',
+    });
+  }
+}
+
+Plugin.registerPlugin(new XAIPlugin());

plugins/xai/src/realtime/index.ts

@@ -0,0 +1,4 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './realtime_model.js';

plugins/xai/src/realtime/realtime_model.ts

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { realtime } from '@livekit/agents-plugin-openai';
+
+const { RealtimeModel: OpenAIRealtimeModel } = realtime;
+type OpenAIRealtimeModelOptions = ConstructorParameters<typeof OpenAIRealtimeModel>[0];
+
+const XAI_BASE_URL = 'wss://api.x.ai/v1';
+const DEFAULT_MODEL = 'grok-4-1-fast-non-reasoning';
+const DEFAULT_VOICE = 'Ara';
+
+const XAI_DEFAULT_TURN_DETECTION = {
+  type: 'server_vad' as const,
+  threshold: 0.5,
+  prefix_padding_ms: 300,
+  silence_duration_ms: 200,
+  create_response: true,
+  interrupt_response: true,
+};
+
+export type GrokVoices = 'Ara' | 'Cora' | 'Sage';
+
+export interface RealtimeModelOptions extends Omit<OpenAIRealtimeModelOptions, 'model'> {
+  model?: string;
+  voice?: GrokVoices | string;
+  apiKey?: string;
+}
+
+export class RealtimeModel extends OpenAIRealtimeModel {
+  constructor(options: RealtimeModelOptions = {}) {
+    super({
+      baseURL: XAI_BASE_URL,
+      model: DEFAULT_MODEL,
+      voice: options.voice || DEFAULT_VOICE,
+      apiKey: options.apiKey || process.env.XAI_API_KEY,
+      modalities: ['audio'],
+      turnDetection: XAI_DEFAULT_TURN_DETECTION,
+      ...options,
+    });
+  }
+}

plugins/xai/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["./src"],
+  "compilerOptions": {
+    // match output dir to input dir. e.g. dist/index instead of dist/src/index
+    "rootDir": "./src",
+    "declarationDir": "./dist",
+    "outDir": "./dist"
+  },
+  "typedocOptions": {
+    "name": "plugins/agents-plugin-xai",
+    "entryPointStrategy": "resolve",
+    "entryPoints": ["src/index.ts"]
+  }
+}

plugins/xai/tsup.config.ts

@@ -0,0 +1,6 @@
+import { defineConfig } from 'tsup';
+import defaults from '../../tsup.config';
+
+export default defineConfig({
+  ...defaults,
+});