internal(voice): wire DebugMessage over remote-session wire

.changeset/bump-protocol-1.46.4-debug-message.md

@@ -0,0 +1,16 @@
+---
+'@livekit/agents': patch
+---
+
+internal(voice): wire `DebugMessage` over the remote-session wire
+
+Bumps `@livekit/protocol` to `^1.46.4` and wires the new
+`AgentSessionEvent.debug_message` (livekit/protocol#1593) through
+`SessionHost` / `RemoteSession`.
+
+Internal-only — the framework exposes an unstable `AgentSession._emitDebugMessage(payload)`
+for the debugger/recorder; not intended for user code.
+
+Also unlocks `AgentSessionEvent.FunctionToolsStarted`,
+`AgentSessionEvent.EotPrediction`, and `SessionRequest.UpdateIO` for
+downstream consumers.

agents/package.json

@@ -53,7 +53,7 @@
     "@bufbuild/protobuf": "^1.10.0",
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@livekit/mutex": "^1.1.1",
-    "@livekit/protocol": "^1.45.7",
+    "@livekit/protocol": "^1.46.4",
     "@livekit/typed-emitter": "^3.0.0",
     "@livekit/throws-transformer": "0.1.8",
     "@opentelemetry/api": "^1.9.0",

agents/src/voice/agent_session.ts

@@ -1,7 +1,10 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { JsonValue } from '@bufbuild/protobuf';
+import { Struct } from '@bufbuild/protobuf';
 import { Mutex } from '@livekit/mutex';
+import { AgentSession as pb } from '@livekit/protocol';
 import type { AudioFrame, Room } from '@livekit/rtc-node';
 import { ThrowsPromise } from '@livekit/throws-transformer/throws';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
@@ -157,6 +160,7 @@ export type AgentSessionCallbacks = {
   [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;
   [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;
   [AgentSessionEventTypes.OverlappingSpeech]: (ev: OverlappingSpeechEvent) => void;
+  [AgentSessionEventTypes.DebugMessage]: (ev: pb.DebugMessage) => void;
 };
 
 export type AgentSessionOptions<UserData = UnknownUserData> = {
@@ -706,6 +710,14 @@ export class AgentSession<
     return this.activity.interrupt(options);
   }
 
+  /* internal */
+  _emitDebugMessage(payload: Record<string, unknown>): void {
+    this.emit(
+      AgentSessionEventTypes.DebugMessage,
+      new pb.DebugMessage({ payload: Struct.fromJson(payload as JsonValue) }),
+    );
+  }
+
   /**
    * The currently bound `AMD` instance, or `null` if AMD is not in use.
    * Mirrors python `AgentSession.amd`.

agents/src/voice/events.ts

@@ -33,6 +33,7 @@ export enum AgentSessionEventTypes {
   SpeechCreated = 'speech_created',
   AgentFalseInterruption = 'agent_false_interruption',
   OverlappingSpeech = 'overlapping_speech',
+  DebugMessage = 'debug_message',
   Error = 'error',
   Close = 'close',
 }

agents/src/voice/remote_session.ts

@@ -62,6 +62,7 @@ export type RemoteSessionEventTypes =
   | 'function_tools_executed'
   | 'overlapping_speech'
   | 'amd_prediction'
+  | 'debug_message'
   | 'session_usage'
   | 'error';
 
@@ -74,6 +75,7 @@ export type RemoteSessionCallbacks = {
   function_tools_executed: (ev: pb.AgentSessionEvent_FunctionToolsExecuted) => void;
   overlapping_speech: (ev: pb.AgentSessionEvent_OverlappingSpeech) => void;
   amd_prediction: (ev: pb.AgentSessionEvent_AmdPrediction) => void;
+  debug_message: (ev: pb.DebugMessage) => void;
   session_usage: (ev: pb.AgentSessionEvent_SessionUsageUpdated) => void;
   error: (ev: pb.AgentSessionEvent_Error) => void;
 };
@@ -522,6 +524,7 @@ export class SessionHost {
       session.on(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected);
       session.on(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech);
       session.on(AgentSessionEventTypes.Error, this.onHostError);
+      session.on(AgentSessionEventTypes.DebugMessage, this.onDebugMessage);
     }
   }
 
@@ -550,6 +553,7 @@ export class SessionHost {
       this.session.off(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected);
       this.session.off(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech);
       this.session.off(AgentSessionEventTypes.Error, this.onHostError);
+      this.session.off(AgentSessionEventTypes.DebugMessage, this.onDebugMessage);
     }
 
     if (this.recvTask) {
@@ -713,6 +717,10 @@ export class SessionHost {
     );
   };
 
+  private onDebugMessage = (event: pb.DebugMessage): void => {
+    this.emitEvent({ case: 'debugMessage', value: event });
+  };
+
   /**
    * @internal — forwards an AMD prediction to the connected
    * {@link RemoteSession} peer. Mirrors python
@@ -1007,6 +1015,9 @@ export class RemoteSession extends (EventEmitter as new () => TypedEventEmitter<
       case 'error':
         this.emit('error', ev.value);
         break;
+      case 'debugMessage':
+        this.emit('debug_message', ev.value);
+        break;
     }
   }