Add support for new bot-output event, providing corresponding callbac…

client-js/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to **Pipecat Client JS** will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- BotOutput
+
+### Deprecated
+
+- BotTranscription
+
 ## [1.4.1]
 
 ### Added

client-js/client/client.ts

@@ -11,6 +11,7 @@ import packageJson from "../package.json";
 import {
   BotLLMSearchResponseData,
   BotLLMTextData,
+  BotOutputData,
   BotReadyData,
   BotTTSTextData,
   ClientMessageData,
@@ -101,6 +102,7 @@ export type RTVIEventCallbacks = Partial<{
   onUserStartedSpeaking: () => void;
   onUserStoppedSpeaking: () => void;
   onUserTranscript: (data: TranscriptData) => void;
+  onBotOutput: (data: BotOutputData) => void;
   onBotTranscript: (data: BotLLMTextData) => void;
 
   onBotLlmText: (data: BotLLMTextData) => void;
@@ -158,6 +160,8 @@ export class PipecatClient extends RTVIEventEmitter {
   protected _functionCallCallbacks: Record<string, FunctionCallCallback> = {};
   protected _abortController: AbortController | undefined;
 
+  private _botTranscriptionWarned = false;
+
   constructor(options: PipecatClientOptions) {
     super();
 
@@ -300,7 +304,17 @@ export class PipecatClient extends RTVIEventEmitter {
         options?.callbacks?.onUserTranscript?.(data);
         this.emit(RTVIEvent.UserTranscript, data);
       },
+      onBotOutput: (data) => {
+        options?.callbacks?.onBotOutput?.(data);
+        this.emit(RTVIEvent.BotOutput, data);
+      },
       onBotTranscript: (text) => {
+        if (!this._botTranscriptionWarned) {
+          logger.warn(
+            "[Pipecat Client] Bot transcription is deprecated. Please use the onBotOutput instead."
+          );
+          this._botTranscriptionWarned = true;
+        }
         options?.callbacks?.onBotTranscript?.(text);
         this.emit(RTVIEvent.BotTranscript, text);
       },
@@ -369,7 +383,7 @@ export class PipecatClient extends RTVIEventEmitter {
   @transportAlreadyStarted
   public async startBot(startBotParams: APIRequest): Promise<unknown> {
     this._transport.state = "authenticating";
-    this._transport.startBotParams = startBotParams
+    this._transport.startBotParams = startBotParams;
     this._abortController = new AbortController();
     let response: unknown;
     try {
@@ -701,6 +715,10 @@ export class PipecatClient extends RTVIEventEmitter {
         this._options.callbacks?.onUserTranscript?.(TranscriptData);
         break;
       }
+      case RTVIMessageType.BOT_OUTPUT: {
+        this._options.callbacks?.onBotOutput?.(ev.data as BotOutputData);
+        break;
+      }
       case RTVIMessageType.BOT_TRANSCRIPTION: {
         this._options.callbacks?.onBotTranscript?.(ev.data as BotLLMTextData);
         break;

client-js/rtvi/events.ts

@@ -9,6 +9,7 @@ import { DeviceError } from "./errors";
 import {
   BotLLMSearchResponseData,
   BotLLMTextData,
+  BotOutputData,
   BotReadyData,
   BotTTSTextData,
   LLMFunctionCallData,
@@ -46,6 +47,8 @@ export enum RTVIEvent {
 
   // stt events
   UserTranscript = "userTranscript",
+  BotOutput = "botOutput",
+  // DEPRECATED
   BotTranscript = "botTranscript",
 
   // llm events
@@ -117,6 +120,7 @@ export type RTVIEvents = Partial<{
 
   // stt events
   userTranscript: (data: TranscriptData) => void;
+  botOutput: (data: BotOutputData) => void;
   botTranscript: (data: BotLLMTextData) => void;
 
   // llm events

client-js/rtvi/messages.ts

@@ -43,6 +43,8 @@ export enum RTVIMessageType {
 
   /** Transcription Messages */
   USER_TRANSCRIPTION = "user-transcription", // Local user speech to text transcription (partials and finals)
+  BOT_OUTPUT = "bot-output", // A best effort aggregation of all bot output along with metadata like if it's spoken
+  // DEPRECATED
   BOT_TRANSCRIPTION = "bot-transcription", // Bot full text transcription (sentence aggregated)
   USER_STARTED_SPEAKING = "user-started-speaking", // User started speaking
   USER_STOPPED_SPEAKING = "user-stopped-speaking", // User stopped speaking
@@ -119,6 +121,12 @@ export type TranscriptData = {
   user_id: string;
 };
 
+export type BotOutputData = {
+  text: string;
+  spoken: boolean;
+  aggregated_by?: "word" | "sentence" | string;
+};
+
 export type BotLLMTextData = {
   text: string;
 };