Show ApplyButton's preparing state when a command is being prepared

packages/ai-bot/lib/matrix.ts

@@ -82,13 +82,11 @@ export async function sendMessageEvent(
 export async function sendCommandEvent(
   client: MatrixClient,
   roomId: string,
+  messageBody: string,
   functionCall: FunctionToolCall,
   eventToUpdate: string | undefined,
 ) {
-  let messageObject = toMatrixMessageCommandContent(
-    functionCall,
-    eventToUpdate,
-  );
+  let messageObject = toMatrixMessageCommandContent(functionCall, messageBody);
 
   if (messageObject !== undefined) {
     return await sendMatrixEvent(
@@ -131,17 +129,17 @@ export async function sendErrorEvent(
 
 export const toMatrixMessageCommandContent = (
   functionCall: FunctionToolCall,
-  eventToUpdate: string | undefined,
+  messageBody: string | undefined,
 ): IContent | undefined => {
   let { arguments: payload } = functionCall;
-  const body = payload['description'] || 'Issuing command';
+  const body = messageBody || payload['description'] || 'Issuing command';
   let messageObject: IContent = {
     body: body,
     msgtype: APP_BOXEL_COMMAND_MSGTYPE,
     formatted_body: body,
     format: 'org.matrix.custom.html',
+    isStreamingFinished: true,
     data: {
-      eventId: eventToUpdate,
       toolCall: functionCall,
     },
   };

packages/ai-bot/lib/responder.ts

@@ -14,6 +14,8 @@ import { ISendEventResponse } from 'matrix-js-sdk/lib/matrix';
 import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions';
 import { FunctionToolCall } from '@cardstack/runtime-common/helpers/ai';
 import { thinkingMessage } from '../constants';
+import { APP_BOXEL_COMMAND_MSGTYPE } from '@cardstack/runtime-common/matrix-constants';
+import type OpenAI from 'openai';
 
 let log = logger('ai-bot');
 
@@ -27,6 +29,7 @@ export class Responder {
   responseEventId: string | undefined;
   initialMessageReplaced = false;
   latestContent = '';
+  includesFunctionToolCall = false;
   isStreamingFinished = false;
 
   sendMessageEventWithDebouncing: () => Promise<void>;
@@ -36,14 +39,21 @@ export class Responder {
     this.client = client;
     this.sendMessageEventWithDebouncing = debounce(
       async () => {
+        let dataOverrides: Record<string, string | boolean> = {
+          isStreamingFinished: this.isStreamingFinished,
+        };
+        if (this.includesFunctionToolCall) {
+          dataOverrides = {
+            ...dataOverrides,
+            msgtype: APP_BOXEL_COMMAND_MSGTYPE,
+          };
+        }
         const messagePromise = sendMessageEvent(
           this.client,
           this.roomId,
           this.latestContent,
           this.responseEventId,
-          {
-            isStreamingFinished: this.isStreamingFinished,
-          },
+          dataOverrides,
         );
         this.messagePromises.push(messagePromise);
         await messagePromise;
@@ -64,9 +74,13 @@ export class Responder {
     this.responseEventId = initialMessage.event_id;
   }
 
-  async onChunk(chunk: {
-    usage?: { prompt_tokens: number; completion_tokens: number };
-  }) {
+  async onChunk(chunk: OpenAI.Chat.Completions.ChatCompletionChunk) {
+    if (chunk.choices[0].delta?.tool_calls?.[0]?.function) {
+      if (!this.includesFunctionToolCall) {
+        this.includesFunctionToolCall = true;
+        await this.sendMessageEventWithDebouncing();
+      }
+    }
     // This usage value is set *once* and *only once* at the end of the conversation
     // It will be null at all other times.
     if (chunk.usage) {
@@ -110,15 +124,24 @@ export class Responder {
     for (const toolCall of msg.tool_calls || []) {
       log.debug('[Room Timeline] Function call', toolCall);
       try {
+        const functionToolCall = this.deserializeToolCall(toolCall);
+        this.latestContent = [
+          this.latestContent,
+          functionToolCall.arguments['description'],
+        ]
+          .filter(Boolean)
+          .join('\n\n');
         let commandEventPromise = sendCommandEvent(
           this.client,
           this.roomId,
-          this.deserializeToolCall(toolCall),
-          this.initialMessageReplaced ? undefined : this.responseEventId,
+          this.latestContent,
+          functionToolCall,
+          this.responseEventId,
         );
         this.messagePromises.push(commandEventPromise);
         await commandEventPromise;
         this.initialMessageReplaced = true;
+        this.isStreamingFinished = true;
       } catch (error) {
         Sentry.captureException(error);
         this.initialMessageReplaced = true;
@@ -145,7 +168,7 @@ export class Responder {
   }
 
   async finalize(finalContent: string | void | null | undefined) {
-    if (finalContent) {
+    if (finalContent && !this.isStreamingFinished) {
       this.latestContent = cleanContent(finalContent);
       this.isStreamingFinished = true;
       await this.sendMessageEventWithDebouncing();

packages/ai-bot/package.json

@@ -9,7 +9,7 @@
     "@types/stream-chain": "^2.0.1",
     "@types/stream-json": "^1.7.3",
     "matrix-js-sdk": "^31.0.0",
-    "openai": "4.47.1",
+    "openai": "4.81.0",
     "qunit": "^2.18.0",
     "stream-chain": "^2.2.5",
     "stream-json": "^1.8.0",

packages/ai-bot/tests/responding-test.ts

@@ -4,6 +4,7 @@ import { IContent } from 'matrix-js-sdk';
 import { MatrixClient } from '../lib/matrix';
 import FakeTimers from '@sinonjs/fake-timers';
 import { thinkingMessage } from '../constants';
+import { APP_BOXEL_COMMAND_MSGTYPE } from '@cardstack/runtime-common/matrix-constants';
 
 class FakeMatrixClient implements MatrixClient {
   private eventId = 0;
@@ -46,6 +47,14 @@ class FakeMatrixClient implements MatrixClient {
     this.sentEvents = [];
     this.eventId = 0;
   }
+  sendStateEvent(
+    _roomId: string,
+    _eventType: string,
+    _content: IContent,
+    _stateKey: string,
+  ): Promise<{ event_id: string }> {
+    throw new Error('Method not implemented.');
+  }
 }
 
 module('Responding', (hooks) => {
@@ -226,7 +235,6 @@ module('Responding', (hooks) => {
     assert.deepEqual(
       JSON.parse(sentEvents[1].content.data),
       {
-        eventId: '0',
         toolCall: {
           type: 'function',
           id: 'some-tool-call-id',
@@ -261,7 +269,7 @@ module('Responding', (hooks) => {
     );
   });
 
-  test('Sends tool call event separately when content is sent before tool call', async () => {
+  test('Sends tool call event and adds to event when content is sent before tool call', async () => {
     const patchArgs = {
       description: 'A new thing',
       attributes: {
@@ -295,13 +303,37 @@ module('Responding', (hooks) => {
     assert.equal(
       sentEvents.length,
       3,
-      'Thinking message, and tool call event should be sent',
+      'Thinking message, and content, and tool call event should be sent',
     );
     assert.equal(
       sentEvents[0].content.body,
       thinkingMessage,
       'Thinking message should be sent first',
     );
+    assert.notOk(
+      sentEvents[0].content['m.relates_to'],
+      'The tool call event should not replace any message',
+    );
+    assert.equal(
+      sentEvents[1].content.body,
+      'some content',
+      'Content message should be sent next',
+    );
+    assert.strictEqual(
+      sentEvents[1].content['m.relates_to']?.event_id,
+      sentEvents[0].eventId,
+      'The content event should replace the initial message',
+    );
+    assert.equal(
+      sentEvents[2].content.body,
+      'some content\n\nA new thing',
+      'Content message plus function description should be sent next',
+    );
+    assert.strictEqual(
+      sentEvents[2].content['m.relates_to']?.event_id,
+      sentEvents[0].eventId,
+      'The command event should replace the initial message',
+    );
     assert.deepEqual(
       JSON.parse(sentEvents[2].content.data),
       {
@@ -324,23 +356,58 @@ module('Responding', (hooks) => {
       },
       'Tool call event should be sent with correct content',
     );
-    assert.notOk(
-      sentEvents[2].content['m.relates_to'],
-      'The tool call event should not replace any message',
-    );
+  });
+
+  test('Updates message type to command when tool call is in progress', async () => {
+    await responder.initialize();
+    await responder.onChunk({
+      id: '0',
+      created: 0,
+      model: 'gpt-3.5-turbo',
+      object: 'chat.completion.chunk',
+      choices: [
+        {
+          delta: {
+            tool_calls: [
+              {
+                index: 0,
+                type: 'function',
+                function: {
+                  name: 'patchCard',
+                  arguments: '',
+                },
+              },
+            ],
+          },
+          index: 0,
+          finish_reason: 'stop',
+        },
+      ],
+    });
 
+    let sentEvents = fakeMatrixClient.getSentEvents();
     assert.equal(
-      sentEvents[1].content.body,
-      'some content',
-      'Content event should be sent',
+      sentEvents.length,
+      2,
+      'Thinking message and event updating message type should be sent',
+    );
+    assert.equal(
+      sentEvents[0].content.body,
+      thinkingMessage,
+      'Thinking message should be sent first',
+    );
+    assert.deepEqual(
+      sentEvents[1].content['msgtype'],
+      APP_BOXEL_COMMAND_MSGTYPE,
+      'The message type should reflect that the model is preparing a tool call',
     );
     assert.deepEqual(
       sentEvents[1].content['m.relates_to'],
       {
         rel_type: 'm.replace',
         event_id: '0',
       },
-      'The content event should replace the thinking message',
+      'The tool call event should replace the thinking message',
     );
   });
 });

packages/base/matrix-event.gts

@@ -134,7 +134,7 @@ export interface CommandEvent extends BaseMatrixEvent {
   };
 }
 
-export interface CommandMessageContent {
+export type CommandMessageContent = {
   'm.relates_to'?: {
     rel_type: string;
     event_id: string;
@@ -143,11 +143,17 @@ export interface CommandMessageContent {
   format: 'org.matrix.custom.html';
   body: string;
   formatted_body: string;
-  data: {
-    toolCall: FunctionToolCall;
-    eventId: string;
-  };
-}
+} & (
+  | {
+      isStreamingFinished: true | undefined;
+      data: {
+        toolCall: FunctionToolCall;
+      };
+    }
+  | {
+      isStreamingFinished: false;
+    }
+);
 
 export interface CardMessageEvent extends BaseMatrixEvent {
   type: 'm.room.message';