From 998f06de99f55260deb1babd6fd75671ee4e2b0a Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 30 Dec 2024 16:06:30 -0500 Subject: [PATCH] WIP BoxelCommandResultEvent instead of ReactionEvent --- packages/ai-bot/helpers.ts | 117 ++++----- packages/ai-bot/lib/set-title.ts | 23 +- packages/ai-bot/main.ts | 12 +- packages/ai-bot/package.json | 1 + packages/ai-bot/tests/chat-titling-test.ts | 54 ++-- .../ai-bot/tests/history-construction-test.ts | 13 +- .../ai-bot/tests/prompt-construction-test.ts | 49 ++-- packages/base/matrix-event.gts | 35 +-- .../matrix/room-message-command.gts | 247 ++++++++++++++++++ .../app/components/matrix/room-message.gts | 239 ++--------------- .../app/lib/matrix-classes/message-builder.ts | 23 +- .../app/lib/matrix-classes/message-command.ts | 2 +- .../host/app/lib/matrix-classes/message.ts | 4 - packages/host/app/resources/room.ts | 13 +- packages/host/app/services/matrix-service.ts | 42 +-- .../host/tests/acceptance/commands-test.gts | 12 +- .../host/tests/helpers/mock-matrix/_client.ts | 5 +- .../components/ai-assistant-panel-test.gts | 29 +- packages/matrix/helpers/matrix-constants.ts | 1 + packages/runtime-common/matrix-constants.ts | 1 + pnpm-lock.yaml | 7 + 21 files changed, 492 insertions(+), 437 deletions(-) create mode 100644 packages/host/app/components/matrix/room-message-command.gts diff --git a/packages/ai-bot/helpers.ts b/packages/ai-bot/helpers.ts index 00d122cdad..23b2d4e98b 100644 --- a/packages/ai-bot/helpers.ts +++ b/packages/ai-bot/helpers.ts @@ -8,20 +8,19 @@ import type { MatrixEvent as DiscreteMatrixEvent, CardFragmentContent, CommandEvent, - CommandResultEvent, - ReactionEvent, Tool, SkillsConfigEvent, + CommandResultEvent, } from 'https://cardstack.com/base/matrix-event'; import { MatrixEvent, type IRoomEvent } from 'matrix-js-sdk'; import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions'; import * as Sentry from '@sentry/node'; import { logger } from '@cardstack/runtime-common'; +import { APP_BOXEL_COMMAND_RESULT_EVENT_TYPE } from '../runtime-common/matrix-constants'; import { APP_BOXEL_CARDFRAGMENT_MSGTYPE, APP_BOXEL_MESSAGE_MSGTYPE, APP_BOXEL_COMMAND_MSGTYPE, - APP_BOXEL_COMMAND_RESULT_MSGTYPE, APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, } from '@cardstack/runtime-common/matrix-constants'; @@ -140,6 +139,15 @@ export function constructHistory( } } let event = { ...rawEvent } as DiscreteMatrixEvent; + if (event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE) { + let { cardEventId } = event.content.data; + if (cardEventId) { + event.content.data.card = serializedCardFromFragments( + cardEventId, + cardFragments, + ); + } + } if (event.type !== 'm.room.message') { continue; } @@ -358,48 +366,11 @@ export function getToolChoice( return 'auto'; } -export function isCommandResultEvent( - event: DiscreteMatrixEvent, -): event is CommandResultEvent { - return ( - event.type === 'm.room.message' && - typeof event.content === 'object' && - event.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE - ); -} - -export function isReactionEvent( - event: DiscreteMatrixEvent, -): event is ReactionEvent { - return ( - event.type === 'm.reaction' && - event.content['m.relates_to'].rel_type === 'm.annotation' - ); -} - -function getReactionStatus( - commandEvent: DiscreteMatrixEvent, - history: DiscreteMatrixEvent[], -) { - let maybeReactionEvent = history.find((e) => { - if ( - isReactionEvent(e) && - e.content['m.relates_to']?.event_id === commandEvent.event_id - ) { - return true; - } - return false; - }); - return maybeReactionEvent && isReactionEvent(maybeReactionEvent) - ? maybeReactionEvent.content['m.relates_to'].key - : undefined; -} - function getCommandResult( commandEvent: CommandEvent, history: DiscreteMatrixEvent[], ) { - let maybeCommandResultEvent = history.find((e) => { + let commandResultEvent = history.find((e) => { if ( isCommandResultEvent(e) && e.content['m.relates_to']?.event_id === commandEvent.event_id @@ -407,11 +378,8 @@ function getCommandResult( return true; } return false; - }); - return maybeCommandResultEvent && - isCommandResultEvent(maybeCommandResultEvent) - ? maybeCommandResultEvent.content.result - : undefined; + }) as CommandResultEvent | undefined; + return commandResultEvent; } function toToolCall(event: CommandEvent): ChatCompletionMessageToolCall { @@ -429,21 +397,23 @@ function toPromptMessageWithToolResult( event: CommandEvent, history: DiscreteMatrixEvent[], ): OpenAIPromptMessage { - let commandResult = getCommandResult(event as CommandEvent, history); + let commandResult = getCommandResult(event, history); + let content = 'pending'; if (commandResult) { - return { - role: 'tool', - content: commandResult, - tool_call_id: event.content.data.toolCall.id, - }; - } else { - let reactionStatus = getReactionStatus(event, history); - return { - role: 'tool', - content: reactionStatus ?? 'pending', - tool_call_id: event.content.data.toolCall.id, - }; + let status = commandResult.content['m.relates_to']?.key; + if (commandResult.content.data.card) { + content = `Command ${status}, with result card: ${JSON.stringify( + commandResult.content.data.card, + )}.\n`; + } else { + content = `Command ${status}.\n`; + } } + return { + role: 'tool', + content, + tool_call_id: event.content.data.toolCall.id, + }; } export function getModifyPrompt( @@ -570,24 +540,13 @@ export function cleanContent(content: string) { return content.trim(); } -export const isCommandReactionEvent = (event?: MatrixEvent) => { - if (event === undefined) { - return false; - } - let content = event.getContent(); - return ( - event.getType() === 'm.reaction' && - content['m.relates_to']?.rel_type === 'm.annotation' - ); -}; - -export const isCommandReactionStatusApplied = (event?: MatrixEvent) => { +export const isCommandResultStatusApplied = (event?: MatrixEvent) => { if (event === undefined) { return false; } - let content = event.getContent(); return ( - isCommandReactionEvent(event) && content['m.relates_to']?.key === 'applied' + isCommandResultEvent(event.event as DiscreteMatrixEvent) && + event.getContent()['m.relates_to']?.key === 'applied' ); }; @@ -603,3 +562,15 @@ export function isCommandEvent( typeof event.content.data.toolCall === 'object' ); } + +export function isCommandResultEvent( + event?: DiscreteMatrixEvent, +): event is CommandResultEvent { + if (event === undefined) { + return false; + } + return ( + event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.rel_type === 'm.annotation' + ); +} diff --git a/packages/ai-bot/lib/set-title.ts b/packages/ai-bot/lib/set-title.ts index e207a80f2d..61dac91c8d 100644 --- a/packages/ai-bot/lib/set-title.ts +++ b/packages/ai-bot/lib/set-title.ts @@ -1,18 +1,23 @@ -import { type MatrixEvent, type IEventRelation } from 'matrix-js-sdk'; +import { + type MatrixEvent, + type IEventRelation, + IRoomEvent, +} from 'matrix-js-sdk'; import OpenAI from 'openai'; import { type OpenAIPromptMessage, - isCommandReactionStatusApplied, + isCommandResultStatusApplied, attachedCardsToMessage, isCommandEvent, getRelevantCards, } from '../helpers'; import { MatrixClient } from './matrix'; import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/base/matrix-event'; +import { ChatCompletionMessageParam } from 'openai/resources'; const SET_TITLE_SYSTEM_MESSAGE = `You are a chat titling system, you must read the conversation and return a suggested title of no more than six words. -Do NOT say talk or discussion or discussing or chat or chatting, this is implied by the context. -The user can optionally apply 'patchCard' by sending data about fields to update. +Do NOT say talk or discussion or discussing or chat or chatting, this is implied by the context. +The user can optionally apply 'patchCard' by sending data about fields to update. Explain the general actions and user intent. If 'patchCard' was used, express the title in an active sentence. Do NOT use the word "patch" in the title.`; export async function setTitle( @@ -39,7 +44,7 @@ export async function setTitle( let result = await openai.chat.completions.create( { model: 'gpt-4o', - messages: startOfConversation, + messages: startOfConversation as ChatCompletionMessageParam[], stream: false, }, { @@ -120,7 +125,7 @@ export const getLatestCommandApplyMessage = ( return []; }; -export const roomTitleAlreadySet = (rawEventLog: DiscreteMatrixEvent[]) => { +export const roomTitleAlreadySet = (rawEventLog: IRoomEvent[]) => { return ( rawEventLog.filter((event) => event.type === 'm.room.name').length > 1 ?? false @@ -128,7 +133,7 @@ export const roomTitleAlreadySet = (rawEventLog: DiscreteMatrixEvent[]) => { }; const userAlreadyHasSentNMessages = ( - rawEventLog: DiscreteMatrixEvent[], + rawEventLog: IRoomEvent[], botUserId: string, n = 5, ) => { @@ -140,12 +145,12 @@ const userAlreadyHasSentNMessages = ( }; export function shouldSetRoomTitle( - rawEventLog: DiscreteMatrixEvent[], + rawEventLog: IRoomEvent[], aiBotUserId: string, event?: MatrixEvent, ) { return ( - (isCommandReactionStatusApplied(event) || + (isCommandResultStatusApplied(event) || userAlreadyHasSentNMessages(rawEventLog, aiBotUserId)) && !roomTitleAlreadySet(rawEventLog) ); diff --git a/packages/ai-bot/main.ts b/packages/ai-bot/main.ts index 82dc5a5877..447e64d37e 100644 --- a/packages/ai-bot/main.ts +++ b/packages/ai-bot/main.ts @@ -11,7 +11,7 @@ import { logger, aiBotUsername } from '@cardstack/runtime-common'; import { type PromptParts, constructHistory, - isCommandReactionStatusApplied, + isCommandResultStatusApplied, getPromptParts, extractCardFragmentsFromEvents, } from './helpers'; @@ -30,6 +30,8 @@ import * as Sentry from '@sentry/node'; import { getAvailableCredits, saveUsageCost } from './lib/ai-billing'; import { PgAdapter } from '@cardstack/postgres'; +import { ChatCompletionMessageParam } from 'openai/resources'; +import { OpenAIError } from 'openai/error'; let log = logger('ai-bot'); @@ -69,12 +71,12 @@ class Assistant { if (prompt.tools.length === 0) { return this.openai.beta.chat.completions.stream({ model: prompt.model, - messages: prompt.messages, + messages: prompt.messages as ChatCompletionMessageParam[], }); } else { return this.openai.beta.chat.completions.stream({ model: prompt.model, - messages: prompt.messages, + messages: prompt.messages as ChatCompletionMessageParam[], tools: prompt.tools, tool_choice: prompt.toolChoice, }); @@ -250,7 +252,7 @@ Common issues are: finalContent = await runner.finalContent(); await responder.finalize(finalContent); } catch (error) { - await responder.onError(error); + await responder.onError(error as OpenAIError); } finally { if (generationId) { assistant.trackAiUsageCost(senderMatrixUserId, generationId); @@ -278,7 +280,7 @@ Common issues are: if (!room) { return; } - if (!isCommandReactionStatusApplied(event)) { + if (!isCommandResultStatusApplied(event)) { return; } log.info( diff --git a/packages/ai-bot/package.json b/packages/ai-bot/package.json index 6f7adb5927..38ac39f742 100644 --- a/packages/ai-bot/package.json +++ b/packages/ai-bot/package.json @@ -18,6 +18,7 @@ }, "devDependencies": { "@sinonjs/fake-timers": "^11.2.2", + "@types/qunit": "^2.19.12", "@types/sinonjs__fake-timers": "^8.1.5", "qunit": "^2.18.0" }, diff --git a/packages/ai-bot/tests/chat-titling-test.ts b/packages/ai-bot/tests/chat-titling-test.ts index f395c62a6f..c66eeb85e8 100644 --- a/packages/ai-bot/tests/chat-titling-test.ts +++ b/packages/ai-bot/tests/chat-titling-test.ts @@ -1,7 +1,11 @@ import { module, test, assert } from 'qunit'; import { shouldSetRoomTitle } from '../lib/set-title'; import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/base/matrix-event'; -import { APP_BOXEL_COMMAND_MSGTYPE } from '@cardstack/runtime-common/matrix-constants'; +import { + APP_BOXEL_COMMAND_MSGTYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, +} from '@cardstack/runtime-common/matrix-constants'; +import { IEvent, IRoomEvent, MatrixEvent } from 'matrix-js-sdk'; module('shouldSetRoomTitle', () => { test('Do not set a title when there is no content', () => { @@ -10,7 +14,7 @@ module('shouldSetRoomTitle', () => { }); test('Do not set a title when there is little content', () => { - const eventLog: DiscreteMatrixEvent[] = [ + const eventLog: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -33,7 +37,7 @@ module('shouldSetRoomTitle', () => { }); test('Do not set a title when there are more than 5 messages but they are state/invites/etc', () => { - const eventLog: DiscreteMatrixEvent[] = [ + const eventLog: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -78,7 +82,7 @@ module('shouldSetRoomTitle', () => { }, sender: '@user:localhost', room_id: 'room1', - state_key: 'a', + state_key: '', unsigned: { age: 1000, }, @@ -90,7 +94,7 @@ module('shouldSetRoomTitle', () => { content: {}, sender: '@user:localhost', room_id: 'room1', - state_key: 'b', + state_key: '', unsigned: { age: 1000, }, @@ -105,7 +109,7 @@ module('shouldSetRoomTitle', () => { }, sender: '@user:localhost', room_id: 'room1', - state_key: 'c', + state_key: '', unsigned: { age: 1000, }, @@ -120,7 +124,7 @@ module('shouldSetRoomTitle', () => { }, sender: '@user:localhost', room_id: 'room1', - state_key: 'd', + state_key: '', unsigned: { age: 1000, }, @@ -130,7 +134,7 @@ module('shouldSetRoomTitle', () => { }); test('Do not set a title when there are under 5 user messages but more than 5 total messages', () => { - const eventLog: DiscreteMatrixEvent[] = [ + const eventLog: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -239,7 +243,7 @@ module('shouldSetRoomTitle', () => { }); test('Set a title when there are 5 or more user messages', () => { - const eventLog: DiscreteMatrixEvent[] = [ + const eventLog: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -348,7 +352,7 @@ module('shouldSetRoomTitle', () => { }); test('Title is not set if the bot has sent ONLY a command', () => { - const eventLog: DiscreteMatrixEvent[] = [ + const eventLog: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -403,21 +407,19 @@ module('shouldSetRoomTitle', () => { }); test('Set a title if the user applied a command', () => { - let patchReactionEvent = { - getContent() { - return { - 'm.relates_to': { - event_id: '1', - key: 'applied', - rel_type: 'm.annotation', - }, - }; - }, - getType() { - return 'm.reaction'; + let patchCommandResultEvent: Partial = { + type: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, + content: { + 'm.relates_to': { + event_id: '1', + key: 'applied', + rel_type: 'm.annotation', + }, + data: {}, + msgtype: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, }, }; - const eventLog: DiscreteMatrixEvent[] = [ + const eventLog: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -469,7 +471,11 @@ module('shouldSetRoomTitle', () => { }, ]; assert.true( - shouldSetRoomTitle(eventLog, '@aibot:localhost', patchReactionEvent), + shouldSetRoomTitle( + eventLog, + '@aibot:localhost', + new MatrixEvent(patchCommandResultEvent), + ), ); }); }); diff --git a/packages/ai-bot/tests/history-construction-test.ts b/packages/ai-bot/tests/history-construction-test.ts index 1694a49a9d..afb7a10344 100644 --- a/packages/ai-bot/tests/history-construction-test.ts +++ b/packages/ai-bot/tests/history-construction-test.ts @@ -10,7 +10,7 @@ import { APP_BOXEL_MESSAGE_MSGTYPE, } from '@cardstack/runtime-common/matrix-constants'; -import { type IRoomEvent } from 'matrix-js-sdk'; +import { EventStatus, type IRoomEvent } from 'matrix-js-sdk'; import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/base/matrix-event'; module('constructHistory', () => { @@ -38,6 +38,7 @@ module('constructHistory', () => { unsigned: { age: 1000, }, + status: EventStatus.SENT, }, { type: 'm.room.join_rules', @@ -50,6 +51,7 @@ module('constructHistory', () => { unsigned: { age: 1001, }, + status: EventStatus.SENT, }, { type: 'm.room.member', @@ -65,6 +67,7 @@ module('constructHistory', () => { unsigned: { age: 1002, }, + status: EventStatus.SENT, }, ]; @@ -74,7 +77,7 @@ module('constructHistory', () => { }); test('should return an array with a single message event when the input array contains only one message event', () => { - const eventlist: DiscreteMatrixEvent[] = [ + const eventlist: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -100,7 +103,7 @@ module('constructHistory', () => { }); test('should return an array with all message events when the input array contains multiple message events', () => { - const history: DiscreteMatrixEvent[] = [ + const history: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -160,7 +163,7 @@ module('constructHistory', () => { }); test('should return an array with all message events when the input array contains multiple events with the same origin_server_ts', () => { - const history: DiscreteMatrixEvent[] = [ + const history: IRoomEvent[] = [ { type: 'm.room.message', event_id: '1', @@ -220,7 +223,7 @@ module('constructHistory', () => { }); test('should return an array of DiscreteMatrixEvent objects with no duplicates based on event_id even when m.relates_to is present and include senders and origin_server_ts', () => { - const history: DiscreteMatrixEvent[] = [ + const history: IRoomEvent[] = [ // this event will _not_ replace event_id 2 since it's timestamp is before event_id 2 { event_id: '1', diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 5acda2222d..a023738a63 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -15,6 +15,7 @@ import { APP_BOXEL_MESSAGE_MSGTYPE, APP_BOXEL_COMMAND_RESULT_MSGTYPE, APP_BOXEL_COMMAND_MSGTYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, } from '@cardstack/runtime-common/matrix-constants'; import type { @@ -1309,7 +1310,6 @@ test('Return host result of tool call back to open ai', () => { age: 20470, }, event_id: '$p_NQ4tvokzQrIkT24Wj08mdAxBBvmdLOz6ph7UQfMDw', - user_id: '@tintinthong:localhost', age: 20470, }, { @@ -1341,7 +1341,6 @@ test('Return host result of tool call back to open ai', () => { transaction_id: 'm1722242836705.8', }, event_id: 'message-event-id-1', - user_id: '@aibot:localhost', age: 17305, }, { @@ -1458,7 +1457,6 @@ test('Return host result of tool call back to open ai', () => { age: 6614, }, event_id: '$FO2XfB0xFiTpm5FmOUiWQqFh_DPQSr4zix41Vj3eqNc', - user_id: '@tintinthong:localhost', age: 6614, }, { @@ -1500,11 +1498,10 @@ test('Return host result of tool call back to open ai', () => { transaction_id: 'm1722242849075.10', }, event_id: 'command-event-id-1', - user_id: '@ai-bot:localhost', age: 4938, }, { - type: 'm.room.message', + type: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, room_id: 'room-id-1', sender: '@tintinthong:localhost', content: { @@ -1513,19 +1510,43 @@ test('Return host result of tool call back to open ai', () => { rel_type: 'm.annotation', key: 'applied', }, - body: 'Command Results from command event $H7dH0ZzG0W3M_1k_YRjnDOirWRthYvWq7TKmfAfhQqw', - formatted_body: - '

Command Results from command event $H7dH0ZzG0W3M_1k_YRjnDOirWRthYvWq7TKmfAfhQqw

\n', msgtype: APP_BOXEL_COMMAND_RESULT_MSGTYPE, - result: - '[{"data":{"type":"card","id":"http://localhost:4201/drafts/Author/1","attributes":{"firstName":"Alice","lastName":"Enwunder","photo":null,"body":"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.","description":null,"thumbnailURL":null},"meta":{"adoptsFrom":{"module":"../author","name":"Author"}}}}]', + data: { + card: JSON.stringify({ + data: { + type: 'card', + attributes: { + title: 'Search Results', + description: 'Here are the search results', + results: [ + { + data: { + type: 'card', + id: 'http://localhost:4201/drafts/Author/1', + attributes: { + firstName: 'Alice', + lastName: 'Enwunder', + photo: null, + body: 'Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.', + description: null, + thumbnailURL: null, + }, + meta: { + adoptsFrom: { module: '../author', name: 'Author' }, + }, + }, + }, + ], + }, + }, + }), + }, }, origin_server_ts: 1722242853988, unsigned: { age: 44, }, event_id: 'command-result-id-1', - user_id: '@tintinthong:localhost', age: 44, }, ]; @@ -1533,10 +1554,8 @@ test('Return host result of tool call back to open ai', () => { const result = getModifyPrompt(history, '@ai-bot:localhost', tools); assert.equal(result[5].role, 'tool'); assert.equal(result[5].tool_call_id, 'tool-call-id-1'); - assert.equal( - result[5].content, - '[{"data":{"type":"card","id":"http://localhost:4201/drafts/Author/1","attributes":{"firstName":"Alice","lastName":"Enwunder","photo":null,"body":"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.","description":null,"thumbnailURL":null},"meta":{"adoptsFrom":{"module":"../author","name":"Author"}}}}]', - ); + const expected = `Command applied, with result card: "{\\"data\\":{\\"type\\":\\"card\\",\\"attributes\\":{\\"title\\":\\"Search Results\\",\\"description\\":\\"Here are the search results\\",\\"results\\":[{\\"data\\":{\\"type\\":\\"card\\",\\"id\\":\\"http://localhost:4201/drafts/Author/1\\",\\"attributes\\":{\\"firstName\\":\\"Alice\\",\\"lastName\\":\\"Enwunder\\",\\"photo\\":null,\\"body\\":\\"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.\\",\\"description\\":null,\\"thumbnailURL\\":null},\\"meta\\":{\\"adoptsFrom\\":{\\"module\\":\\"../author\\",\\"name\\":\\"Author\\"}}}}]}}}".\n`; + assert.equal(result[5].content, expected); }); test('Tools remain available in prompt parts even when not in last message', () => { diff --git a/packages/base/matrix-event.gts b/packages/base/matrix-event.gts index 8afecdcaca..07a1dda771 100644 --- a/packages/base/matrix-event.gts +++ b/packages/base/matrix-event.gts @@ -9,6 +9,7 @@ import { APP_BOXEL_CARD_FORMAT, APP_BOXEL_CARDFRAGMENT_MSGTYPE, APP_BOXEL_COMMAND_MSGTYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, APP_BOXEL_COMMAND_RESULT_MSGTYPE, APP_BOXEL_MESSAGE_MSGTYPE, APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, @@ -146,26 +147,6 @@ export interface CommandMessageContent { }; } -export interface ReactionEvent extends BaseMatrixEvent { - type: 'm.reaction'; - content: ReactionEventContent; -} - -export interface ReactionEventContent { - 'm.relates_to': { - event_id: string; - key: string; - rel_type: 'm.annotation'; - }; -} - -export type CommandReactionEventContent = ReactionEventContent & { - msgtype: 'org.boxel.command_result'; - data: { - card_event_id: string | null; - }; -}; - export interface CardMessageEvent extends BaseMatrixEvent { type: 'm.room.message'; content: CardMessageContent | CardFragmentContent; @@ -246,7 +227,7 @@ export interface SkillsConfigEvent extends RoomStateEvent { } export interface CommandResultEvent extends BaseMatrixEvent { - type: 'm.room.message'; + type: typeof APP_BOXEL_COMMAND_RESULT_EVENT_TYPE; content: CommandResultContent; unsigned: { age: number; @@ -265,10 +246,14 @@ export interface CommandResultContent { event_id: string; }; }; - formatted_body: string; - body: string; + data: { + // we use this field over the wire since the matrix message protocol + // limits us to 65KB per message + cardEventId?: string; + // we materialize this field on the server + card?: LooseSingleCardDocument; + }; msgtype: typeof APP_BOXEL_COMMAND_RESULT_MSGTYPE; - result: any; } export type MatrixEvent = @@ -277,7 +262,7 @@ export type MatrixEvent = | RoomPowerLevels | MessageEvent | CommandEvent - | ReactionEvent + | CommandResultEvent | CardMessageEvent | RoomNameEvent | RoomTopicEvent diff --git a/packages/host/app/components/matrix/room-message-command.gts b/packages/host/app/components/matrix/room-message-command.gts new file mode 100644 index 0000000000..b42213e72b --- /dev/null +++ b/packages/host/app/components/matrix/room-message-command.gts @@ -0,0 +1,247 @@ +import { fn } from '@ember/helper'; +import { on } from '@ember/modifier'; +import { action } from '@ember/object'; +import { inject as service } from '@ember/service'; +import Component from '@glimmer/component'; + +import { cached, tracked } from '@glimmer/tracking'; + +import { task } from 'ember-concurrency'; + +import perform from 'ember-concurrency/helpers/perform'; +import { modifier } from 'ember-modifier'; + +import { Button } from '@cardstack/boxel-ui/components'; + +import { cn } from '@cardstack/boxel-ui/helpers'; +import { Copy as CopyIcon } from '@cardstack/boxel-ui/icons'; + +import MessageCommand from '@cardstack/host/lib/matrix-classes/message-command'; +import type { MonacoEditorOptions } from '@cardstack/host/modifiers/monaco'; +import monacoModifier from '@cardstack/host/modifiers/monaco'; +import type CommandService from '@cardstack/host/services/command-service'; +import type MatrixService from '@cardstack/host/services/matrix-service'; +import type MonacoService from '@cardstack/host/services/monaco-service'; + +import { type MonacoSDK } from '@cardstack/host/services/monaco-service'; + +import ApplyButton from '../ai-assistant/apply-button'; +import { type ApplyButtonState } from '../ai-assistant/apply-button'; + +interface Signature { + Element: HTMLDivElement; + Args: { + messageCommand: MessageCommand; + messageIndex: number | undefined; + roomId: string; + isError?: boolean; + isPending?: boolean; + failedCommandState: Error | undefined; + monacoSDK: MonacoSDK; + currentEditor: number | undefined; + setCurrentEditor: (editor: number | undefined) => void; + }; +} + +export default class RoomMessageCommand extends Component { + @service private declare commandService: CommandService; + @service private declare matrixService: MatrixService; + @service private declare monacoService: MonacoService; + + @tracked private isDisplayingCode = false; + + editorDisplayOptions: MonacoEditorOptions = { + wordWrap: 'on', + wrappingIndent: 'indent', + fontWeight: 'bold', + scrollbar: { + alwaysConsumeMouseWheel: false, + }, + }; + + private get previewCommandCode() { + let { name, payload } = this.args.messageCommand; + return JSON.stringify({ name, payload }, null, 2); + } + + private copyToClipboard = task(async () => { + await navigator.clipboard.writeText(this.previewCommandCode); + }); + + @cached + private get applyButtonState(): ApplyButtonState { + if (this.args.failedCommandState) { + return 'failed'; + } + return this.args.messageCommand?.status ?? 'ready'; + } + + @action private viewCodeToggle() { + this.isDisplayingCode = !this.isDisplayingCode; + if (this.isDisplayingCode) { + this.args.setCurrentEditor(this.args.messageIndex); + } + } + + private get getCommandResultComponent() { + let commandResultCardEventId = + this.args.messageCommand?.commandResultCardEventId; + if (!commandResultCardEventId) { + return undefined; + } + // TODO: load the card from the the room (commandResultCardEventId) + return undefined; + // return commandResult.constructor.getComponent(commandResult); + } + + // TODO need to reevalutate this modifier--do we want to hijack the scroll + // when the user views the code? + private scrollBottomIntoView = modifier((element: HTMLElement) => { + if (this.args.currentEditor !== this.args.messageIndex) { + return; + } + + let height = this.monacoService.getContentHeight(); + if (!height || height < 0) { + return; + } + element.style.height = `${height}px`; + + let outerContainer = document.getElementById( + `message-container-${this.args.messageIndex}`, + ); + if (!outerContainer) { + return; + } + this.scrollIntoView(outerContainer); + }); + + private scrollIntoView(element: HTMLElement) { + let { top, bottom } = element.getBoundingClientRect(); + let isVerticallyInView = top >= 0 && bottom <= window.innerHeight; + + if (!isVerticallyInView) { + element.scrollIntoView({ block: 'end' }); + } + } + + +} diff --git a/packages/host/app/components/matrix/room-message.gts b/packages/host/app/components/matrix/room-message.gts index 8dc6d765b7..db00bc2ce4 100644 --- a/packages/host/app/components/matrix/room-message.gts +++ b/packages/host/app/components/matrix/room-message.gts @@ -1,39 +1,34 @@ import { fn } from '@ember/helper'; -import { on } from '@ember/modifier'; -import { action } from '@ember/object'; import { service } from '@ember/service'; import { htmlSafe } from '@ember/template'; import Component from '@glimmer/component'; -import { tracked, cached } from '@glimmer/tracking'; +import { cached, tracked } from '@glimmer/tracking'; import { task } from 'ember-concurrency'; import perform from 'ember-concurrency/helpers/perform'; -import { modifier } from 'ember-modifier'; import { trackedFunction } from 'ember-resources/util/function'; -import { Avatar, Button } from '@cardstack/boxel-ui/components'; -import { Copy as CopyIcon } from '@cardstack/boxel-ui/icons'; +import { Avatar } from '@cardstack/boxel-ui/components'; -import { isCardInstance, markdownToHtml } from '@cardstack/runtime-common'; +import { bool } from '@cardstack/boxel-ui/helpers'; + +import { markdownToHtml } from '@cardstack/runtime-common'; import { Message } from '@cardstack/host/lib/matrix-classes/message'; import MessageCommand from '@cardstack/host/lib/matrix-classes/message-command'; -import monacoModifier from '@cardstack/host/modifiers/monaco'; -import type { MonacoEditorOptions } from '@cardstack/host/modifiers/monaco'; import CommandService from '@cardstack/host/services/command-service'; import type MatrixService from '@cardstack/host/services/matrix-service'; -import type MonacoService from '@cardstack/host/services/monaco-service'; import { type MonacoSDK } from '@cardstack/host/services/monaco-service'; import type OperatorModeStateService from '@cardstack/host/services/operator-mode-state-service'; import { type CardDef } from 'https://cardstack.com/base/card-api'; -import ApplyButton from '../ai-assistant/apply-button'; -import { type ApplyButtonState } from '../ai-assistant/apply-button'; import AiAssistantMessage from '../ai-assistant/message'; import { aiBotUserId } from '../ai-assistant/panel'; +import RoomMessageCommand from './room-message-command'; + interface Signature { Element: HTMLDivElement; Args: { @@ -86,14 +81,6 @@ export default class RoomMessage extends Component { return this.args.message.author.userId === aiBotUserId; } - private get getComponent() { - let { commandResult } = this.args.message; - if (!commandResult || !isCardInstance(commandResult)) { - return undefined; - } - return commandResult.constructor.getComponent(commandResult); - } - - editorDisplayOptions: MonacoEditorOptions = { - wordWrap: 'on', - wrappingIndent: 'indent', - fontWeight: 'bold', - scrollbar: { - alwaysConsumeMouseWheel: false, - }, - }; - @service private declare operatorModeStateService: OperatorModeStateService; @service private declare matrixService: MatrixService; - @service private declare monacoService: MonacoService; @service declare commandService: CommandService; - @tracked private isDisplayingCode = false; - - private copyToClipboard = task(async () => { - await navigator.clipboard.writeText(this.previewCommandCode); - }); - private loadMessageResources = trackedFunction(this, async () => { let cards: CardDef[] = []; let errors: { id: string; error: Error }[] = []; @@ -308,6 +177,16 @@ export default class RoomMessage extends Component { return this.loadMessageResources.value; } + @cached + private get failedCommandState() { + if (!this.args.message.command?.eventId) { + return undefined; + } + return this.matrixService.failedCommandState.get( + this.args.message.command.eventId, + ); + } + private get errorMessage() { if (this.failedCommandState) { return `Failed to apply changes. ${this.failedCommandState.message}`; @@ -332,73 +211,11 @@ export default class RoomMessage extends Component { .join(', '); } - private get previewCommandCode() { - if (!this.command) { - return JSON.stringify({}, null, 2); - } - let { name, payload } = this.command; - return JSON.stringify({ name, payload }, null, 2); - } - get command() { return this.args.message.command; } - @cached - private get failedCommandState() { - if (!this.command?.eventId) { - return undefined; - } - return this.matrixService.failedCommandState.get(this.command.eventId); - } - run = task(async (command: MessageCommand, roomId: string) => { return this.commandService.run.unlinked().perform(command, roomId); }); - - @cached - private get applyButtonState(): ApplyButtonState { - if (this.failedCommandState) { - return 'failed'; - } - return this.command?.status ?? 'ready'; - } - - @action private viewCodeToggle() { - this.isDisplayingCode = !this.isDisplayingCode; - if (this.isDisplayingCode) { - this.args.setCurrentEditor(this.args.message.index); - } - } - - // TODO need to reevalutate this modifier--do we want to hijack the scroll - // when the user views the code? - private scrollBottomIntoView = modifier((element: HTMLElement) => { - if (this.args.currentEditor !== this.args.message.index) { - return; - } - - let height = this.monacoService.getContentHeight(); - if (!height || height < 0) { - return; - } - element.style.height = `${height}px`; - - let outerContainer = document.getElementById( - `message-container-${this.args.index}`, - ); - if (!outerContainer) { - return; - } - this.scrollIntoView(outerContainer); - }); - - private scrollIntoView(element: HTMLElement) { - let { top, bottom } = element.getBoundingClientRect(); - let isVerticallyInView = top >= 0 && bottom <= window.innerHeight; - - if (!isVerticallyInView) { - element.scrollIntoView({ block: 'end' }); - } - } } diff --git a/packages/host/app/lib/matrix-classes/message-builder.ts b/packages/host/app/lib/matrix-classes/message-builder.ts index e97dcf2761..26fd6a51dd 100644 --- a/packages/host/app/lib/matrix-classes/message-builder.ts +++ b/packages/host/app/lib/matrix-classes/message-builder.ts @@ -8,6 +8,7 @@ import { LooseSingleCardDocument } from '@cardstack/runtime-common'; import { APP_BOXEL_COMMAND_MSGTYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, APP_BOXEL_COMMAND_RESULT_MSGTYPE, APP_BOXEL_MESSAGE_MSGTYPE, } from '@cardstack/runtime-common/matrix-constants'; @@ -20,10 +21,10 @@ import type { CardMessageContent, CardMessageEvent, CommandEvent, - CommandReactionEventContent, + CommandResultEvent, + CommandResultContent, MatrixEvent as DiscreteMatrixEvent, MessageEvent, - ReactionEvent, } from 'https://cardstack.com/base/matrix-event'; import { RoomMember } from './member'; @@ -138,30 +139,30 @@ export default class MessageBuilder { let annotation = this.builderContext.events.find((e: any) => { let r = e.content['m.relates_to']; return ( - e.type === 'm.reaction' && - e.content.msgtype === 'org.boxel.command_result' && + e.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE && + e.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE && r?.rel_type === 'm.annotation' && (r?.event_id === event.content.data.eventId || r?.event_id === event.event_id || r?.event_id === this.builderContext.effectiveEventId) ); - }) as ReactionEvent | undefined; + }) as CommandResultEvent | undefined; let status: CommandStatus = 'ready'; - let reactionContent = annotation?.content as - | CommandReactionEventContent + let commandResultContent = annotation?.content as + | CommandResultContent | undefined; - if (reactionContent && reactionContent['m.relates_to'].key === 'applied') { + if (commandResultContent?.['m.relates_to']?.key === 'applied') { status = 'applied'; } - let commandResultCardId: string | undefined = - reactionContent?.data.card_event_id ?? undefined; + let commandResultCardEventId: string | undefined = + commandResultContent?.data.cardEventId ?? undefined; let messageCommand = new MessageCommand( command.id, command.name, command.arguments, this.builderContext.effectiveEventId, status, - commandResultCardId, + commandResultCardEventId, getOwner(this)!, ); return messageCommand; diff --git a/packages/host/app/lib/matrix-classes/message-command.ts b/packages/host/app/lib/matrix-classes/message-command.ts index 1759a4f75e..b9f193e3ff 100644 --- a/packages/host/app/lib/matrix-classes/message-command.ts +++ b/packages/host/app/lib/matrix-classes/message-command.ts @@ -13,7 +13,7 @@ export default class MessageCommand { public payload: any, //arguments of toolCall. Its not called arguments due to lint public eventId: string, private commandStatus: CommandStatus, - public commandResultCardId: string | undefined, + public commandResultCardEventId: string | undefined, owner: Owner, ) { setOwner(this, owner); diff --git a/packages/host/app/lib/matrix-classes/message.ts b/packages/host/app/lib/matrix-classes/message.ts index f67c08372f..36191a49af 100644 --- a/packages/host/app/lib/matrix-classes/message.ts +++ b/packages/host/app/lib/matrix-classes/message.ts @@ -4,8 +4,6 @@ import { getCard } from '@cardstack/runtime-common'; import { CardDef } from 'https://cardstack.com/base/card-api'; -import type { CommandResult } from 'https://cardstack.com/base/command-result'; - import { RoomMember } from './member'; import MessageCommand from './message-command'; @@ -39,7 +37,6 @@ interface RoomMessageOptional { errorMessage?: string; clientGeneratedId?: string | null; command?: MessageCommand | null; - commandResult?: CommandResult | null; } export class Message implements RoomMessageInterface { @@ -51,7 +48,6 @@ export class Message implements RoomMessageInterface { errorMessage?: string; clientGeneratedId?: string; command?: MessageCommand | null; - commandResult?: CommandResult | null; author: RoomMember; formattedMessage: string; diff --git a/packages/host/app/resources/room.ts b/packages/host/app/resources/room.ts index ed83b944e9..2732d26ac3 100644 --- a/packages/host/app/resources/room.ts +++ b/packages/host/app/resources/room.ts @@ -9,15 +9,11 @@ import { TrackedMap } from 'tracked-built-ins'; import { type LooseSingleCardDocument } from '@cardstack/runtime-common'; -import { - APP_BOXEL_CARDFRAGMENT_MSGTYPE, - APP_BOXEL_COMMAND_RESULT_MSGTYPE, -} from '@cardstack/runtime-common/matrix-constants'; +import { APP_BOXEL_CARDFRAGMENT_MSGTYPE } from '@cardstack/runtime-common/matrix-constants'; import type { CardFragmentContent, CommandEvent, - CommandResultEvent, MatrixEvent as DiscreteMatrixEvent, RoomCreateEvent, RoomNameEvent, @@ -251,7 +247,7 @@ export class RoomResource extends Resource { private async loadRoomMessage( roomId: string, - event: MessageEvent | CommandEvent | CardMessageEvent | CommandResultEvent, + event: MessageEvent | CommandEvent | CardMessageEvent, index: number, ) { let effectiveEventId = event.event_id; @@ -297,11 +293,6 @@ export class RoomResource extends Resource { } return; } - if (event.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE) { - //don't display command result in the room as a message - return; - } - let author = this.upsertRoomMember({ roomId, userId: event.sender, diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts index d5cdfe4c18..a801576784 100644 --- a/packages/host/app/services/matrix-service.ts +++ b/packages/host/app/services/matrix-service.ts @@ -40,6 +40,7 @@ import { APP_BOXEL_CARD_FORMAT, APP_BOXEL_CARDFRAGMENT_MSGTYPE, APP_BOXEL_COMMAND_MSGTYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, APP_BOXEL_COMMAND_RESULT_MSGTYPE, APP_BOXEL_MESSAGE_MSGTYPE, APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE, @@ -60,14 +61,11 @@ import { getMatrixProfile } from '@cardstack/host/resources/matrix-profile'; import type { Base64ImageField as Base64ImageFieldType } from 'https://cardstack.com/base/base64-image'; import { BaseDef, type CardDef } from 'https://cardstack.com/base/card-api'; import type * as CardAPI from 'https://cardstack.com/base/card-api'; -import type { - CommandReactionEventContent, - MatrixEvent as DiscreteMatrixEvent, -} from 'https://cardstack.com/base/matrix-event'; import type { CardMessageContent, CardFragmentContent, - ReactionEventContent, + CommandResultContent, + MatrixEvent as DiscreteMatrixEvent, } from 'https://cardstack.com/base/matrix-event'; import { SkillCard } from 'https://cardstack.com/base/skill-card'; @@ -489,7 +487,7 @@ export default class MatrixService extends Service { async sendEvent( roomId: string, eventType: string, - content: CardMessageContent | CardFragmentContent | ReactionEventContent, + content: CardMessageContent | CardFragmentContent | CommandResultContent, ) { let roomData = await this.ensureRoomData(roomId); return roomData.mutex.dispatch(async () => { @@ -514,22 +512,26 @@ export default class MatrixService extends Service { if (resultCard) { [resultCardEventId] = await this.addCardsToRoom([resultCard], roomId); } - let content: CommandReactionEventContent = { - msgtype: 'org.boxel.command_result', + let content: CommandResultContent = { + msgtype: APP_BOXEL_COMMAND_RESULT_MSGTYPE, 'm.relates_to': { event_id: invokedToolFromEventId, key: 'applied', rel_type: 'm.annotation', }, data: { - card_event_id: resultCardEventId ?? null, + cardEventId: resultCardEventId ?? undefined, }, }; try { - return await this.sendEvent(roomId, 'm.reaction', content); + return await this.sendEvent( + roomId, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, + content, + ); } catch (e) { throw new Error( - `Error sending command result reaction event: ${ + `Error sending command result event: ${ 'message' in (e as Error) ? (e as Error).message : e }`, ); @@ -1177,10 +1179,10 @@ export default class MatrixService extends Service { )) as DiscreteMatrixEvent; if ( fragmentEvent.type !== 'm.room.message' || - fragmentEvent.content.msgtype !== 'org.boxel.cardFragment' + fragmentEvent.content.msgtype !== APP_BOXEL_CARDFRAGMENT_MSGTYPE ) { throw new Error( - `Expected event ${currentFragmentId} to be 'org.boxel.card' but was ${JSON.stringify( + `Expected event ${currentFragmentId} to be ${APP_BOXEL_CARDFRAGMENT_MSGTYPE} but was ${JSON.stringify( fragmentEvent, )}`, ); @@ -1196,10 +1198,10 @@ export default class MatrixService extends Service { } else { if ( fragmentEvent.type !== 'm.room.message' || - fragmentEvent.content.msgtype !== 'org.boxel.cardFragment' + fragmentEvent.content.msgtype !== APP_BOXEL_CARDFRAGMENT_MSGTYPE ) { throw new Error( - `Expected event to be 'org.boxel.cardFragment' but was ${JSON.stringify( + `Expected event to be '${APP_BOXEL_CARDFRAGMENT_MSGTYPE}' but was ${JSON.stringify( fragmentEvent, )}`, ); @@ -1262,16 +1264,16 @@ export default class MatrixService extends Service { } } else if ( roomData && - event.type === 'm.reaction' && - event.content?.msgtype === 'org.boxel.command_result' + event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE && + event.content?.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE ) { let data = ( typeof event.content.data === 'string' ? JSON.parse(event.content.data) : event.content.data - ) as CommandReactionEventContent['data']; - if (data.card_event_id) { - this.ensureCardFragmentsLoaded(data.card_event_id, roomData); + ) as CommandResultContent['data']; + if (data.cardEventId) { + this.ensureCardFragmentsLoaded(data.cardEventId, roomData); } } else if ( event.type === 'm.room.message' && diff --git a/packages/host/tests/acceptance/commands-test.gts b/packages/host/tests/acceptance/commands-test.gts index 66c83479a5..f84d67f2af 100644 --- a/packages/host/tests/acceptance/commands-test.gts +++ b/packages/host/tests/acceptance/commands-test.gts @@ -12,10 +12,7 @@ import { import { module, test } from 'qunit'; -import { - BoxelInputValidationState, - GridContainer, -} from '@cardstack/boxel-ui/components'; +import { GridContainer } from '@cardstack/boxel-ui/components'; import { baseRealm, Command } from '@cardstack/runtime-common'; @@ -25,6 +22,7 @@ import { } from '@cardstack/runtime-common/matrix-constants'; import CreateAIAssistantRoomCommand from '@cardstack/host/commands/create-ai-assistant-room'; +import GetBoxelUIStateCommand from '@cardstack/host/commands/get-boxel-ui-state'; import PatchCardCommand from '@cardstack/host/commands/patch-card'; import SaveCardCommand from '@cardstack/host/commands/save-card'; import SendAiAssistantMessageCommand from '@cardstack/host/commands/send-ai-assistant-message'; @@ -59,7 +57,6 @@ import { import { setupMockMatrix } from '../helpers/mock-matrix'; import { setupApplicationTest } from '../helpers/setup'; -import GetBoxelUIStateCommand from '@cardstack/host/commands/get-boxel-ui-state'; let matrixRoomId = ''; module('Acceptance | Commands tests', function (hooks) { @@ -765,7 +762,7 @@ module('Acceptance | Commands tests', function (hooks) { await waitUntil(() => getRoomIds().length > 0); let roomId = getRoomIds().pop()!; let message = getRoomEvents(roomId).pop()!; - assert.strictEqual(message.content.msgtype, 'org.boxel.message'); + assert.strictEqual(message.content.msgtype, APP_BOXEL_MESSAGE_MSGTYPE); let boxelMessageData = JSON.parse(message.content.data); assert.strictEqual(boxelMessageData.context.tools.length, 1); assert.strictEqual(boxelMessageData.context.tools[0].type, 'function'); @@ -798,7 +795,7 @@ module('Acceptance | Commands tests', function (hooks) { }); simulateRemoteMessage(roomId, '@aibot:localhost', { body: 'Inspecting the current UI state', - msgtype: 'org.boxel.command', + msgtype: APP_BOXEL_COMMAND_MSGTYPE, formatted_body: 'Inspecting the current UI state', format: 'org.matrix.custom.html', data: JSON.stringify({ @@ -824,6 +821,7 @@ module('Acceptance | Commands tests', function (hooks) { assert .dom('[data-test-message-idx="1"][data-test-boxel-message-from="aibot"]') .containsText('Inspecting the current UI state'); + await this.pauseTest(); assert .dom('[data-test-message-idx="1"] [data-test-apply-state="applied"]') .exists(); diff --git a/packages/host/tests/helpers/mock-matrix/_client.ts b/packages/host/tests/helpers/mock-matrix/_client.ts index f3f7d2678a..42fef1a640 100644 --- a/packages/host/tests/helpers/mock-matrix/_client.ts +++ b/packages/host/tests/helpers/mock-matrix/_client.ts @@ -5,8 +5,9 @@ import * as MatrixSDK from 'matrix-js-sdk'; import { baseRealm, unixTime } from '@cardstack/runtime-common'; import { - APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, APP_BOXEL_REALMS_EVENT_TYPE, + APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, LEGACY_APP_BOXEL_REALMS_EVENT_TYPE, } from '@cardstack/runtime-common/matrix-constants'; @@ -423,7 +424,7 @@ export class MockClient implements ExtendedClient { case APP_BOXEL_REALMS_EVENT_TYPE: return this.sdk.ClientEvent.AccountData; case APP_BOXEL_ROOM_SKILLS_EVENT_TYPE: - case 'm.reaction': + case APP_BOXEL_COMMAND_RESULT_EVENT_TYPE: case 'm.room.create': case 'm.room.message': case 'm.room.name': diff --git a/packages/host/tests/integration/components/ai-assistant-panel-test.gts b/packages/host/tests/integration/components/ai-assistant-panel-test.gts index 22a7976a73..e085c532d9 100644 --- a/packages/host/tests/integration/components/ai-assistant-panel-test.gts +++ b/packages/host/tests/integration/components/ai-assistant-panel-test.gts @@ -19,6 +19,7 @@ import { Loader } from '@cardstack/runtime-common/loader'; import { APP_BOXEL_CARDFRAGMENT_MSGTYPE, APP_BOXEL_COMMAND_MSGTYPE, + APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, APP_BOXEL_COMMAND_RESULT_MSGTYPE, APP_BOXEL_MESSAGE_MSGTYPE, } from '@cardstack/runtime-common/matrix-constants'; @@ -1907,16 +1908,16 @@ module('Integration | ai-assistant-panel', function (hooks) { event_id: '__EVENT_ID__', }, }); - let commandReactionEvents = getRoomEvents(roomId).filter( + let commandResultEvents = getRoomEvents(roomId).filter( (event) => - event.type === 'm.reaction' && + event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE && event.content['m.relates_to']?.rel_type === 'm.annotation' && event.content['m.relates_to']?.key === 'applied', ); assert.equal( - commandReactionEvents.length, + commandResultEvents.length, 0, - 'reaction event is not dispatched', + 'command result event is not dispatched', ); await settled(); @@ -1931,16 +1932,16 @@ module('Integration | ai-assistant-panel', function (hooks) { .dom('[data-test-message-idx="0"] [data-test-apply-state="applied"]') .exists(); - commandReactionEvents = await getRoomEvents(roomId).filter( + commandResultEvents = await getRoomEvents(roomId).filter( (event) => - event.type === 'm.reaction' && + event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE && event.content['m.relates_to']?.rel_type === 'm.annotation' && event.content['m.relates_to']?.key === 'applied', ); assert.equal( - commandReactionEvents.length, + commandResultEvents.length, 1, - 'reaction event is dispatched', + 'command result event is dispatched', ); }); @@ -2060,9 +2061,9 @@ module('Integration | ai-assistant-panel', function (hooks) { e.content['m.relates_to']?.rel_type === 'm.annotation', ) as CommandResultEvent; let serializedResults = - typeof commandResultEvent?.content?.result === 'string' - ? JSON.parse(commandResultEvent.content.result) - : commandResultEvent.content.result; + typeof commandResultEvent?.content?.data.card === 'string' + ? JSON.parse(commandResultEvent.content.data.card) + : commandResultEvent.content.data.card; serializedResults = Array.isArray(serializedResults) ? serializedResults : []; @@ -2120,9 +2121,9 @@ module('Integration | ai-assistant-panel', function (hooks) { e.content['m.relates_to']?.rel_type === 'm.annotation', ) as CommandResultEvent; let serializedResults = - typeof commandResultEvent?.content?.result === 'string' - ? JSON.parse(commandResultEvent.content.result) - : commandResultEvent.content.result; + typeof commandResultEvent?.content?.data.card === 'string' + ? JSON.parse(commandResultEvent.content.data.card) + : commandResultEvent.content.data.card; serializedResults = Array.isArray(serializedResults) ? serializedResults : []; diff --git a/packages/matrix/helpers/matrix-constants.ts b/packages/matrix/helpers/matrix-constants.ts index d7e4a9495b..c2c5531354 100644 --- a/packages/matrix/helpers/matrix-constants.ts +++ b/packages/matrix/helpers/matrix-constants.ts @@ -2,6 +2,7 @@ export const APP_BOXEL_CARDFRAGMENT_MSGTYPE = 'app.boxel.cardFragment'; export const APP_BOXEL_MESSAGE_MSGTYPE = 'app.boxel.message'; export const APP_BOXEL_COMMAND_MSGTYPE = 'app.boxel.command'; export const APP_BOXEL_CARD_FORMAT = 'app.boxel.card'; +export const APP_BOXEL_COMMAND_RESULT_EVENT_TYPE = 'app.boxel.commandResult'; export const APP_BOXEL_COMMAND_RESULT_MSGTYPE = 'app.boxel.commandResult'; export const APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE = 'app.boxel.realm-server-event'; diff --git a/packages/runtime-common/matrix-constants.ts b/packages/runtime-common/matrix-constants.ts index edbe488e0f..d79f427607 100644 --- a/packages/runtime-common/matrix-constants.ts +++ b/packages/runtime-common/matrix-constants.ts @@ -2,6 +2,7 @@ export const APP_BOXEL_CARDFRAGMENT_MSGTYPE = 'app.boxel.cardFragment'; export const APP_BOXEL_MESSAGE_MSGTYPE = 'app.boxel.message'; export const APP_BOXEL_COMMAND_MSGTYPE = 'app.boxel.command'; export const APP_BOXEL_CARD_FORMAT = 'app.boxel.card'; +export const APP_BOXEL_COMMAND_RESULT_EVENT_TYPE = 'app.boxel.commandResult'; export const APP_BOXEL_COMMAND_RESULT_MSGTYPE = 'app.boxel.commandResult'; export const APP_BOXEL_REALM_SERVER_EVENT_MSGTYPE = 'app.boxel.realm-server-event'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 05d468dca1..6983a56d76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,6 +163,9 @@ importers: '@sinonjs/fake-timers': specifier: ^11.2.2 version: 11.2.2 + '@types/qunit': + specifier: ^2.19.12 + version: 2.19.12 '@types/sinonjs__fake-timers': specifier: ^8.1.5 version: 8.1.5 @@ -7350,6 +7353,10 @@ packages: resolution: {integrity: sha512-gVB+rxvxmbyPFWa6yjjKgcumWal3hyqoTXI0Oil161uWfo1OCzWZ/rnEumsx+6uVgrwPrCrhpQbLkzfildkSbg==} dev: true + /@types/qunit@2.19.12: + resolution: {integrity: sha512-II+C1wgzUia0g+tGAH+PBb4XiTm8/C/i6sN23r21NNskBYOYrv+qnW0tFQ/IxZzKVwrK4CTglf8YO3poJUclQA==} + dev: true + /@types/range-parser@1.2.6: resolution: {integrity: sha512-+0autS93xyXizIYiyL02FCY8N+KkKPhILhcUSA276HxzreZ16kl+cmwvV2qAM/PuCCwPXzOXOWhiPcw20uSFcA==} dev: true