diff --git a/packages/ai-bot/helpers.ts b/packages/ai-bot/helpers.ts index 99faca4b80..b1af6ecff2 100644 --- a/packages/ai-bot/helpers.ts +++ b/packages/ai-bot/helpers.ts @@ -75,13 +75,13 @@ export class HistoryConstructionError extends Error { } } -export function getPromptParts( +export async function getPromptParts( eventList: DiscreteMatrixEvent[], aiBotUserId: string, -): PromptParts { +): Promise { let cardFragments: Map = extractCardFragmentsFromEvents(eventList); - let history: DiscreteMatrixEvent[] = constructHistory( + let history: DiscreteMatrixEvent[] = await constructHistory( eventList, cardFragments, ); @@ -113,7 +113,7 @@ export function extractCardFragmentsFromEvents( return fragments; } -export function constructHistory( +export async function constructHistory( eventlist: IRoomEvent[], cardFragments: Map, ) { @@ -173,6 +173,17 @@ export function constructHistory( serializedCardFromFragments(id, cardFragments), ); } + let { attachedFiles } = event.content.data; + if (attachedFiles && attachedFiles.length > 0) { + let fileTextContents = await Promise.all( + attachedFiles.map(async (file) => { + let response = await fetch(file.url); + let text = await response.text(); // TODO: handle binary files, such as images + return { url: file.url, name: file.name, text }; + }), + ); + event.content.data.attachedFiles = fileTextContents; + } } if (event.content['m.relates_to']?.rel_type === 'm.replace') { eventId = event.content['m.relates_to']!.event_id!; @@ -287,6 +298,10 @@ function getMostRecentlyAttachedCard(attachedCards: LooseSingleCardDocument[]) { : undefined; } +function getMostRecentlyAttachedFilesEvent(events: DiscreteMatrixEvent[]) { + return events.findLast((e) => e.content.data?.attachedFiles); +} + export function getRelevantCards( history: DiscreteMatrixEvent[], aiBotUserId: string, @@ -500,11 +515,20 @@ export function getModifyPrompt( history, aiBotUserId, ); + + // FIXME: Only include attached files if they are actually attached to the message that was sent by the user + let mostRecentlyAttachedFilesEvent = + getMostRecentlyAttachedFilesEvent(history); + let systemMessage = MODIFY_SYSTEM_MESSAGE + ` - The user currently has given you the following data to work with: - Cards:\n`; + The user currently has given you the following data to work with: \n + Attached code files:\n + ${mostRecentlyAttachedFilesEvent?.content.data?.attachedFiles + ?.map((f) => `${f.name}: ${f.text}`) + .join('\n')} + \n Cards:\n`; systemMessage += attachedCardsToMessage( mostRecentlyAttachedCard, attachedCards, diff --git a/packages/ai-bot/main.ts b/packages/ai-bot/main.ts index 2394dd1be8..17adc0d274 100644 --- a/packages/ai-bot/main.ts +++ b/packages/ai-bot/main.ts @@ -214,7 +214,7 @@ Common issues are: let eventList = (initial!.messages?.chunk || []) as DiscreteMatrixEvent[]; try { - promptParts = getPromptParts(eventList, aiBotUserId); + promptParts = await getPromptParts(eventList, aiBotUserId); } catch (e) { log.error(e); responder.finalize( diff --git a/packages/base/file-api.gts b/packages/base/file-api.gts new file mode 100644 index 0000000000..8e40beef1f --- /dev/null +++ b/packages/base/file-api.gts @@ -0,0 +1,51 @@ +import FileIcon from '@cardstack/boxel-icons/file'; +import { + BaseDef, + BaseDefComponent, + Component, + StringField, + contains, + field, +} from './card-api'; + +class View extends Component { + +} + +export class FileDef extends BaseDef { + static displayName = 'File'; + static icon = FileIcon; + + @field sourceUrl = contains(StringField); + @field url = contains(StringField); + @field name = contains(StringField); + @field type = contains(StringField); + + static embedded: BaseDefComponent = View; + static fitted: BaseDefComponent = View; + static isolated: BaseDefComponent = View; + static atom: BaseDefComponent = View; + + serialize() { + return { + sourceUrl: this.sourceUrl, + url: this.url, + name: this.name, + type: this.type, + }; + } +} + +export function createFileDef({ + url, + sourceUrl, + name, +}: { + url: string; + sourceUrl: string; + name: string; +}) { + return new FileDef({ url, sourceUrl, name }); +} diff --git a/packages/base/matrix-event.gts b/packages/base/matrix-event.gts index ee1cd1d55b..950073c3f1 100644 --- a/packages/base/matrix-event.gts +++ b/packages/base/matrix-event.gts @@ -192,6 +192,10 @@ export interface CardMessageContent { // fragments that we receive attachedCards?: LooseSingleCardDocument[]; skillCards?: LooseSingleCardDocument[]; + attachedFiles?: { + url: string; + name: string; + }[]; context: { openCardIds?: string[]; tools: Tool[]; diff --git a/packages/host/app/components/ai-assistant/card-picker/index.gts b/packages/host/app/components/ai-assistant/card-picker/index.gts index fbe07af34c..2e3154506c 100644 --- a/packages/host/app/components/ai-assistant/card-picker/index.gts +++ b/packages/host/app/components/ai-assistant/card-picker/index.gts @@ -10,16 +10,19 @@ import { TrackedSet } from 'tracked-built-ins'; import { AddButton, Tooltip, Pill } from '@cardstack/boxel-ui/components'; import { and, cn, gt, not } from '@cardstack/boxel-ui/helpers'; +import FileCode from '@cardstack/boxel-icons/file-code'; + import { chooseCard, baseCardRef } from '@cardstack/runtime-common'; import CardPill from '@cardstack/host/components/card-pill'; import { type CardDef } from 'https://cardstack.com/base/card-api'; - +import { type FileDef } from 'https://cardstack.com/base/file-api'; interface Signature { Element: HTMLDivElement; Args: { autoAttachedCards?: TrackedSet; + attachedFiles?: FileDef[]; cardsToAttach: CardDef[] | undefined; chooseCard: (card: CardDef) => void; removeCard: (card: CardDef) => void; @@ -31,6 +34,18 @@ const MAX_CARDS_TO_DISPLAY = 4; export default class AiAssistantCardPicker extends Component {