FileDef + (auto)attaching files for the AI bot to read

packages/ai-bot/helpers.ts

@@ -75,13 +75,13 @@ export class HistoryConstructionError extends Error {
   }
 }
 
-export function getPromptParts(
+export async function getPromptParts(
   eventList: DiscreteMatrixEvent[],
   aiBotUserId: string,
-): PromptParts {
+): Promise<PromptParts> {
   let cardFragments: Map<string, CardFragmentContent> =
     extractCardFragmentsFromEvents(eventList);
-  let history: DiscreteMatrixEvent[] = constructHistory(
+  let history: DiscreteMatrixEvent[] = await constructHistory(
     eventList,
     cardFragments,
   );
@@ -113,7 +113,7 @@ export function extractCardFragmentsFromEvents(
   return fragments;
 }
 
-export function constructHistory(
+export async function constructHistory(
   eventlist: IRoomEvent[],
   cardFragments: Map<string, CardFragmentContent>,
 ) {
@@ -173,6 +173,17 @@ export function constructHistory(
           serializedCardFromFragments(id, cardFragments),
         );
       }
+      let { attachedFiles } = event.content.data;
+      if (attachedFiles && attachedFiles.length > 0) {
+        let fileTextContents = await Promise.all(
+          attachedFiles.map(async (file) => {
+            let response = await fetch(file.url);
+            let text = await response.text(); // TODO: handle binary files, such as images
+            return { url: file.url, name: file.name, text };
+          }),
+        );
+        event.content.data.attachedFiles = fileTextContents;
+      }
     }
     if (event.content['m.relates_to']?.rel_type === 'm.replace') {
       eventId = event.content['m.relates_to']!.event_id!;
@@ -287,6 +298,10 @@ function getMostRecentlyAttachedCard(attachedCards: LooseSingleCardDocument[]) {
     : undefined;
 }
 
+function getMostRecentlyAttachedFilesEvent(events: DiscreteMatrixEvent[]) {
+  return events.findLast((e) => e.content.data?.attachedFiles);
+}
+
 export function getRelevantCards(
   history: DiscreteMatrixEvent[],
   aiBotUserId: string,
@@ -500,11 +515,20 @@ export function getModifyPrompt(
     history,
     aiBotUserId,
   );
+
+  // FIXME: Only include attached files if they are actually attached to the message that was sent by the user
+  let mostRecentlyAttachedFilesEvent =
+    getMostRecentlyAttachedFilesEvent(history);
+
   let systemMessage =
     MODIFY_SYSTEM_MESSAGE +
     `
-  The user currently has given you the following data to work with:
-  Cards:\n`;
+  The user currently has given you the following data to work with: \n
+  Attached code files:\n
+  ${mostRecentlyAttachedFilesEvent?.content.data?.attachedFiles
+    ?.map((f) => `${f.name}: ${f.text}`)
+    .join('\n')}
+  \n Cards:\n`;
   systemMessage += attachedCardsToMessage(
     mostRecentlyAttachedCard,
     attachedCards,

packages/ai-bot/main.ts

@@ -214,7 +214,7 @@ Common issues are:
         let eventList = (initial!.messages?.chunk ||
           []) as DiscreteMatrixEvent[];
         try {
-          promptParts = getPromptParts(eventList, aiBotUserId);
+          promptParts = await getPromptParts(eventList, aiBotUserId);
         } catch (e) {
           log.error(e);
           responder.finalize(

packages/base/file-api.gts

@@ -0,0 +1,51 @@
+import FileIcon from '@cardstack/boxel-icons/file';
+import {
+  BaseDef,
+  BaseDefComponent,
+  Component,
+  StringField,
+  contains,
+  field,
+} from './card-api';
+
+class View extends Component<typeof FileDef> {
+  <template>
+    {{@model.name}}
+  </template>
+}
+
+export class FileDef extends BaseDef {
+  static displayName = 'File';
+  static icon = FileIcon;
+
+  @field sourceUrl = contains(StringField);
+  @field url = contains(StringField);
+  @field name = contains(StringField);
+  @field type = contains(StringField);
+
+  static embedded: BaseDefComponent = View;
+  static fitted: BaseDefComponent = View;
+  static isolated: BaseDefComponent = View;
+  static atom: BaseDefComponent = View;
+
+  serialize() {
+    return {
+      sourceUrl: this.sourceUrl,
+      url: this.url,
+      name: this.name,
+      type: this.type,
+    };
+  }
+}
+
+export function createFileDef({
+  url,
+  sourceUrl,
+  name,
+}: {
+  url: string;
+  sourceUrl: string;
+  name: string;
+}) {
+  return new FileDef({ url, sourceUrl, name });
+}

packages/base/matrix-event.gts

@@ -192,6 +192,10 @@ export interface CardMessageContent {
     // fragments that we receive
     attachedCards?: LooseSingleCardDocument[];
     skillCards?: LooseSingleCardDocument[];
+    attachedFiles?: {
+      url: string;
+      name: string;
+    }[];
     context: {
       openCardIds?: string[];
       tools: Tool[];

packages/host/app/components/ai-assistant/card-picker/index.gts

@@ -10,16 +10,19 @@ import { TrackedSet } from 'tracked-built-ins';
 import { AddButton, Tooltip, Pill } from '@cardstack/boxel-ui/components';
 import { and, cn, gt, not } from '@cardstack/boxel-ui/helpers';
 
+import FileCode from '@cardstack/boxel-icons/file-code';
+
 import { chooseCard, baseCardRef } from '@cardstack/runtime-common';
 
 import CardPill from '@cardstack/host/components/card-pill';
 
 import { type CardDef } from 'https://cardstack.com/base/card-api';
-
+import { type FileDef } from 'https://cardstack.com/base/file-api';
 interface Signature {
   Element: HTMLDivElement;
   Args: {
     autoAttachedCards?: TrackedSet<CardDef>;
+    attachedFiles?: FileDef[];
     cardsToAttach: CardDef[] | undefined;
     chooseCard: (card: CardDef) => void;
     removeCard: (card: CardDef) => void;
@@ -31,6 +34,18 @@ const MAX_CARDS_TO_DISPLAY = 4;
 export default class AiAssistantCardPicker extends Component<Signature> {
   <template>
     <div class='card-picker'>
+      {{#each @attachedFiles as |file|}}
+        <Pill class={{cn 'card-pill'}} ...attributes>
+          <:iconLeft>
+            <FileCode />
+          </:iconLeft>
+          <:default>
+            <div class='card-content' title={{file.name}}>
+              {{file.name}}
+            </div>
+          </:default>
+        </Pill>
+      {{/each}}
       {{#each this.cardsToDisplay as |card i|}}
         {{#if (this.isCardDisplayed card i)}}
           {{#if (this.isAutoAttachedCard card)}}

packages/host/app/components/matrix/room.gts

@@ -60,7 +60,8 @@ import RoomMessage from './room-message';
 
 import type RoomData from '../../lib/matrix-classes/room';
 import type { Skill } from '../ai-assistant/skill-menu';
-
+import type { FileAPI } from 'https://cardstack.com/base/file-api';
+import LoaderService from '@cardstack/host/services/loader-service';
 interface Signature {
   Args: {
     roomId: string;
@@ -136,6 +137,7 @@ export default class Room extends Component<Signature> {
             <div class='chat-input-area__bottom-section'>
               <AiAssistantCardPicker
                 @autoAttachedCards={{this.autoAttachedCards}}
+                @attachedFiles={{this.attachedFiles}}
                 @cardsToAttach={{this.cardsToAttach}}
                 @chooseCard={{this.chooseCard}}
                 @removeCard={{this.removeCard}}
@@ -207,12 +209,15 @@ export default class Room extends Component<Signature> {
   @service private declare commandService: CommandService;
   @service private declare matrixService: MatrixService;
   @service private declare operatorModeStateService: OperatorModeStateService;
+  @service private declare loaderService: LoaderService;
 
+  @tracked private fileAPI: FileAPI | undefined;
   private roomResource = getRoom(
     this,
     () => this.args.roomId,
     () => this.matrixService.getRoomData(this.args.roomId)?.events,
   );
+
   private autoAttachmentResource = getAutoAttachment(
     this,
     () => this.topMostStackItems,
@@ -236,6 +241,7 @@ export default class Room extends Component<Signature> {
   constructor(owner: Owner, args: Signature['Args']) {
     super(owner, args);
     this.doMatrixEventFlush.perform();
+    this.doLoadFileAPI.perform();
     registerDestructor(this, () => {
       this.scrollState().messageVisibilityObserver.disconnect();
     });
@@ -272,6 +278,37 @@ export default class Room extends Component<Signature> {
     return state;
   }
 
+  private get autoAttachedFileUrl() {
+    if (!this.fileAPI) {
+      return undefined;
+    }
+    return this.operatorModeStateService.state.codePath?.href;
+  }
+
+  private get autoAttachedFile() {
+    debugger;
+    if (!this.autoAttachedFileUrl) {
+      return undefined;
+    }
+
+    return this.fileAPI.createFileDef({
+      sourceUrl: this.autoAttachedFileUrl,
+      name: this.autoAttachedFileUrl.split('/').pop(),
+    });
+  }
+
+  private get attachedFiles() {
+    if (!this.fileAPI) {
+      return [];
+    }
+    // User will be eventually able to attach files manually (from the realm file system, or uploaded from the device),
+    // but for now we only support auto-attaching files from the operator mode code path
+    if (!this.autoAttachedFile) {
+      return [];
+    }
+    return [this.autoAttachedFile];
+  }
+
   private get isScrolledToBottom() {
     return this.scrollState().isScrolledToBottom;
   }
@@ -474,6 +511,14 @@ export default class Room extends Component<Signature> {
     await this.roomResource.loading;
   });
 
+  private doLoadFileAPI = restartableTask(async () => {
+    let fileAPI = await this.loaderService.loader.import<typeof FileAPI>(
+      'https://cardstack.com/base/file-api',
+    );
+    debugger;
+    this.fileAPI = fileAPI;
+  });
+
   private get messages() {
     return this.roomResource.messages;
   }
@@ -558,6 +603,7 @@ export default class Room extends Component<Signature> {
         cards.push(card);
       });
     }
+
     this.doSendMessage.perform(
       prompt ?? this.messageToSend,
       cards.length ? cards : undefined,
@@ -598,6 +644,7 @@ export default class Room extends Component<Signature> {
       this.cardsToAttach?.length ? this.cardsToAttach : undefined,
     );
   }
+
   private doSendMessage = enqueueTask(
     async (
       message: string | undefined,
@@ -619,11 +666,19 @@ export default class Room extends Component<Signature> {
           .filter((stackItem) => stackItem)
           .map((stackItem) => stackItem.card.id),
       };
+
+      // TODO: reuplad if file content changed, and add upload error handling
+      debugger;
+      let uploadedFiles = await this.matrixService.uploadFiles(
+        this.attachedFiles,
+      );
+
       try {
         await this.matrixService.sendMessage(
           this.args.roomId,
           message,
           cards,
+          uploadedFiles,
           clientGeneratedId,
           context,
         );

packages/host/app/resources/auto-attached-card.ts

@@ -9,6 +9,9 @@ import type { StackItem } from '@cardstack/host/lib/stack-item';
 import { isIndexCard } from '@cardstack/host/lib/stack-item';
 
 import { type CardDef } from 'https://cardstack.com/base/card-api';
+import { Submodes } from '../components/submode-switcher';
+import OperatorModeStateService from '../services/operator-mode-state-service';
+import { service } from '@ember/service';
 
 interface Args {
   named: {
@@ -26,8 +29,13 @@ export class AutoAttachment extends Resource<Args> {
   private lastStackedItems: StackItem[] = [];
   private lastRemovedCards: Set<string> = new Set(); // internal state, changed from the outside. It tracks, everytime a card is removed in the ai-panel
 
+  @service private declare operatorModeStateService: OperatorModeStateService;
+
   modify(_positional: never[], named: Args['named']) {
     const { topMostStackItems, attachedCards } = named;
+    if (this.operatorModeStateService.state.submode === Submodes.Code) {
+      return;
+    }
     this.updateAutoAttachedCardsTask.perform(topMostStackItems, attachedCards);
   }
 

packages/host/app/services/matrix-sdk-loader.ts

@@ -113,6 +113,11 @@ export type ExtendedClient = Pick<
   ): Promise<MatrixSDK.LoginResponse>;
   createRealmSession(realmURL: URL): Promise<string>;
   hashMessageWithSecret(message: string): Promise<string>;
+  uploadContent(
+    file: MatrixSDK.FileType,
+    opts: MatrixSDK.UploadOpts,
+  ): Promise<MatrixSDK.UploadResponse>;
+  mxcUrlToHttp(mxcUrl: string): string;
 };
 
 async function hashMessageWithSecret(