From 4af7ddcd27bb6338e67deabbd76c466bf2f3569a Mon Sep 17 00:00:00 2001
From: Matic Jurglic <matic@jurglic.si>
Date: Fri, 7 Feb 2025 15:21:59 +0100
Subject: [PATCH 1/4] Add method signatures to matrix client

---
 packages/host/app/services/matrix-sdk-loader.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/host/app/services/matrix-sdk-loader.ts b/packages/host/app/services/matrix-sdk-loader.ts
index 97afc9797b..060a3a6d08 100644
--- a/packages/host/app/services/matrix-sdk-loader.ts
+++ b/packages/host/app/services/matrix-sdk-loader.ts
@@ -113,6 +113,11 @@ export type ExtendedClient = Pick<
   ): Promise<MatrixSDK.LoginResponse>;
   createRealmSession(realmURL: URL): Promise<string>;
   hashMessageWithSecret(message: string): Promise<string>;
+  uploadContent(
+    file: MatrixSDK.FileType,
+    opts: MatrixSDK.UploadOpts,
+  ): Promise<MatrixSDK.UploadResponse>;
+  mxcUrlToHttp(mxcUrl: string): string;
 };
 
 async function hashMessageWithSecret(

From 457ad379d6bd20324a8a05a741153f2e826c1b8d Mon Sep 17 00:00:00 2001
From: Matic Jurglic <matic@jurglic.si>
Date: Fri, 7 Feb 2025 15:25:26 +0100
Subject: [PATCH 2/4] Implement file attachment upload

---
 packages/base/file-api.gts                    | 51 +++++++++++++++++
 packages/base/matrix-event.gts                |  4 ++
 .../ai-assistant/card-picker/index.gts        | 17 +++++-
 packages/host/app/components/matrix/room.gts  | 57 ++++++++++++++++++-
 packages/host/app/services/matrix-service.ts  | 39 ++++++++++++-
 5 files changed, 164 insertions(+), 4 deletions(-)
 create mode 100644 packages/base/file-api.gts

diff --git a/packages/base/file-api.gts b/packages/base/file-api.gts
new file mode 100644
index 0000000000..8e40beef1f
--- /dev/null
+++ b/packages/base/file-api.gts
@@ -0,0 +1,51 @@
+import FileIcon from '@cardstack/boxel-icons/file';
+import {
+  BaseDef,
+  BaseDefComponent,
+  Component,
+  StringField,
+  contains,
+  field,
+} from './card-api';
+
+class View extends Component<typeof FileDef> {
+  <template>
+    {{@model.name}}
+  </template>
+}
+
+export class FileDef extends BaseDef {
+  static displayName = 'File';
+  static icon = FileIcon;
+
+  @field sourceUrl = contains(StringField);
+  @field url = contains(StringField);
+  @field name = contains(StringField);
+  @field type = contains(StringField);
+
+  static embedded: BaseDefComponent = View;
+  static fitted: BaseDefComponent = View;
+  static isolated: BaseDefComponent = View;
+  static atom: BaseDefComponent = View;
+
+  serialize() {
+    return {
+      sourceUrl: this.sourceUrl,
+      url: this.url,
+      name: this.name,
+      type: this.type,
+    };
+  }
+}
+
+export function createFileDef({
+  url,
+  sourceUrl,
+  name,
+}: {
+  url: string;
+  sourceUrl: string;
+  name: string;
+}) {
+  return new FileDef({ url, sourceUrl, name });
+}
diff --git a/packages/base/matrix-event.gts b/packages/base/matrix-event.gts
index ee1cd1d55b..950073c3f1 100644
--- a/packages/base/matrix-event.gts
+++ b/packages/base/matrix-event.gts
@@ -192,6 +192,10 @@ export interface CardMessageContent {
     // fragments that we receive
     attachedCards?: LooseSingleCardDocument[];
     skillCards?: LooseSingleCardDocument[];
+    attachedFiles?: {
+      url: string;
+      name: string;
+    }[];
     context: {
       openCardIds?: string[];
       tools: Tool[];
diff --git a/packages/host/app/components/ai-assistant/card-picker/index.gts b/packages/host/app/components/ai-assistant/card-picker/index.gts
index fbe07af34c..2e3154506c 100644
--- a/packages/host/app/components/ai-assistant/card-picker/index.gts
+++ b/packages/host/app/components/ai-assistant/card-picker/index.gts
@@ -10,16 +10,19 @@ import { TrackedSet } from 'tracked-built-ins';
 import { AddButton, Tooltip, Pill } from '@cardstack/boxel-ui/components';
 import { and, cn, gt, not } from '@cardstack/boxel-ui/helpers';
 
+import FileCode from '@cardstack/boxel-icons/file-code';
+
 import { chooseCard, baseCardRef } from '@cardstack/runtime-common';
 
 import CardPill from '@cardstack/host/components/card-pill';
 
 import { type CardDef } from 'https://cardstack.com/base/card-api';
-
+import { type FileDef } from 'https://cardstack.com/base/file-api';
 interface Signature {
   Element: HTMLDivElement;
   Args: {
     autoAttachedCards?: TrackedSet<CardDef>;
+    attachedFiles?: FileDef[];
     cardsToAttach: CardDef[] | undefined;
     chooseCard: (card: CardDef) => void;
     removeCard: (card: CardDef) => void;
@@ -31,6 +34,18 @@ const MAX_CARDS_TO_DISPLAY = 4;
 export default class AiAssistantCardPicker extends Component<Signature> {
   <template>
     <div class='card-picker'>
+      {{#each @attachedFiles as |file|}}
+        <Pill class={{cn 'card-pill'}} ...attributes>
+          <:iconLeft>
+            <FileCode />
+          </:iconLeft>
+          <:default>
+            <div class='card-content' title={{file.name}}>
+              {{file.name}}
+            </div>
+          </:default>
+        </Pill>
+      {{/each}}
       {{#each this.cardsToDisplay as |card i|}}
         {{#if (this.isCardDisplayed card i)}}
           {{#if (this.isAutoAttachedCard card)}}
diff --git a/packages/host/app/components/matrix/room.gts b/packages/host/app/components/matrix/room.gts
index 085adcff58..3b6dd37062 100644
--- a/packages/host/app/components/matrix/room.gts
+++ b/packages/host/app/components/matrix/room.gts
@@ -60,7 +60,8 @@ import RoomMessage from './room-message';
 
 import type RoomData from '../../lib/matrix-classes/room';
 import type { Skill } from '../ai-assistant/skill-menu';
-
+import type { FileAPI } from 'https://cardstack.com/base/file-api';
+import LoaderService from '@cardstack/host/services/loader-service';
 interface Signature {
   Args: {
     roomId: string;
@@ -136,6 +137,7 @@ export default class Room extends Component<Signature> {
             <div class='chat-input-area__bottom-section'>
               <AiAssistantCardPicker
                 @autoAttachedCards={{this.autoAttachedCards}}
+                @attachedFiles={{this.attachedFiles}}
                 @cardsToAttach={{this.cardsToAttach}}
                 @chooseCard={{this.chooseCard}}
                 @removeCard={{this.removeCard}}
@@ -207,12 +209,15 @@ export default class Room extends Component<Signature> {
   @service private declare commandService: CommandService;
   @service private declare matrixService: MatrixService;
   @service private declare operatorModeStateService: OperatorModeStateService;
+  @service private declare loaderService: LoaderService;
 
+  @tracked private fileAPI: FileAPI | undefined;
   private roomResource = getRoom(
     this,
     () => this.args.roomId,
     () => this.matrixService.getRoomData(this.args.roomId)?.events,
   );
+
   private autoAttachmentResource = getAutoAttachment(
     this,
     () => this.topMostStackItems,
@@ -236,6 +241,7 @@ export default class Room extends Component<Signature> {
   constructor(owner: Owner, args: Signature['Args']) {
     super(owner, args);
     this.doMatrixEventFlush.perform();
+    this.doLoadFileAPI.perform();
     registerDestructor(this, () => {
       this.scrollState().messageVisibilityObserver.disconnect();
     });
@@ -272,6 +278,37 @@ export default class Room extends Component<Signature> {
     return state;
   }
 
+  private get autoAttachedFileUrl() {
+    if (!this.fileAPI) {
+      return undefined;
+    }
+    return this.operatorModeStateService.state.codePath?.href;
+  }
+
+  private get autoAttachedFile() {
+    debugger;
+    if (!this.autoAttachedFileUrl) {
+      return undefined;
+    }
+
+    return this.fileAPI.createFileDef({
+      sourceUrl: this.autoAttachedFileUrl,
+      name: this.autoAttachedFileUrl.split('/').pop(),
+    });
+  }
+
+  private get attachedFiles() {
+    if (!this.fileAPI) {
+      return [];
+    }
+    // User will be eventually able to attach files manually (from the realm file system, or uploaded from the device),
+    // but for now we only support auto-attaching files from the operator mode code path
+    if (!this.autoAttachedFile) {
+      return [];
+    }
+    return [this.autoAttachedFile];
+  }
+
   private get isScrolledToBottom() {
     return this.scrollState().isScrolledToBottom;
   }
@@ -474,6 +511,14 @@ export default class Room extends Component<Signature> {
     await this.roomResource.loading;
   });
 
+  private doLoadFileAPI = restartableTask(async () => {
+    let fileAPI = await this.loaderService.loader.import<typeof FileAPI>(
+      'https://cardstack.com/base/file-api',
+    );
+    debugger;
+    this.fileAPI = fileAPI;
+  });
+
   private get messages() {
     return this.roomResource.messages;
   }
@@ -558,6 +603,7 @@ export default class Room extends Component<Signature> {
         cards.push(card);
       });
     }
+
     this.doSendMessage.perform(
       prompt ?? this.messageToSend,
       cards.length ? cards : undefined,
@@ -598,6 +644,7 @@ export default class Room extends Component<Signature> {
       this.cardsToAttach?.length ? this.cardsToAttach : undefined,
     );
   }
+
   private doSendMessage = enqueueTask(
     async (
       message: string | undefined,
@@ -619,11 +666,19 @@ export default class Room extends Component<Signature> {
           .filter((stackItem) => stackItem)
           .map((stackItem) => stackItem.card.id),
       };
+
+      // TODO: reuplad if file content changed, and add upload error handling
+      debugger;
+      let uploadedFiles = await this.matrixService.uploadFiles(
+        this.attachedFiles,
+      );
+
       try {
         await this.matrixService.sendMessage(
           this.args.roomId,
           message,
           cards,
+          uploadedFiles,
           clientGeneratedId,
           context,
         );
diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts
index 7129215fd6..409a92d856 100644
--- a/packages/host/app/services/matrix-service.ts
+++ b/packages/host/app/services/matrix-service.ts
@@ -63,6 +63,7 @@ import { getMatrixProfile } from '@cardstack/host/resources/matrix-profile';
 import type { Base64ImageField as Base64ImageFieldType } from 'https://cardstack.com/base/base64-image';
 import { BaseDef, type CardDef } from 'https://cardstack.com/base/card-api';
 import type * as CardAPI from 'https://cardstack.com/base/card-api';
+import { type FileDef } from 'https://cardstack.com/base/file-api';
 import type {
   CardMessageContent,
   CardFragmentContent,
@@ -80,7 +81,9 @@ import { RoomResource, getRoom } from '../resources/room';
 
 import { CurrentRoomIdPersistenceKey } from '../utils/local-storage-keys';
 
-import { type SerializedState as OperatorModeSerializedState } from './operator-mode-state-service';
+import OperatorModeStateService, {
+  type SerializedState as OperatorModeSerializedState,
+} from './operator-mode-state-service';
 
 import type CardService from './card-service';
 import type CommandService from './command-service';
@@ -92,7 +95,7 @@ import type RealmServerService from './realm-server';
 import type ResetService from './reset';
 
 import type * as MatrixSDK from 'matrix-js-sdk';
-
+import type NetworkService from './network';
 const { matrixURL } = ENV;
 const MAX_CARD_SIZE_KB = 60;
 const STATE_EVENTS_OF_INTEREST = ['m.room.create', 'm.room.name'];
@@ -112,6 +115,8 @@ export default class MatrixService extends Service {
   @service private declare realmServer: RealmServerService;
   @service private declare router: RouterService;
   @service private declare reset: ResetService;
+  @service private declare network: NetworkService;
+  @service private declare operatorModeStateService: OperatorModeStateService;
   @tracked private _client: ExtendedClient | undefined;
   @tracked private _isInitializingNewUser = false;
   @tracked private _isNewUser = false;
@@ -619,10 +624,39 @@ export default class MatrixService extends Service {
     return eventIds;
   }
 
+  async uploadFiles(files: FileDef[]) {
+    const uploadedFiles = await Promise.all(
+      files.map(async (file) => {
+        if (!file.sourceUrl) {
+          throw new Error('File has no source URL');
+        }
+        let response = await this.network.authedFetch(file.sourceUrl, {
+          headers: {
+            Accept: 'application/vnd.card+source',
+          },
+        });
+
+        let blob = await response.blob();
+        let type = response.headers.get('content-type') || '';
+        let uploadResponse = await this.client.uploadContent(blob, {
+          type,
+        });
+
+        file.url = this.client.mxcUrlToHttp(uploadResponse.content_uri);
+        file.type = type;
+
+        return file;
+      }),
+    );
+    console.log('uploadedFiles', uploadedFiles);
+    return uploadedFiles;
+  }
+
   async sendMessage(
     roomId: string,
     body: string | undefined,
     attachedCards: CardDef[] = [],
+    attachedFiles: FileDef[] = [],
     clientGeneratedId = uuidv4(),
     context?: OperatorModeContext,
   ): Promise<void> {
@@ -663,6 +697,7 @@ export default class MatrixService extends Service {
       formatted_body: html,
       clientGeneratedId,
       data: {
+        attachedFiles: attachedFiles.map((file: FileDef) => file.serialize()),
         attachedCardsEventIds,
         context: {
           openCardIds: attachedOpenCards.map((c) => c.id),

From d986123c1c41a6bf0bccc5ea6817c32fffa7f433 Mon Sep 17 00:00:00 2001
From: Matic Jurglic <matic@jurglic.si>
Date: Fri, 7 Feb 2025 15:25:42 +0100
Subject: [PATCH 3/4] Do not auto attach a card in code mode

---
 packages/host/app/resources/auto-attached-card.ts | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/packages/host/app/resources/auto-attached-card.ts b/packages/host/app/resources/auto-attached-card.ts
index fbdbf1dfb2..dbab6456ad 100644
--- a/packages/host/app/resources/auto-attached-card.ts
+++ b/packages/host/app/resources/auto-attached-card.ts
@@ -9,6 +9,9 @@ import type { StackItem } from '@cardstack/host/lib/stack-item';
 import { isIndexCard } from '@cardstack/host/lib/stack-item';
 
 import { type CardDef } from 'https://cardstack.com/base/card-api';
+import { Submodes } from '../components/submode-switcher';
+import OperatorModeStateService from '../services/operator-mode-state-service';
+import { service } from '@ember/service';
 
 interface Args {
   named: {
@@ -26,8 +29,13 @@ export class AutoAttachment extends Resource<Args> {
   private lastStackedItems: StackItem[] = [];
   private lastRemovedCards: Set<string> = new Set(); // internal state, changed from the outside. It tracks, everytime a card is removed in the ai-panel
 
+  @service private declare operatorModeStateService: OperatorModeStateService;
+
   modify(_positional: never[], named: Args['named']) {
     const { topMostStackItems, attachedCards } = named;
+    if (this.operatorModeStateService.state.submode === Submodes.Code) {
+      return;
+    }
     this.updateAutoAttachedCardsTask.perform(topMostStackItems, attachedCards);
   }
 

From bdb5d2ed2a5b9deea39320ce0782dcb92ff3c6bb Mon Sep 17 00:00:00 2001
From: Matic Jurglic <matic@jurglic.si>
Date: Fri, 7 Feb 2025 15:26:00 +0100
Subject: [PATCH 4/4] Bot downloads attached file and includes it in the prompt

---
 packages/ai-bot/helpers.ts | 36 ++++++++++++++++++++++++++++++------
 packages/ai-bot/main.ts    |  2 +-
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/packages/ai-bot/helpers.ts b/packages/ai-bot/helpers.ts
index 99faca4b80..b1af6ecff2 100644
--- a/packages/ai-bot/helpers.ts
+++ b/packages/ai-bot/helpers.ts
@@ -75,13 +75,13 @@ export class HistoryConstructionError extends Error {
   }
 }
 
-export function getPromptParts(
+export async function getPromptParts(
   eventList: DiscreteMatrixEvent[],
   aiBotUserId: string,
-): PromptParts {
+): Promise<PromptParts> {
   let cardFragments: Map<string, CardFragmentContent> =
     extractCardFragmentsFromEvents(eventList);
-  let history: DiscreteMatrixEvent[] = constructHistory(
+  let history: DiscreteMatrixEvent[] = await constructHistory(
     eventList,
     cardFragments,
   );
@@ -113,7 +113,7 @@ export function extractCardFragmentsFromEvents(
   return fragments;
 }
 
-export function constructHistory(
+export async function constructHistory(
   eventlist: IRoomEvent[],
   cardFragments: Map<string, CardFragmentContent>,
 ) {
@@ -173,6 +173,17 @@ export function constructHistory(
           serializedCardFromFragments(id, cardFragments),
         );
       }
+      let { attachedFiles } = event.content.data;
+      if (attachedFiles && attachedFiles.length > 0) {
+        let fileTextContents = await Promise.all(
+          attachedFiles.map(async (file) => {
+            let response = await fetch(file.url);
+            let text = await response.text(); // TODO: handle binary files, such as images
+            return { url: file.url, name: file.name, text };
+          }),
+        );
+        event.content.data.attachedFiles = fileTextContents;
+      }
     }
     if (event.content['m.relates_to']?.rel_type === 'm.replace') {
       eventId = event.content['m.relates_to']!.event_id!;
@@ -287,6 +298,10 @@ function getMostRecentlyAttachedCard(attachedCards: LooseSingleCardDocument[]) {
     : undefined;
 }
 
+function getMostRecentlyAttachedFilesEvent(events: DiscreteMatrixEvent[]) {
+  return events.findLast((e) => e.content.data?.attachedFiles);
+}
+
 export function getRelevantCards(
   history: DiscreteMatrixEvent[],
   aiBotUserId: string,
@@ -500,11 +515,20 @@ export function getModifyPrompt(
     history,
     aiBotUserId,
   );
+
+  // FIXME: Only include attached files if they are actually attached to the message that was sent by the user
+  let mostRecentlyAttachedFilesEvent =
+    getMostRecentlyAttachedFilesEvent(history);
+
   let systemMessage =
     MODIFY_SYSTEM_MESSAGE +
     `
-  The user currently has given you the following data to work with:
-  Cards:\n`;
+  The user currently has given you the following data to work with: \n
+  Attached code files:\n
+  ${mostRecentlyAttachedFilesEvent?.content.data?.attachedFiles
+    ?.map((f) => `${f.name}: ${f.text}`)
+    .join('\n')}
+  \n Cards:\n`;
   systemMessage += attachedCardsToMessage(
     mostRecentlyAttachedCard,
     attachedCards,
diff --git a/packages/ai-bot/main.ts b/packages/ai-bot/main.ts
index 2394dd1be8..17adc0d274 100644
--- a/packages/ai-bot/main.ts
+++ b/packages/ai-bot/main.ts
@@ -214,7 +214,7 @@ Common issues are:
         let eventList = (initial!.messages?.chunk ||
           []) as DiscreteMatrixEvent[];
         try {
-          promptParts = getPromptParts(eventList, aiBotUserId);
+          promptParts = await getPromptParts(eventList, aiBotUserId);
         } catch (e) {
           log.error(e);
           responder.finalize(