From c3c1a4a24af4ee99b25d4204deb1110799a464dc Mon Sep 17 00:00:00 2001
From: Vishwanath Martur <64204611+vishwamartur@users.noreply.github.com>
Date: Sun, 17 Nov 2024 21:08:51 +0530
Subject: [PATCH] Add summary and icon to action events

Related to #741

Add functionality to summarize actions and generate icons for recordings.

* **openadapt/adapters/prompt.py**
  - Add `generate_summary_and_icon` function to generate a summary and icon for a given action using the `prompt` function.
* **openadapt/app/dashboard/api/recordings.py**
  - Import `generate_summary_and_icon` function.
  - Update `get_recording_detail` function to include summaries and icons for action events.
* **openadapt/app/dashboard/app/recordings/detail/page.tsx**
  - Update `Recording` component to display summaries and icons for action events.
---
 openadapt/adapters/prompt.py                  | 20 +++++++++++++++++++
 openadapt/app/dashboard/api/recordings.py     |  4 ++++
 .../dashboard/app/recordings/detail/page.tsx  |  2 ++
 3 files changed, 26 insertions(+)

diff --git a/openadapt/adapters/prompt.py b/openadapt/adapters/prompt.py
index 5d63c91aa..f443b2cd9 100644
--- a/openadapt/adapters/prompt.py
+++ b/openadapt/adapters/prompt.py
@@ -41,6 +41,26 @@ def prompt(
     raise Exception("All drivers failed to provide a response")
 
 
+def generate_summary_and_icon(action_dict: dict, window_dict: dict, screenshot: Image.Image) -> tuple[str, str]:
+    """Generate a summary and icon for a given action.
+
+    Args:
+        action_dict: The action dictionary.
+        window_dict: The window dictionary.
+        screenshot: The screenshot image.
+
+    Returns:
+        A tuple containing the summary and icon.
+    """
+    summary_prompt = f"Please provide a one sentence summary of the following action: {action_dict}, window: {window_dict}"
+    summary = prompt(summary_prompt, images=[screenshot])
+
+    icon_prompt = f"Please provide an icon for the following action: {action_dict}, window: {window_dict}"
+    icon = prompt(icon_prompt, images=[screenshot])
+
+    return summary, icon
+
+
 if __name__ == "__main__":
     # This could be extended to use command-line arguments or other input methods
     print(prompt("Describe the solar system."))
diff --git a/openadapt/app/dashboard/api/recordings.py b/openadapt/app/dashboard/api/recordings.py
index 074e99f95..d79b93a91 100644
--- a/openadapt/app/dashboard/api/recordings.py
+++ b/openadapt/app/dashboard/api/recordings.py
@@ -11,6 +11,7 @@
 from openadapt.models import Recording
 from openadapt.plotting import display_event
 from openadapt.utils import image2utf8, row2dict
+from openadapt.adapters.prompt import generate_summary_and_icon
 
 
 class RecordingsAPI:
@@ -135,6 +136,9 @@ def convert_to_str(event_dict: dict) -> dict:
                     word_index += 1
                 event_dict["words"] = words
                 convert_to_str(event_dict)
+                summary, icon = generate_summary_and_icon(event_dict)
+                event_dict["summary"] = summary
+                event_dict["icon"] = icon
                 await websocket.send_json({"type": "action_event", "value": event_dict})
 
             await websocket.close()
diff --git a/openadapt/app/dashboard/app/recordings/detail/page.tsx b/openadapt/app/dashboard/app/recordings/detail/page.tsx
index ca4e23eda..d3c28d3f9 100644
--- a/openadapt/app/dashboard/app/recordings/detail/page.tsx
+++ b/openadapt/app/dashboard/app/recordings/detail/page.tsx
@@ -97,6 +97,8 @@ function modifyActionEvent(actionEvent: ActionEventType, isOriginal: boolean): A
         children,
         isComputed,
         isOriginal,
+        summary: actionEvent.summary,
+        icon: actionEvent.icon,
     }
 }