Add summary and icon to action events

openadapt/adapters/prompt.py

@@ -41,6 +41,26 @@ def prompt(
     raise Exception("All drivers failed to provide a response")
 
 
+def generate_summary_and_icon(action_dict: dict, window_dict: dict, screenshot: Image.Image) -> tuple[str, str]:
+    """Generate a summary and icon for a given action.
+
+    Args:
+        action_dict: The action dictionary.
+        window_dict: The window dictionary.
+        screenshot: The screenshot image.
+
+    Returns:
+        A tuple containing the summary and icon.
+    """
+    summary_prompt = f"Please provide a one sentence summary of the following action: {action_dict}, window: {window_dict}"
+    summary = prompt(summary_prompt, images=[screenshot])
+
+    icon_prompt = f"Please provide an icon for the following action: {action_dict}, window: {window_dict}"
+    icon = prompt(icon_prompt, images=[screenshot])
+
+    return summary, icon
+
+
 if __name__ == "__main__":
     # This could be extended to use command-line arguments or other input methods
     print(prompt("Describe the solar system."))

openadapt/app/dashboard/api/recordings.py

@@ -11,6 +11,7 @@
 from openadapt.models import Recording
 from openadapt.plotting import display_event
 from openadapt.utils import image2utf8, row2dict
+from openadapt.adapters.prompt import generate_summary_and_icon
 
 
 class RecordingsAPI:
@@ -135,6 +136,9 @@ def convert_to_str(event_dict: dict) -> dict:
                     word_index += 1
                 event_dict["words"] = words
                 convert_to_str(event_dict)
+                summary, icon = generate_summary_and_icon(event_dict)
+                event_dict["summary"] = summary
+                event_dict["icon"] = icon
                 await websocket.send_json({"type": "action_event", "value": event_dict})
 
             await websocket.close()

openadapt/app/dashboard/app/recordings/detail/page.tsx

@@ -97,6 +97,8 @@ function modifyActionEvent(actionEvent: ActionEventType, isOriginal: boolean): A
         children,
         isComputed,
         isOriginal,
+        summary: actionEvent.summary,
+        icon: actionEvent.icon,
     }
 }