From 89e6aec219d266553e62ba486638ea056abeac3c Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Tue, 25 Jun 2024 17:34:24 +0530 Subject: [PATCH 1/9] feat: Add replay logging mechanism --- .../c84664aeb5ae_add_replay_models.py | 62 ++++++++++++++++ openadapt/app/tray.py | 2 +- openadapt/db/crud.py | 72 +++++++++++++++++++ openadapt/models.py | 19 ++++- openadapt/replay.py | 14 +++- openadapt/strategies/base.py | 48 +++++++++++++ openadapt/utils.py | 13 ++++ 7 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 openadapt/alembic/versions/c84664aeb5ae_add_replay_models.py diff --git a/openadapt/alembic/versions/c84664aeb5ae_add_replay_models.py b/openadapt/alembic/versions/c84664aeb5ae_add_replay_models.py new file mode 100644 index 000000000..7a24c9d0e --- /dev/null +++ b/openadapt/alembic/versions/c84664aeb5ae_add_replay_models.py @@ -0,0 +1,62 @@ +"""add_replay_models + +Revision ID: c84664aeb5ae +Revises: bb25e889ad71 +Create Date: 2024-06-25 15:05:09.110171 + +""" +from alembic import op +import sqlalchemy as sa + +import openadapt + +# revision identifiers, used by Alembic. +revision = "c84664aeb5ae" +down_revision = "bb25e889ad71" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "replay", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "timestamp", + openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), + nullable=True, + ), + sa.Column("strategy_name", sa.String(), nullable=True), + sa.Column("strategy_args", sa.JSON(), nullable=True), + sa.Column("git_hash", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replay")), + ) + op.create_table( + "replay_log", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("replay_id", sa.Integer(), nullable=True), + sa.Column("lineno", sa.Integer(), nullable=True), + sa.Column("filename", sa.String(), nullable=True), + sa.Column("git_hash", sa.String(), nullable=True), + sa.Column( + "timestamp", + openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), + nullable=True, + ), + sa.Column("log_level", sa.String(), nullable=True), + sa.Column("key", sa.String(), nullable=True), + sa.Column("data", sa.LargeBinary(), nullable=True), + sa.ForeignKeyConstraint( + ["replay_id"], ["replay.id"], name=op.f("fk_replay_log_replay_id_replay") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replay_log")), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("replay_log") + op.drop_table("replay") + # ### end Alembic commands ### diff --git a/openadapt/app/tray.py b/openadapt/app/tray.py index 3aa03ab06..1fdc2c396 100644 --- a/openadapt/app/tray.py +++ b/openadapt/app/tray.py @@ -377,7 +377,7 @@ def update_args_inputs() -> None: logger.info(f"kwargs=\n{pformat(kwargs)}") self.child_conn.send({"type": "replay.starting"}) - record_replay = False + record_replay = True recording_timestamp = None strategy_name = selected_strategy.__name__ replay_proc = multiprocessing.Process( diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index 96de81333..95a1cf631 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -5,12 +5,16 @@ from typing import Any, TypeVar import asyncio +import importlib.metadata import json import os +import pickle +import sys import time from loguru import logger from sqlalchemy.orm import Session as SaSession +import git import psutil import sqlalchemy as sa @@ -23,6 +27,8 @@ MemoryStat, PerformanceStat, Recording, + Replay, + ReplayLog, Screenshot, ScrubbedRecording, WindowEvent, @@ -692,6 +698,72 @@ def get_audio_info( return audio_infos[0] if audio_infos else None +def add_replay( + session: SaSession, + strategy_name: str, + strategy_args: dict, +) -> int: + """Add a replay to the database. + + Args: + session (sa.orm.Session): The database session. + strategy_name (str): The name of the replay strategy. + strategy_args (dict): The arguments of the replay strategy. + + Returns: + int: The id of the replay. + """ + git_hash = utils.get_git_hash() + timestamp = utils.get_timestamp() + replay = Replay( + timestamp=timestamp, + strategy_name=strategy_name, + strategy_args=strategy_args, + git_hash=git_hash, + ) + session.add(replay) + session.commit() + session.refresh(replay) + return replay.id + + +def add_replay_log(*, replay_id: int, log_level: str, key: str, data: Any) -> None: + """Add a replay log entry to the database. + + Args: + replay_id (int): The id of the replay. + log_level (str): The log level of the log entry. + key (str): The key of the log entry. + data (Any): The data of the log entry. + """ + with get_new_session(read_and_write=True) as session: + pickled_data = pickle.dumps(data) + + frame = sys._getframe(1) + caller_line = frame.f_lineno + caller_file = frame.f_code.co_filename + + git_hash = utils.get_git_hash() + timestamp = utils.get_timestamp() + + logger.info( + f"{caller_line=}, {caller_file=}, {git_hash=}, {timestamp=}, {log_level=}," + f" {key=}" + ) + replay_log = ReplayLog( + replay_id=replay_id, + lineno=caller_line, + filename=caller_file, + git_hash=git_hash, + timestamp=timestamp, + log_level=log_level, + key=key, + data=pickled_data, + ) + session.add(replay_log) + session.commit() + + def post_process_events(session: SaSession, recording: Recording) -> None: """Post-process events. diff --git a/openadapt/models.py b/openadapt/models.py index 76d424425..3546df157 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -19,7 +19,6 @@ from openadapt.privacy.base import ScrubbingProvider, TextScrubbingMixin from openadapt.privacy.providers import ScrubProvider - EMPTY_VALS = [None, "", [], (), {}] @@ -930,6 +929,24 @@ class Replay(db.Base): strategy_name = sa.Column(sa.String) strategy_args = sa.Column(sa.JSON) git_hash = sa.Column(sa.String) + logs = sa.orm.relationship("ReplayLog", back_populates="replay") + + +class ReplayLog(db.Base): + """Class representing a replay log in the database.""" + + __tablename__ = "replay_log" + + id = sa.Column(sa.Integer, primary_key=True) + replay_id = sa.Column(sa.ForeignKey("replay.id")) + replay = sa.orm.relationship("Replay", back_populates="logs") + lineno = sa.Column(sa.Integer) + filename = sa.Column(sa.String) + git_hash = sa.Column(sa.String) + timestamp = sa.Column(ForceFloat) + log_level = sa.Column(sa.String) + key = sa.Column(sa.String) + data = sa.Column(sa.LargeBinary) def copy_sa_instance(sa_instance: db.Base, **kwargs: dict) -> db.Base: diff --git a/openadapt/replay.py b/openadapt/replay.py index 79d9bf425..d2d20134b 100644 --- a/openadapt/replay.py +++ b/openadapt/replay.py @@ -16,7 +16,8 @@ with redirect_stdout_stderr(): import fire -from openadapt import capture as _capture, utils +from openadapt import capture as _capture +from openadapt import utils from openadapt.config import CAPTURE_DIR_PATH, print_config from openadapt.db import crud from openadapt.models import Recording @@ -48,6 +49,7 @@ def replay( Returns: bool: True if replay was successful, None otherwise. """ + utils.set_start_time() utils.configure_logging(logger, LOG_LEVEL) print_config() posthog.capture(event="replay.started", properties={"strategy_name": strategy_name}) @@ -81,9 +83,17 @@ def replay( strategy_class = strategy_class_by_name[strategy_name] logger.info(f"{strategy_class=}") + write_session = crud.get_new_session(read_and_write=True) + replay_id = crud.add_replay(write_session, strategy_name, strategy_args=kwargs) + strategy = strategy_class(recording, **kwargs) + strategy.attach_replay_id(replay_id) logger.info(f"{strategy=}") + if not crud.acquire_db_lock(): + logger.error("Failed to acquire lock") + return + handler = None rval = True if capture: @@ -113,6 +123,8 @@ def replay( _capture.stop() logger.remove(handler) + crud.release_db_lock() + return rval diff --git a/openadapt/strategies/base.py b/openadapt/strategies/base.py index 96f8b012f..28cdcdef7 100644 --- a/openadapt/strategies/base.py +++ b/openadapt/strategies/base.py @@ -9,6 +9,7 @@ import numpy as np from openadapt import models, playback, utils +from openadapt.db import crud MAX_FRAME_TIMES = 1000 @@ -49,20 +50,46 @@ def get_next_action_event( """ pass + def attach_replay_id(self, replay_id: int) -> None: + """Attach the replay ID to the strategy. + + Args: + replay_id (int): The replay ID. + """ + self._replay_id = replay_id + def run(self) -> None: """Run the replay strategy.""" keyboard_controller = keyboard.Controller() mouse_controller = mouse.Controller() while True: screenshot = models.Screenshot.take_screenshot() + crud.add_replay_log( + replay_id=self._replay_id, + log_level="INFO", + key="screenshot", + data=screenshot.png_data, + ) self.screenshots.append(screenshot) window_event = models.WindowEvent.get_active_window_event() + crud.add_replay_log( + replay_id=self._replay_id, + log_level="INFO", + key="window_event", + data=window_event, + ) self.window_events.append(window_event) try: action_event = self.get_next_action_event( screenshot, window_event, ) + crud.add_replay_log( + replay_id=self._replay_id, + log_level="INFO", + key="action_event", + data=action_event, + ) except StopIteration: break if self.action_events: @@ -83,6 +110,12 @@ def run(self) -> None: drop_constant=False, )[0] logger.debug(f"action_event=\n{pformat(action_event_dict)}") + crud.add_replay_log( + replay_id=self._replay_id, + log_level="INFO", + key="action_event_dict", + data=action_event_dict, + ) self.action_events.append(action_event) try: playback.play_action_event( @@ -90,6 +123,12 @@ def run(self) -> None: mouse_controller, keyboard_controller, ) + crud.add_replay_log( + replay_id=self._replay_id, + log_level="INFO", + key="playback", + data="success", + ) except Exception as exc: logger.exception(exc) import ipdb @@ -106,5 +145,14 @@ def log_fps(self) -> None: mean_dt = np.mean(dts) fps = 1 / mean_dt logger.info(f"{fps=:.2f}") + crud.add_replay_log( + replay_id=self._replay_id, log_level="INFO", key="fps", data=fps + ) if len(self.frame_times) > self.max_frame_times: self.frame_times.pop(0) + crud.add_replay_log( + replay_id=self._replay_id, + log_level="INFO", + key="frame_times", + data=self.frame_times, + ) diff --git a/openadapt/utils.py b/openadapt/utils.py index 80c8f1b1d..ff5fd2c43 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -20,6 +20,7 @@ from loguru import logger from PIL import Image, ImageEnhance from posthog import Posthog +import git from openadapt.build_utils import is_running_from_executable, redirect_stdout_stderr @@ -952,5 +953,17 @@ def get_posthog_instance() -> DistinctIDPosthog: return posthog +def get_git_hash(): + git_hash = None + try: + repo = git.Repo(search_parent_directories=True) + git_hash = repo.head.commit.hexsha + except git.InvalidGitRepositoryError: + git_hash = importlib.metadata.version("openadapt") + except Exception as exc: + logger.warning(f"{exc=}") + return git_hash + + if __name__ == "__main__": fire.Fire(get_functions(__name__)) From 244e5eab2e44d6447ca414cc17fea320a1feef7a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Fri, 5 Jul 2024 13:00:50 -0400 Subject: [PATCH 2/9] log dumping state --- openadapt/window/_macos.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openadapt/window/_macos.py b/openadapt/window/_macos.py index afb8237d7..077320d81 100644 --- a/openadapt/window/_macos.py +++ b/openadapt/window/_macos.py @@ -3,6 +3,7 @@ import pickle import plistlib import re +import time from loguru import logger import AppKit @@ -110,7 +111,11 @@ def get_window_data(window_meta: dict) -> dict: dict: A dictionary containing the data of the window. """ window = get_active_window(window_meta) + start_time = time.time() + logger.debug("dumping state...") state = dump_state(window) + duration = time.time() - start_time + logger.debug(f"{duration=}") return state From 7cd70768fb2b3d995384c77cdd45a7c7e211fa96 Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Mon, 8 Jul 2024 17:57:15 +0530 Subject: [PATCH 3/9] lint: Fix linting errors --- openadapt/db/crud.py | 1 - openadapt/utils.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index e18b8dcd5..420215f29 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -5,7 +5,6 @@ from typing import Any, TypeVar import asyncio -import importlib.metadata import json import os import pickle diff --git a/openadapt/utils.py b/openadapt/utils.py index ff5fd2c43..96c4a6326 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -953,7 +953,8 @@ def get_posthog_instance() -> DistinctIDPosthog: return posthog -def get_git_hash(): +def get_git_hash() -> str: + """Get the Git hash of the current commit.""" git_hash = None try: repo = git.Repo(search_parent_directories=True) From 04121f19d0009a954fe7565710e0e43fd57cc035 Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Mon, 15 Jul 2024 12:20:26 +0530 Subject: [PATCH 4/9] feat: Reduce size of replay logs by commiting relevant data --- openadapt/models.py | 20 ++++++++++++++++++++ openadapt/strategies/base.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/openadapt/models.py b/openadapt/models.py index 3546df157..60b1d49bd 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -181,6 +181,26 @@ def __init__(self, **kwargs: dict) -> None: for key, value in properties.items(): setattr(self, key, value) + def to_log_dict(self) -> dict[str, Any]: + """Convert the action event to a log dictionary.""" + return { + "name": self.name, + "timestamp": self.timestamp, + "mouse_x": self.mouse_x, + "mouse_y": self.mouse_y, + "mouse_dx": self.mouse_dx, + "mouse_dy": self.mouse_dy, + "mouse_button_name": self.mouse_button_name, + "mouse_pressed": self.mouse_pressed, + "key_name": self.key_name, + "key_char": self.key_char, + "key_vk": self.key_vk, + "canonical_key_name": self.canonical_key_name, + "canonical_key_char": self.canonical_key_char, + "canonical_key_vk": self.canonical_key_vk, + "element_state": self.element_state, + } + @property def available_segment_descriptions(self) -> list[str]: """Gets the available segment descriptions.""" diff --git a/openadapt/strategies/base.py b/openadapt/strategies/base.py index 28cdcdef7..a3cc87ffc 100644 --- a/openadapt/strategies/base.py +++ b/openadapt/strategies/base.py @@ -88,7 +88,7 @@ def run(self) -> None: replay_id=self._replay_id, log_level="INFO", key="action_event", - data=action_event, + data=action_event.to_log_dict(), ) except StopIteration: break From 5f1f1b9f199ffa69f6600e21e4ec06134c37effb Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Thu, 18 Jul 2024 10:58:53 +0530 Subject: [PATCH 5/9] temp --- openadapt/app/dashboard/api/index.py | 3 ++ openadapt/app/dashboard/api/replays.py | 52 +++++++++++++++++++ .../app/dashboard/app/replay_logs/page.tsx | 5 ++ openadapt/app/dashboard/app/routes.ts | 4 ++ openadapt/config.defaults.json | 2 +- openadapt/db/crud.py | 30 +++++++++++ openadapt/strategies/base.py | 2 +- 7 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 openadapt/app/dashboard/api/replays.py create mode 100644 openadapt/app/dashboard/app/replay_logs/page.tsx diff --git a/openadapt/app/dashboard/api/index.py b/openadapt/app/dashboard/api/index.py index aff6decd9..6d14af51c 100644 --- a/openadapt/app/dashboard/api/index.py +++ b/openadapt/app/dashboard/api/index.py @@ -12,6 +12,7 @@ from openadapt.app.dashboard.api.action_events import ActionEventsAPI from openadapt.app.dashboard.api.recordings import RecordingsAPI +from openadapt.app.dashboard.api.replays import ReplaysAPI from openadapt.app.dashboard.api.scrubbing import ScrubbingAPI from openadapt.app.dashboard.api.settings import SettingsAPI from openadapt.build_utils import is_running_from_executable @@ -23,11 +24,13 @@ action_events_app = ActionEventsAPI().attach_routes() recordings_app = RecordingsAPI().attach_routes() +replays_app = ReplaysAPI().attach_routes() scrubbing_app = ScrubbingAPI().attach_routes() settings_app = SettingsAPI().attach_routes() api.include_router(action_events_app, prefix="/action-events") api.include_router(recordings_app, prefix="/recordings") +api.include_router(replays_app, prefix="/replays") api.include_router(scrubbing_app, prefix="/scrubbing") api.include_router(settings_app, prefix="/settings") diff --git a/openadapt/app/dashboard/api/replays.py b/openadapt/app/dashboard/api/replays.py new file mode 100644 index 000000000..7004cb427 --- /dev/null +++ b/openadapt/app/dashboard/api/replays.py @@ -0,0 +1,52 @@ +"""API endpoints for replays.""" + +from typing import Any, Literal + +from fastapi import APIRouter + +from openadapt.config import Config, config, persist_config +from openadapt.db import crud +from openadapt.utils import image2utf8 + + +class ReplaysAPI: + """API endpoints for replays.""" + + def __init__(self) -> None: + """Initialize the ReplaysAPI class.""" + self.app = APIRouter() + + def attach_routes(self) -> APIRouter: + """Attach routes to the FastAPI app.""" + self.app.add_api_route("", self.get_replays, methods=["GET"]) + self.app.add_api_route( + "/{replay_id}/logs", self.get_replay_logs, methods=["GET"] + ) + return self.app + + @staticmethod + def get_replays(): + """Get all replays.""" + session = crud.get_new_session(read_only=True) + return crud.get_replays(session) + + @staticmethod + def get_replay_logs(replay_id: int): + """Get all logs for a replay.""" + session = crud.get_new_session(read_only=True) + logs = crud.get_replay_logs(session, replay_id) + unpickled_logs = [] + import base64 + import pickle + + def binary_to_base64_uri(binary_data): + base64_data = base64.b64encode(binary_data).decode("utf-8") + base64_uri = f"data:image/png;base64,{base64_data}" + return base64_uri + + for log in logs: + log.data = pickle.loads(log.data) + if log.key == "screenshot": + log.data = image2utf8(log.data) + unpickled_logs.append(log) + return unpickled_logs diff --git a/openadapt/app/dashboard/app/replay_logs/page.tsx b/openadapt/app/dashboard/app/replay_logs/page.tsx new file mode 100644 index 000000000..70a6ea447 --- /dev/null +++ b/openadapt/app/dashboard/app/replay_logs/page.tsx @@ -0,0 +1,5 @@ +export default function ReplayLogs() { + return ( + Replay logs + ) +} diff --git a/openadapt/app/dashboard/app/routes.ts b/openadapt/app/dashboard/app/routes.ts index b0ef5f0d0..c5dabf8c8 100644 --- a/openadapt/app/dashboard/app/routes.ts +++ b/openadapt/app/dashboard/app/routes.ts @@ -19,5 +19,9 @@ export const routes: Route[] = [ { name: 'Onboarding', path: '/onboarding', + }, + { + name: "Replay logs", + path: "/replay_logs", } ] diff --git a/openadapt/config.defaults.json b/openadapt/config.defaults.json index a74f15eb9..783f9cfb2 100644 --- a/openadapt/config.defaults.json +++ b/openadapt/config.defaults.json @@ -19,7 +19,7 @@ "RECORD_VIDEO": true, "RECORD_AUDIO": true, "RECORD_FULL_VIDEO": false, - "RECORD_IMAGES": false, + "RECORD_IMAGES": true, "LOG_MEMORY": false, "STOP_SEQUENCES": [ [ diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index 420215f29..609fdd21b 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -794,6 +794,36 @@ def add_replay_log(*, replay_id: int, log_level: str, key: str, data: Any) -> No session.commit() +def get_replays( + session: SaSession, +) -> list[Replay]: + """Get all replays. + + Args: + session (sa.orm.Session): The database session. + + Returns: + list[Replay]: A list of all replays. + """ + return session.query(Replay).order_by(sa.desc(Replay.timestamp)).all() + + +def get_replay_logs( + session: SaSession, + replay_id: int, +) -> list[ReplayLog]: + """Get all logs for a replay. + + Args: + session (sa.orm.Session): The database session. + replay_id (int): The id of the replay. + + Returns: + list[ReplayLog]: A list of all logs for the replay. + """ + return session.query(ReplayLog).filter(ReplayLog.replay_id == replay_id).all() + + def post_process_events(session: SaSession, recording: Recording) -> None: """Post-process events. diff --git a/openadapt/strategies/base.py b/openadapt/strategies/base.py index a3cc87ffc..cb71f3990 100644 --- a/openadapt/strategies/base.py +++ b/openadapt/strategies/base.py @@ -68,7 +68,7 @@ def run(self) -> None: replay_id=self._replay_id, log_level="INFO", key="screenshot", - data=screenshot.png_data, + data=screenshot.image, ) self.screenshots.append(screenshot) window_event = models.WindowEvent.get_active_window_event() From 265bd0dbd069d8055662f8d692ddb516b5531591 Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Wed, 24 Jul 2024 02:21:44 +0530 Subject: [PATCH 6/9] feat: Add segmentation logs to replay logs --- openadapt/app/dashboard/api/replays.py | 71 +++++++++------ .../app/dashboard/app/replay_logs/page.tsx | 5 -- .../app/dashboard/app/replays/logs/page.tsx | 87 +++++++++++++++++++ openadapt/app/dashboard/app/replays/page.tsx | 47 ++++++++++ openadapt/app/dashboard/app/routes.ts | 4 +- .../ReplayDetails/ReplayDetails.tsx | 49 +++++++++++ .../components/ReplayDetails/index.tsx | 1 + .../dashboard/components/ReplayLogs/index.tsx | 65 ++++++++++++++ .../components/SimpleTable/SimpleTable.tsx | 4 +- openadapt/app/dashboard/types/replay.ts | 17 ++++ openadapt/db/crud.py | 16 ++++ openadapt/strategies/visual.py | 40 ++++++--- 12 files changed, 355 insertions(+), 51 deletions(-) delete mode 100644 openadapt/app/dashboard/app/replay_logs/page.tsx create mode 100644 openadapt/app/dashboard/app/replays/logs/page.tsx create mode 100644 openadapt/app/dashboard/app/replays/page.tsx create mode 100644 openadapt/app/dashboard/components/ReplayDetails/ReplayDetails.tsx create mode 100644 openadapt/app/dashboard/components/ReplayDetails/index.tsx create mode 100644 openadapt/app/dashboard/components/ReplayLogs/index.tsx create mode 100644 openadapt/app/dashboard/types/replay.ts diff --git a/openadapt/app/dashboard/api/replays.py b/openadapt/app/dashboard/api/replays.py index 7004cb427..3c7f1a777 100644 --- a/openadapt/app/dashboard/api/replays.py +++ b/openadapt/app/dashboard/api/replays.py @@ -1,10 +1,9 @@ """API endpoints for replays.""" +import pickle -from typing import Any, Literal +from fastapi import APIRouter, WebSocket +from loguru import logger -from fastapi import APIRouter - -from openadapt.config import Config, config, persist_config from openadapt.db import crud from openadapt.utils import image2utf8 @@ -19,34 +18,50 @@ def __init__(self) -> None: def attach_routes(self) -> APIRouter: """Attach routes to the FastAPI app.""" self.app.add_api_route("", self.get_replays, methods=["GET"]) - self.app.add_api_route( - "/{replay_id}/logs", self.get_replay_logs, methods=["GET"] - ) + self.replay_logs_route() return self.app @staticmethod - def get_replays(): + def get_replays() -> list[dict]: """Get all replays.""" session = crud.get_new_session(read_only=True) return crud.get_replays(session) - @staticmethod - def get_replay_logs(replay_id: int): - """Get all logs for a replay.""" - session = crud.get_new_session(read_only=True) - logs = crud.get_replay_logs(session, replay_id) - unpickled_logs = [] - import base64 - import pickle - - def binary_to_base64_uri(binary_data): - base64_data = base64.b64encode(binary_data).decode("utf-8") - base64_uri = f"data:image/png;base64,{base64_data}" - return base64_uri - - for log in logs: - log.data = pickle.loads(log.data) - if log.key == "screenshot": - log.data = image2utf8(log.data) - unpickled_logs.append(log) - return unpickled_logs + def replay_logs_route(self) -> None: + """Add the replay detail route as a websocket.""" + + @self.app.websocket("/{replay_id}/logs") + async def get_replay_logs(websocket: WebSocket, replay_id: int) -> None: + """Get a specific replay and its logs.""" + await websocket.accept() + session = crud.get_new_session(read_only=True) + + replay = crud.get_replay(session, replay_id) + + await websocket.send_json({"type": "replay", "value": replay.asdict()}) + + logs = crud.get_replay_logs(session, replay_id) + + await websocket.send_json({"type": "num_logs", "value": len(logs)}) + + for log in logs: + log.data = pickle.loads(log.data) + if log.key == "screenshot": + log.data = image2utf8(log.data) + log_dict = log.asdict() + if log.key == "window_event": + log_dict["data"] = log_dict["data"].asdict() + if log.key == "action_event_dict": + log_dict["data"]["reducer_names"] = list( + log_dict["data"]["reducer_names"] + ) + if log.key == "segmentation": + log_dict["data"] = log_dict["data"].asdict() + try: + await websocket.send_json({"type": "log", "value": log_dict}) + except Exception as e: + logger.error(f"Error sending log: {e}") + logger.info(log_dict["data"]) + logger.info(log_dict["key"]) + + await websocket.close() diff --git a/openadapt/app/dashboard/app/replay_logs/page.tsx b/openadapt/app/dashboard/app/replay_logs/page.tsx deleted file mode 100644 index 70a6ea447..000000000 --- a/openadapt/app/dashboard/app/replay_logs/page.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export default function ReplayLogs() { - return ( - Replay logs - ) -} diff --git a/openadapt/app/dashboard/app/replays/logs/page.tsx b/openadapt/app/dashboard/app/replays/logs/page.tsx new file mode 100644 index 000000000..9990194a0 --- /dev/null +++ b/openadapt/app/dashboard/app/replays/logs/page.tsx @@ -0,0 +1,87 @@ +'use client'; + +import { ReplayDetails } from "@/components/ReplayDetails"; +import { ReplayLogs } from "@/components/ReplayLogs"; +import { ReplayLog, Replay as ReplayType } from "@/types/replay"; +import { Box, Loader, Progress } from "@mantine/core"; +import { useSearchParams } from "next/navigation"; +import { Suspense, useEffect, useState } from "react"; + +function Replay() { + const searchParams = useSearchParams(); + const id = searchParams.get("id"); + const [replayInfo, setReplayInfo] = useState<{ + replay: ReplayType, + logs: ReplayLog[], + num_logs: number, + }>(); + useEffect(() => { + if (!id) { + return; + } + const websocket = new WebSocket(`ws://${window.location.host}/api/replays/${id}/logs`); + websocket.onmessage = (event) => { + const data = JSON.parse(event.data); + if (data.type === "replay") { + setReplayInfo(prev => { + if (!prev) { + return { + "replay": data.value, + "logs": [], + "num_logs": 0, + } + } + return prev; + }); + } else if (data.type === "log") { + setReplayInfo(prev => { + if (!prev) return prev; + return { + ...prev, + "logs": [...prev.logs, data.value], + } + }); + } else if (data.type === "num_logs") { + setReplayInfo(prev => { + if (!prev) return prev; + return { + ...prev, + "num_logs": data.value, + } + }); + } + } + + return () => { + websocket.close(); + } + }, [id]); + if (!replayInfo) { + return ; + } + + const logs = replayInfo.logs; + + return ( + + + {logs.length && logs.length < replayInfo.num_logs && ( + + + Loading events {logs.length}/{replayInfo.num_logs} + + + )} + + + ) +} + + +export default function ReplayPage() { + return ( + + + + ) +} diff --git a/openadapt/app/dashboard/app/replays/page.tsx b/openadapt/app/dashboard/app/replays/page.tsx new file mode 100644 index 000000000..d2e65ef0c --- /dev/null +++ b/openadapt/app/dashboard/app/replays/page.tsx @@ -0,0 +1,47 @@ +'use client'; + +import { SimpleTable } from "@/components/SimpleTable"; +import { Replay } from "@/types/replay"; +import { useEffect, useState } from "react"; +import { timeStampToDateString } from "../utils"; +import { Box, Text } from "@mantine/core"; +import { useRouter } from "next/navigation"; + +export default function Replays() { + const [replays, setReplays] = useState([]); + const fetchReplays = () => { + fetch('/api/replays').then(res => { + if (res.ok) { + res.json().then((data) => { + setReplays(data); + }); + } + }); + } + const router = useRouter(); + + function onClickRow(replay: Replay) { + return () => router.push(`/replays/logs?id=${replay.id}`); + } + + useEffect(() => { + fetchReplays(); + }, []); + + return ( + + Replay logs + JSON.stringify(replay.strategy_args)}, + {name: 'Time', accessor: (replay) => timeStampToDateString(replay.timestamp)}, + ]} + data={replays} + refreshData={fetchReplays} + onClickRow={onClickRow} + /> + + ) +} diff --git a/openadapt/app/dashboard/app/routes.ts b/openadapt/app/dashboard/app/routes.ts index c5dabf8c8..12dc7dbb6 100644 --- a/openadapt/app/dashboard/app/routes.ts +++ b/openadapt/app/dashboard/app/routes.ts @@ -21,7 +21,7 @@ export const routes: Route[] = [ path: '/onboarding', }, { - name: "Replay logs", - path: "/replay_logs", + name: "Replays", + path: "/replays", } ] diff --git a/openadapt/app/dashboard/components/ReplayDetails/ReplayDetails.tsx b/openadapt/app/dashboard/components/ReplayDetails/ReplayDetails.tsx new file mode 100644 index 000000000..ee4bf867c --- /dev/null +++ b/openadapt/app/dashboard/components/ReplayDetails/ReplayDetails.tsx @@ -0,0 +1,49 @@ +'use client'; + +import { timeStampToDateString } from '@/app/utils'; +import { Replay } from '@/types/replay' +import { Code, Table } from '@mantine/core' +import React from 'react' + +type Props = { + replay: Replay; +} + +const TableRowWithBorder = ({ children }: { children: React.ReactNode }) => ( + + {children} + +) + +const TableCellWithBorder = ({ children }: { children: React.ReactNode }) => ( + + {children} + +) + +export const ReplayDetails = ({ + replay +}: Props) => { + return ( + + + + Replay ID + {replay.id} + + + timestamp + {timeStampToDateString(replay.timestamp)} + + + strategy name + {replay.strategy_name} + + + strategy args + {JSON.stringify(replay.strategy_args)} + + +
+ ) +} diff --git a/openadapt/app/dashboard/components/ReplayDetails/index.tsx b/openadapt/app/dashboard/components/ReplayDetails/index.tsx new file mode 100644 index 000000000..395c81d8c --- /dev/null +++ b/openadapt/app/dashboard/components/ReplayDetails/index.tsx @@ -0,0 +1 @@ +export { ReplayDetails } from './ReplayDetails'; diff --git a/openadapt/app/dashboard/components/ReplayLogs/index.tsx b/openadapt/app/dashboard/components/ReplayLogs/index.tsx new file mode 100644 index 000000000..6fe3c18d1 --- /dev/null +++ b/openadapt/app/dashboard/components/ReplayLogs/index.tsx @@ -0,0 +1,65 @@ +import { ReplayLog as ReplayLogType } from '@/types/replay'; +import { Accordion, Box, Code, Grid, Image, Text } from '@mantine/core'; +import React from 'react' + +type Props = { + logs: ReplayLogType[]; +} + +export const ReplayLogs = ({ + logs +}: Props) => { + return ( + + {logs.map((log) => ( + + {log.key} - {log.filename}:{log.lineno} + + {log.key === "screenshot" ? ( + screenshot + ) : (log.key === "segmentation" ? ( + <> + segmentation + Marked Image + segmentation + Masked Images + + {log.data.masked_images.map((masked_image: string, i: number) => { + let boundingBox = log.data.bounding_boxes[i]; + let centroid = log.data.centroids[i]; + const description = log.data.descriptions[i]; + const shape = log.data.image_shapes[i]; + + boundingBox = { + left: boundingBox[0] * 160 / shape[0], + top: boundingBox[1] * 100 / shape[1], + width: boundingBox[2] * 160 / shape[0], + height: boundingBox[3] * 100 / shape[1], + } + centroid = [ + centroid[0] * 160 / shape[0], + centroid[1] * 100 / shape[1], + ] + + return ( + + + + + + + {description} + + ) + })} + + + ): ( + 
{JSON.stringify(log.data, null, 4)}
+ ))} + + + ))} + + ) +} diff --git a/openadapt/app/dashboard/components/SimpleTable/SimpleTable.tsx b/openadapt/app/dashboard/components/SimpleTable/SimpleTable.tsx index b80ae2838..218a53ba8 100644 --- a/openadapt/app/dashboard/components/SimpleTable/SimpleTable.tsx +++ b/openadapt/app/dashboard/components/SimpleTable/SimpleTable.tsx @@ -9,7 +9,7 @@ type Props> = { }[]; data: T[], refreshData: () => void, - onClickRow: (row: T) => (event: React.MouseEvent) => void, + onClickRow?: (row: T) => (event: React.MouseEvent) => void, } export function SimpleTable>({ @@ -33,7 +33,7 @@ export function SimpleTable>({ {data.map((row, rowIndex) => ( - + {}}> {columns.map(({accessor}, accesorIndex) => ( {typeof accessor === 'string' ? row[accessor] : accessor(row)} diff --git a/openadapt/app/dashboard/types/replay.ts b/openadapt/app/dashboard/types/replay.ts new file mode 100644 index 000000000..c4770bf0f --- /dev/null +++ b/openadapt/app/dashboard/types/replay.ts @@ -0,0 +1,17 @@ +export type Replay = { + id: number; + strategy_name: string; + strategy_args: Record; + timestamp: number; +} + + +export type ReplayLog = { + id: number; + replay_id: number; + lineno: number; + filename: string; + timestamp: number; + key: string; + data: Record; +} diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index 609fdd21b..5ad153a78 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -808,6 +808,22 @@ def get_replays( return session.query(Replay).order_by(sa.desc(Replay.timestamp)).all() +def get_replay( + session: SaSession, + replay_id: int, +) -> Replay: + """Get a replay by its id. + + Args: + session (sa.orm.Session): The database session. + replay_id (int): The id of the replay. + + Returns: + Replay: The replay object. + """ + return session.query(Replay).get(replay_id) + + def get_replay_logs( session: SaSession, replay_id: int, diff --git a/openadapt/strategies/visual.py b/openadapt/strategies/visual.py index 7aaa00cab..da007ff4e 100644 --- a/openadapt/strategies/visual.py +++ b/openadapt/strategies/visual.py @@ -52,15 +52,8 @@ from PIL import Image, ImageDraw import numpy as np -from openadapt import ( - adapters, - common, - models, - plotting, - strategies, - utils, - vision, -) +from openadapt import adapters, common, models, plotting, strategies, utils, vision +from openadapt.db import crud DEBUG = False DEBUG_REPLAY = False @@ -97,16 +90,19 @@ class Segmentation: centroids: list[tuple[float, float]] -def add_active_segment_descriptions(action_events: list[models.ActionEvent]) -> None: +def add_active_segment_descriptions( + action_events: list[models.ActionEvent], replay_id: int +) -> None: """Set the ActionEvent.active_segment_description where appropriate. Args: action_events: list of ActionEvents to modify in-place. + replay_id: the replay ID """ for action in action_events: # TODO: handle terminal event if action.name in common.MOUSE_EVENTS: - window_segmentation = get_window_segmentation(action) + window_segmentation = get_window_segmentation(action, replay_id=replay_id) active_segment_idx = get_active_segment(action, window_segmentation) if not active_segment_idx: logger.warning(f"{active_segment_idx=}") @@ -179,14 +175,21 @@ def __init__( super().__init__(recording) self.recording_action_idx = 0 self.action_history = [] - add_active_segment_descriptions(recording.processed_action_events) + self.instructions = instructions + + def run(self) -> None: + """Run the VisualReplayStrategy.""" + add_active_segment_descriptions( + self.recording.processed_action_events, self._replay_id + ) self.modified_actions = apply_replay_instructions( - recording.processed_action_events, - instructions, + self.recording.processed_action_events, + self.instructions, ) # TODO: make this less of a hack global DEBUG DEBUG = DEBUG_REPLAY + super().run() def get_next_action_event( self, @@ -228,6 +231,7 @@ def get_next_action_event( active_window_segmentation = get_window_segmentation( modified_reference_action, exceptions=exceptions, + replay_id=self._replay_id, ) try: target_segment_idx = active_window_segmentation.descriptions.index( @@ -378,6 +382,7 @@ def get_window_segmentation( action_event: models.ActionEvent, exceptions: list[Exception] | None = None, handle_similar_image_groups: bool = False, + replay_id: int | None = None, ) -> Segmentation: """Segments the active window from the action event's screenshot. @@ -456,6 +461,13 @@ def get_window_segmentation( plotting.display_images_table_with_titles(masked_images, descriptions) SEGMENTATIONS.append(segmentation) + if replay_id: + crud.add_replay_log( + replay_id=replay_id, + log_level="INFO", + key="segmentation", + data=segmentation, + ) return segmentation From 85027e954f0e16b3d9bb9080cc744ab12a7e5ce6 Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Wed, 24 Jul 2024 02:54:00 +0530 Subject: [PATCH 7/9] fix: Fix visual replay strategy logs --- openadapt/app/dashboard/api/replays.py | 7 ++++-- .../dashboard/components/ReplayLogs/index.tsx | 24 +++++-------------- openadapt/strategies/visual.py | 12 ++++++++++ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/openadapt/app/dashboard/api/replays.py b/openadapt/app/dashboard/api/replays.py index 3c7f1a777..6c02a2662 100644 --- a/openadapt/app/dashboard/api/replays.py +++ b/openadapt/app/dashboard/api/replays.py @@ -5,6 +5,7 @@ from loguru import logger from openadapt.db import crud +from openadapt.models import Replay from openadapt.utils import image2utf8 @@ -17,12 +18,14 @@ def __init__(self) -> None: def attach_routes(self) -> APIRouter: """Attach routes to the FastAPI app.""" - self.app.add_api_route("", self.get_replays, methods=["GET"]) + self.app.add_api_route( + "", self.get_replays, methods=["GET"], response_model=None + ) self.replay_logs_route() return self.app @staticmethod - def get_replays() -> list[dict]: + def get_replays() -> list[Replay]: """Get all replays.""" session = crud.get_new_session(read_only=True) return crud.get_replays(session) diff --git a/openadapt/app/dashboard/components/ReplayLogs/index.tsx b/openadapt/app/dashboard/components/ReplayLogs/index.tsx index 6fe3c18d1..356ef5f00 100644 --- a/openadapt/app/dashboard/components/ReplayLogs/index.tsx +++ b/openadapt/app/dashboard/components/ReplayLogs/index.tsx @@ -24,28 +24,16 @@ export const ReplayLogs = ({ segmentation Masked Images - {log.data.masked_images.map((masked_image: string, i: number) => { + {log.data.centroids.map((centroid: [number, number], i: number) => { let boundingBox = log.data.bounding_boxes[i]; - let centroid = log.data.centroids[i]; const description = log.data.descriptions[i]; - const shape = log.data.image_shapes[i]; - - boundingBox = { - left: boundingBox[0] * 160 / shape[0], - top: boundingBox[1] * 100 / shape[1], - width: boundingBox[2] * 160 / shape[0], - height: boundingBox[3] * 100 / shape[1], - } - centroid = [ - centroid[0] * 160 / shape[0], - centroid[1] * 100 / shape[1], - ] + const imageShape = log.data.image_shape; return ( - - - - + + + + {description} diff --git a/openadapt/strategies/visual.py b/openadapt/strategies/visual.py index 5047c5261..91b90fa16 100644 --- a/openadapt/strategies/visual.py +++ b/openadapt/strategies/visual.py @@ -89,6 +89,18 @@ class Segmentation: bounding_boxes: list[dict[str, float]] # "top", "left", "height", "width" centroids: list[tuple[float, float]] + def asdict(self) -> dict: + from openadapt.utils import image2utf8 + + return { + "image": image2utf8(self.image), + "image_shape": self.image.size, + "marked_image": image2utf8(self.marked_image), + "descriptions": self.descriptions, + "bounding_boxes": self.bounding_boxes, + "centroids": self.centroids, + } + def add_active_segment_descriptions( action_events: list[models.ActionEvent], replay_id: int From f81dde6a9b90ab3a07ce70d6d42b12e12b220b51 Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Wed, 24 Jul 2024 17:12:14 +0530 Subject: [PATCH 8/9] lint: flake8 lint --- openadapt/strategies/visual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openadapt/strategies/visual.py b/openadapt/strategies/visual.py index 91b90fa16..e7d781261 100644 --- a/openadapt/strategies/visual.py +++ b/openadapt/strategies/visual.py @@ -54,6 +54,7 @@ from openadapt import adapters, common, models, plotting, strategies, utils, vision from openadapt.custom_logger import logger from openadapt.db import crud +from openadapt.utils import image2utf8 DEBUG = False DEBUG_REPLAY = False @@ -90,8 +91,7 @@ class Segmentation: centroids: list[tuple[float, float]] def asdict(self) -> dict: - from openadapt.utils import image2utf8 - + """Return the Segmentation as a dictionary.""" return { "image": image2utf8(self.image), "image_shape": self.image.size, From fa137a704b5a62349a3a046937fcd8176e8a12c4 Mon Sep 17 00:00:00 2001 From: Shohan Dutta Roy Date: Fri, 26 Jul 2024 00:09:41 +0530 Subject: [PATCH 9/9] feat: Add depth kwarg to replay log utility function --- openadapt/db/crud.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index c5eab6ebe..2ae77ace9 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -757,7 +757,9 @@ def add_replay( return replay.id -def add_replay_log(*, replay_id: int, log_level: str, key: str, data: Any) -> None: +def add_replay_log( + *, replay_id: int, log_level: str, key: str, data: Any, depth: int = 2 +) -> None: """Add a replay log entry to the database. Args: @@ -765,11 +767,12 @@ def add_replay_log(*, replay_id: int, log_level: str, key: str, data: Any) -> No log_level (str): The log level of the log entry. key (str): The key of the log entry. data (Any): The data of the log entry. + depth (int): The depth of the stack frame to get the caller info from. """ with get_new_session(read_and_write=True) as session: pickled_data = pickle.dumps(data) - frame = sys._getframe(1) + frame = sys._getframe(depth) caller_line = frame.f_lineno caller_file = frame.f_code.co_filename