refactor: make timeline a timeseries data structure for efficiency

mediar-ai · Nov 3, 2024 · 3c2010e · 3c2010e
1 parent 68a45a0
commit 3c2010e
Show file tree

Hide file tree

Showing 10 changed files with 407 additions and 531 deletions.
diff --git a/screenpipe-app-tauri/app/timeline/page.tsx b/screenpipe-app-tauri/app/timeline/page.tsx
diff --git a/screenpipe-app-tauri/src-tauri/Cargo.lock b/screenpipe-app-tauri/src-tauri/Cargo.lock
diff --git a/screenpipe-app-tauri/src-tauri/Cargo.toml b/screenpipe-app-tauri/src-tauri/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "screenpipe-app"
-version = "0.8.5"
+version = "0.8.6"
 description = ""
 authors = ["you"]
 license = ""

diff --git a/screenpipe-server/examples/timeline_ui_simple/index.html b/screenpipe-server/examples/timeline_ui_simple/index.html
@@ -97,6 +97,18 @@
         .icon:hover .tooltip {
             display: block;
         }
+
+        .devices-container {
+            display: flex;
+            gap: 20px;
+            border-bottom: 1px solid #333;
+            padding-bottom: 20px;
+        }
+
+        .device-frame {
+            flex: 1;
+            min-width: 0;
+        }
     </style>
 </head>
 <body>
@@ -128,55 +140,72 @@
             document.getElementById('frames').innerHTML = '';
         }
 
-        function appendFrame(frame, timestamp, data) {
+        function appendFrame(data) {
             const container = document.createElement('div');
             container.className = 'frame-container';
 
-            const img = document.createElement('img');
-            img.onerror = (e) => console.error('Failed to load image:', e);
-            img.onload = () => console.log('Image loaded successfully');
-
-            const cleanBase64 = frame.trim().replace(/\s/g, '');
-            img.src = `data:image/jpeg;base64,${cleanBase64}`;
-
-            img.style.display = 'block';
-            img.style.minHeight = '100px';
-
             const ts = document.createElement('div');
             ts.className = 'frame-timestamp';
-            ts.innerText = new Date(timestamp).toLocaleString();
-
-            const metadata = document.createElement('div');
-            metadata.className = 'metadata-icons';
+            ts.innerText = new Date(data.timestamp).toLocaleString();
+            container.appendChild(ts);
 
-            metadata.innerHTML = `
-                <span>window: ${(data && data.window_name) || 'N/A'}</span>
-                <span>app: ${(data && data.app_name) || 'N/A'}</span>
-            `;
-
-            if (data && data.ocr_text) {
-                const ocrIcon = document.createElement('div');
-                ocrIcon.className = 'icon';
-                ocrIcon.innerHTML = `
-                    📝
-                    <div class="tooltip">OCR: ${data.ocr_text}</div>
+            const devicesContainer = document.createElement('div');
+            devicesContainer.className = 'devices-container';
+
+            data.devices.forEach(device => {
+                const deviceFrame = document.createElement('div');
+                deviceFrame.className = 'device-frame';
+
+                const img = document.createElement('img');
+                img.onerror = (e) => console.error('Failed to load image:', e);
+                img.onload = () => console.log('Image loaded successfully');
+
+                const cleanBase64 = device.frame.trim().replace(/\s/g, '');
+                img.src = `data:image/jpeg;base64,${cleanBase64}`;
+
+                img.style.display = 'block';
+                img.style.minHeight = '100px';
+
+                const metadata = document.createElement('div');
+                metadata.className = 'metadata-icons';
+
+                metadata.innerHTML = `
+                    <span>device: ${device.device_id}</span>
+                    <span>window: ${device.metadata.window_name || 'N/A'}</span>
+                    <span>app: ${device.metadata.app_name || 'N/A'}</span>
                 `;
-                metadata.appendChild(ocrIcon);
-            }
 
-            if (data && data.transcription) {
-                const transcriptionIcon = document.createElement('div');
-                transcriptionIcon.className = 'icon';
-                transcriptionIcon.innerHTML = `
-                    🎤
-                    <div class="tooltip">Transcription: ${data.transcription}</div>
-                `;
-                metadata.appendChild(transcriptionIcon);
-            }
+                if (device.metadata.ocr_text) {
+                    const ocrIcon = document.createElement('div');
+                    ocrIcon.className = 'icon';
+                    ocrIcon.innerHTML = `
+                        📝
+                        <div class="tooltip">OCR: ${device.metadata.ocr_text}</div>
+                    `;
+                    metadata.appendChild(ocrIcon);
+                }
+
+                device.audio.forEach(audio => {
+                    const audioIcon = document.createElement('div');
+                    audioIcon.className = 'icon';
+                    audioIcon.innerHTML = `
+                        ${audio.is_input ? '🎤' : '🔊'}
+                        <div class="tooltip">
+                            Device: ${audio.device_name}
+                            Duration: ${audio.duration_secs}s
+                            Offset: ${audio.start_offset}s
+                            Transcription: ${audio.transcription}
+                        </div>
+                    `;
+                    metadata.appendChild(audioIcon);
+                });
+
+                deviceFrame.appendChild(img);
+                deviceFrame.appendChild(metadata);
+                devicesContainer.appendChild(deviceFrame);
+            });
 
-            container.appendChild(img);
-            container.appendChild(ts);
-            container.appendChild(metadata);
+            container.appendChild(devicesContainer);
 
             const frames = document.getElementById('frames');
             frames.appendChild(container);
@@ -203,9 +232,9 @@
                         return;
                     }
 
-                    if (data.frame && data.timestamp) {
-                        console.log('appending frame from:', data.timestamp, data);
-                        appendFrame(data.frame, data.timestamp, data);
+                    if (data.timestamp && data.devices) {
+                        console.log('appending frames from:', data.timestamp, data);
+                        appendFrame(data);
                     }
                 } catch (e) {
                     console.error('Failed to parse frame data:', e);

diff --git a/screenpipe-server/src/bin/screenpipe-server.rs b/screenpipe-server/src/bin/screenpipe-server.rs
@@ -100,7 +100,6 @@ fn setup_logging(local_data_dir: &PathBuf, cli: &Cli) -> anyhow::Result<WorkerGu
         .with(fmt::layer().with_writer(non_blocking))
         .init();
 
-    info!("logging initialized");
     Ok(guard)
 }
 
@@ -264,7 +263,6 @@ async fn main() -> anyhow::Result<()> {
             eprintln!("no audio devices available. audio recording will be disabled.");
         } else {
             for device in &audio_devices {
-                info!("  {}", device);
 
                 let device_control = DeviceControl {
                     is_running: true,
@@ -292,10 +290,7 @@ async fn main() -> anyhow::Result<()> {
                 e
             })?,
     );
-    info!(
-        "database initialized, will store files in {}",
-        local_data_dir.to_string_lossy()
-    );
+
     let db_server = db.clone();
 
     // Channel for controlling the recorder ! TODO RENAME SHIT
@@ -496,6 +491,9 @@ async fn main() -> anyhow::Result<()> {
         "│ friend wearable uid │ {:<34} │",
         cli.friend_wearable_uid.as_deref().unwrap_or("not set")
     );
+    println!("│ ui monitoring       │ {:<34} │", cli.enable_ui_monitoring);
+    println!("│ frame cache         │ {:<34} │", cli.enable_frame_cache);
+
     const VALUE_WIDTH: usize = 34;
 
     // Function to truncate and pad strings
@@ -613,8 +611,6 @@ async fn main() -> anyhow::Result<()> {
         }
     }
 
-    println!("│ ui monitoring       │ {:<34} │", cli.enable_ui_monitoring);
-
     println!("└─────────────────────┴────────────────────────────────────┘");
 
     // Add warning for cloud arguments and telemetry

diff --git a/screenpipe-server/src/db.rs b/screenpipe-server/src/db.rs
@@ -189,7 +189,7 @@ impl DatabaseManager {
             return Err(e);
         }
 
-        info!("Migrations executed successfully.");
+        debug!("migrations executed successfully.");
         Ok(db_manager)
     }
 
@@ -1175,7 +1175,7 @@ impl DatabaseManager {
             JOIN video_chunks vc ON f.video_chunk_id = vc.id
             LEFT JOIN ocr_text ot ON f.id = ot.frame_id
             WHERE f.timestamp >= ?1 AND f.timestamp <= ?2
-            ORDER BY f.timestamp, f.offset_index
+            ORDER BY f.timestamp DESC, f.offset_index DESC
         "#;
 
         // Then get audio data that overlaps with these frames
@@ -1189,7 +1189,7 @@ impl DatabaseManager {
             FROM audio_transcriptions at
             JOIN audio_chunks ac ON at.audio_chunk_id = ac.id
             WHERE at.timestamp >= ?1 AND at.timestamp <= ?2
-            ORDER BY at.timestamp
+            ORDER BY at.timestamp DESC
         "#;
 
         // Execute both queries
@@ -1251,7 +1251,7 @@ impl DatabaseManager {
         }
 
         Ok(TimeSeriesChunk {
-            frames: frames_map.into_values().collect(),
+            frames: frames_map.into_values().rev().collect(),
             start_time: start,
             end_time: end,
         })

diff --git a/screenpipe-server/src/migrations/20241103222151_timeline_indexes.sql b/screenpipe-server/src/migrations/20241103222151_timeline_indexes.sql
@@ -0,0 +1,9 @@
+-- Add migration script here
+
+-- For the frames query optimization
+CREATE INDEX IF NOT EXISTS idx_frames_timestamp_offset_index ON frames(timestamp, offset_index);
+CREATE INDEX IF NOT EXISTS idx_ocr_text_frame_id ON ocr_text(frame_id);
+
+-- Composite index for ocr_text that might help with the LEFT JOIN
+CREATE INDEX IF NOT EXISTS idx_ocr_text_frame_app_window ON ocr_text(frame_id, app_name, window_name);
+
diff --git a/screenpipe-server/src/video.rs b/screenpipe-server/src/video.rs
@@ -39,7 +39,6 @@ impl VideoCapture {
         include_list: &[String],
         languages: Vec<Language>,
     ) -> Self {
-        info!("Starting new video capture");
         let fps = if fps.is_finite() && fps > 0.0 {
             fps
         } else {
@@ -71,7 +70,6 @@ impl VideoCapture {
             .await;
         });
 
-        info!("Started capture thread");
 
         // In the _queue_thread
         let _queue_thread = tokio::spawn(async move {