Add ffmpeg-sidecar support (#580)

.github/scripts/run_screenpipe.sh

@@ -4,4 +4,9 @@ export PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native
 ./target/release/screenpipe --debug > screenpipe_output.log 2>&1 &
 SCREENPIPE_PID=$!
 echo $SCREENPIPE_PID > screenpipe.pid
-sleep 60
+# Check resource usage every 10 seconds, for 1 minute
+for i in {1..6}
+do
+   sleep 10
+   ps -p $SCREENPIPE_PID -o %cpu,%mem,cmd
+done

.github/scripts/test_audio_capture.sh

@@ -4,7 +4,12 @@ pulseaudio --check
 ps aux | grep pulseaudio
 ls -l /run/user/$(id -u)/pulse/
 PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native paplay --verbose .github/scripts/audio_test.wav
-sleep 30
+# Check resource usage every 10 seconds, for 30 seconds
+for i in {1..3}
+do
+   sleep 10
+   ps -p $(cat screenpipe.pid) -o %cpu,%mem,cmd
+done
 cat screenpipe_output.log
 if grep -qi "human world" screenpipe_output.log; then
   echo "Audio capture test passed: 'human world' found in logs"

.github/scripts/test_ocr.sh

@@ -2,7 +2,12 @@
 convert -size 300x100 xc:white -font DejaVu-Sans -pointsize 24 -fill black -draw "text 10,50 'Hello, Screenpipe OCR'" test_image.png
 DISPLAY=:99 display test_image.png &
 DISPLAY_PID=$!
-sleep 30
+# Check resource usage every 10 seconds, for 30 seconds
+for i in {1..3}
+do
+   sleep 10
+   ps -p $(cat screenpipe.pid) -o %cpu,%mem,cmd
+done
 kill $DISPLAY_PID
 if grep -qi "Hello, Screenpipe OCR" screenpipe_output.log; then
   echo "OCR test passed: Text was recognized"

.github/workflows/linux-integration-test.yml

@@ -49,7 +49,10 @@ jobs:
 
     - name: Check for crashes and expected behavior
       run: .github/scripts/check_logs.sh
-
+
+    - name: Check final storage usage
+      run: du -ha ~/.screenpipe/data
+
     - name: Upload logs
       uses: actions/upload-artifact@v4
       with:

screenpipe-audio/Cargo.toml

@@ -13,6 +13,7 @@ edition = { workspace = true }
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+tar = "=0.4.42"
 
 # Cross-platform audio capture
 # cpal = "0.15.2"

screenpipe-core/Cargo.toml

@@ -11,6 +11,7 @@ edition = { workspace = true }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 which = "6.0.1"
+ffmpeg-sidecar = { git = "https://github.com/nathanbabcock/ffmpeg-sidecar", branch = "main" }
 log = "0.4.17"
 anyhow = "1.0.86"
 candle = { workspace = true, optional = true }

screenpipe-core/src/ffmpeg.rs

@@ -1,6 +1,12 @@
 use log::{debug, error};
 use std::path::PathBuf;
 use which::which;
+use ffmpeg_sidecar::{
+    command::ffmpeg_is_installed,
+    download::{check_latest_version, download_ffmpeg_package, ffmpeg_download_url, unpack_ffmpeg},
+    paths::sidecar_dir,
+    version::ffmpeg_version,
+};
 use once_cell::sync::Lazy;
 
 #[cfg(not(windows))]
@@ -83,6 +89,54 @@ fn find_ffmpeg_path_internal() -> Option<PathBuf> {
         }
     }
 
-    error!("ffmpeg not found");
+    debug!("ffmpeg not found. installing...");
+
+    if let Err(error) = handle_ffmpeg_installation() {
+        error!("failed to install ffmpeg: {}", error);
+        return None;
+    }
+
+    if let Ok(path) = which(EXECUTABLE_NAME) {
+        debug!("found ffmpeg after installation: {:?}", path);
+        return Some(path);
+    }
+
+    let installation_dir = sidecar_dir().map_err(|e| e.to_string()).unwrap();
+    let ffmpeg_in_installation = installation_dir.join(EXECUTABLE_NAME);
+    if ffmpeg_in_installation.is_file() {
+        debug!("found ffmpeg in directory: {:?}", ffmpeg_in_installation);
+        return Some(ffmpeg_in_installation);
+    }
+
+    error!("ffmpeg not found even after installation");
     None // Return None if ffmpeg is not found
 }
+
+fn handle_ffmpeg_installation() -> Result<(), String> {
+    if ffmpeg_is_installed() {
+        debug!("ffmpeg is already installed");
+        return Ok(());
+    }
+
+    debug!("ffmpeg not found. installing...");
+    match check_latest_version() {
+        Ok(version) => debug!("latest version: {}", version),
+        Err(e) => debug!("skipping version check due to error: {e}"),
+    }
+
+    let download_url = ffmpeg_download_url().map_err(|e| e.to_string())?;
+    let destination = sidecar_dir().map_err(|e| e.to_string())?;
+
+    debug!("downloading from: {:?}", download_url);
+    let archive_path =
+        download_ffmpeg_package(download_url, &destination).map_err(|e| e.to_string())?;
+    debug!("downloaded package: {:?}", archive_path);
+
+    debug!("extracting...");
+    unpack_ffmpeg(&archive_path, &destination).map_err(|e| e.to_string())?;
+
+    let version = ffmpeg_version().map_err(|e| e.to_string())?;
+
+    debug!("done! installed ffmpeg version {}", version);
+    Ok(())
+}

screenpipe-server/src/bin/screenpipe-server.rs

@@ -185,6 +185,13 @@ async fn main() -> anyhow::Result<()> {
                 SileroVad::new().await.unwrap();
 
                 // Check if FFmpeg is working properly
+                if let Some(ffmpeg_path) = find_ffmpeg_path() {
+                    println!("ffmpeg found at: {:?}", ffmpeg_path);
+                } else {
+                    eprintln!("failed to find or install ffmpeg.");
+                    return Err(anyhow::anyhow!("ffmpeg installation failed"));
+                }
+
                 match check_ffmpeg().await {
                     Ok(_) => info!("FFmpeg is working properly"),
                     Err(e) => {
@@ -196,7 +203,6 @@ async fn main() -> anyhow::Result<()> {
                 }
 
                 info!("screenpipe setup complete");
-                // TODO: ffmpeg sidecar thing here
                 return Ok(());
             }
         }

screenpipe-server/src/video.rs

@@ -168,21 +168,7 @@ pub async fn start_ffmpeg_process(output_file: &str, fps: f64) -> Result<Child,
         "pad=width=ceil(iw/2)*2:height=ceil(ih/2)*2",
     ];
 
-    if env::consts::OS == "windows" {
-        // TODO switch back to libx264 when ffmpeg is updated in pre_build.js
-        // Use H264_mf encoder for Windows
-        args.extend_from_slice(&[
-            "-vcodec",
-            "h264_mf",
-            "-q:v",
-            "5", // Adjust quality (1-31, lower is better)
-            "-preset",
-            "ultrafast",
-        ]);
-    } else {
-        // Use libx264 for other platforms
-        args.extend_from_slice(&["-vcodec", "libx264", "-preset", "ultrafast", "-crf", "23"]);
-    }
+    args.extend_from_slice(&["-vcodec", "libx265", "-preset", "ultrafast", "-crf", "23"]);
 
     args.extend_from_slice(&["-pix_fmt", "yuv420p", output_file]);