From 93a4f6b7c259b34510bd17ec0a5d3517782ff467 Mon Sep 17 00:00:00 2001 From: Neptune Date: Mon, 11 Nov 2024 14:01:46 -0500 Subject: [PATCH] Add ffmpeg-sidecar support (#580) --- .github/scripts/run_screenpipe.sh | 7 ++- .github/scripts/test_audio_capture.sh | 7 ++- .github/scripts/test_ocr.sh | 7 ++- .github/workflows/linux-integration-test.yml | 5 +- screenpipe-audio/Cargo.toml | 1 + screenpipe-core/Cargo.toml | 1 + screenpipe-core/src/ffmpeg.rs | 56 ++++++++++++++++++- .../src/bin/screenpipe-server.rs | 8 ++- screenpipe-server/src/video.rs | 16 +----- 9 files changed, 87 insertions(+), 21 deletions(-) diff --git a/.github/scripts/run_screenpipe.sh b/.github/scripts/run_screenpipe.sh index 2e065b64c..8307037f9 100755 --- a/.github/scripts/run_screenpipe.sh +++ b/.github/scripts/run_screenpipe.sh @@ -4,4 +4,9 @@ export PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native ./target/release/screenpipe --debug > screenpipe_output.log 2>&1 & SCREENPIPE_PID=$! echo $SCREENPIPE_PID > screenpipe.pid -sleep 60 +# Check resource usage every 10 seconds, for 1 minute +for i in {1..6} +do + sleep 10 + ps -p $SCREENPIPE_PID -o %cpu,%mem,cmd +done diff --git a/.github/scripts/test_audio_capture.sh b/.github/scripts/test_audio_capture.sh index 76f1b7602..af0f0c7fb 100755 --- a/.github/scripts/test_audio_capture.sh +++ b/.github/scripts/test_audio_capture.sh @@ -4,7 +4,12 @@ pulseaudio --check ps aux | grep pulseaudio ls -l /run/user/$(id -u)/pulse/ PULSE_SERVER=unix:${XDG_RUNTIME_DIR}/pulse/native paplay --verbose .github/scripts/audio_test.wav -sleep 30 +# Check resource usage every 10 seconds, for 30 seconds +for i in {1..3} +do + sleep 10 + ps -p $(cat screenpipe.pid) -o %cpu,%mem,cmd +done cat screenpipe_output.log if grep -qi "human world" screenpipe_output.log; then echo "Audio capture test passed: 'human world' found in logs" diff --git a/.github/scripts/test_ocr.sh b/.github/scripts/test_ocr.sh index 9e7498441..b5f247043 100755 --- a/.github/scripts/test_ocr.sh +++ b/.github/scripts/test_ocr.sh @@ -2,7 +2,12 @@ convert -size 300x100 xc:white -font DejaVu-Sans -pointsize 24 -fill black -draw "text 10,50 'Hello, Screenpipe OCR'" test_image.png DISPLAY=:99 display test_image.png & DISPLAY_PID=$! -sleep 30 +# Check resource usage every 10 seconds, for 30 seconds +for i in {1..3} +do + sleep 10 + ps -p $(cat screenpipe.pid) -o %cpu,%mem,cmd +done kill $DISPLAY_PID if grep -qi "Hello, Screenpipe OCR" screenpipe_output.log; then echo "OCR test passed: Text was recognized" diff --git a/.github/workflows/linux-integration-test.yml b/.github/workflows/linux-integration-test.yml index 9ef397058..4d67009d5 100644 --- a/.github/workflows/linux-integration-test.yml +++ b/.github/workflows/linux-integration-test.yml @@ -49,7 +49,10 @@ jobs: - name: Check for crashes and expected behavior run: .github/scripts/check_logs.sh - + + - name: Check final storage usage + run: du -ha ~/.screenpipe/data + - name: Upload logs uses: actions/upload-artifact@v4 with: diff --git a/screenpipe-audio/Cargo.toml b/screenpipe-audio/Cargo.toml index 659e820dd..8db523e3f 100644 --- a/screenpipe-audio/Cargo.toml +++ b/screenpipe-audio/Cargo.toml @@ -13,6 +13,7 @@ edition = { workspace = true } [dependencies] serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +tar = "=0.4.42" # Cross-platform audio capture # cpal = "0.15.2" diff --git a/screenpipe-core/Cargo.toml b/screenpipe-core/Cargo.toml index a695c7528..6eff4069d 100644 --- a/screenpipe-core/Cargo.toml +++ b/screenpipe-core/Cargo.toml @@ -11,6 +11,7 @@ edition = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" which = "6.0.1" +ffmpeg-sidecar = { git = "https://github.com/nathanbabcock/ffmpeg-sidecar", branch = "main" } log = "0.4.17" anyhow = "1.0.86" candle = { workspace = true, optional = true } diff --git a/screenpipe-core/src/ffmpeg.rs b/screenpipe-core/src/ffmpeg.rs index c4f63a32c..1c390dad1 100644 --- a/screenpipe-core/src/ffmpeg.rs +++ b/screenpipe-core/src/ffmpeg.rs @@ -1,6 +1,12 @@ use log::{debug, error}; use std::path::PathBuf; use which::which; +use ffmpeg_sidecar::{ + command::ffmpeg_is_installed, + download::{check_latest_version, download_ffmpeg_package, ffmpeg_download_url, unpack_ffmpeg}, + paths::sidecar_dir, + version::ffmpeg_version, +}; use once_cell::sync::Lazy; #[cfg(not(windows))] @@ -83,6 +89,54 @@ fn find_ffmpeg_path_internal() -> Option { } } - error!("ffmpeg not found"); + debug!("ffmpeg not found. installing..."); + + if let Err(error) = handle_ffmpeg_installation() { + error!("failed to install ffmpeg: {}", error); + return None; + } + + if let Ok(path) = which(EXECUTABLE_NAME) { + debug!("found ffmpeg after installation: {:?}", path); + return Some(path); + } + + let installation_dir = sidecar_dir().map_err(|e| e.to_string()).unwrap(); + let ffmpeg_in_installation = installation_dir.join(EXECUTABLE_NAME); + if ffmpeg_in_installation.is_file() { + debug!("found ffmpeg in directory: {:?}", ffmpeg_in_installation); + return Some(ffmpeg_in_installation); + } + + error!("ffmpeg not found even after installation"); None // Return None if ffmpeg is not found } + +fn handle_ffmpeg_installation() -> Result<(), String> { + if ffmpeg_is_installed() { + debug!("ffmpeg is already installed"); + return Ok(()); + } + + debug!("ffmpeg not found. installing..."); + match check_latest_version() { + Ok(version) => debug!("latest version: {}", version), + Err(e) => debug!("skipping version check due to error: {e}"), + } + + let download_url = ffmpeg_download_url().map_err(|e| e.to_string())?; + let destination = sidecar_dir().map_err(|e| e.to_string())?; + + debug!("downloading from: {:?}", download_url); + let archive_path = + download_ffmpeg_package(download_url, &destination).map_err(|e| e.to_string())?; + debug!("downloaded package: {:?}", archive_path); + + debug!("extracting..."); + unpack_ffmpeg(&archive_path, &destination).map_err(|e| e.to_string())?; + + let version = ffmpeg_version().map_err(|e| e.to_string())?; + + debug!("done! installed ffmpeg version {}", version); + Ok(()) +} diff --git a/screenpipe-server/src/bin/screenpipe-server.rs b/screenpipe-server/src/bin/screenpipe-server.rs index 785fc99d3..c6458f7ec 100644 --- a/screenpipe-server/src/bin/screenpipe-server.rs +++ b/screenpipe-server/src/bin/screenpipe-server.rs @@ -185,6 +185,13 @@ async fn main() -> anyhow::Result<()> { SileroVad::new().await.unwrap(); // Check if FFmpeg is working properly + if let Some(ffmpeg_path) = find_ffmpeg_path() { + println!("ffmpeg found at: {:?}", ffmpeg_path); + } else { + eprintln!("failed to find or install ffmpeg."); + return Err(anyhow::anyhow!("ffmpeg installation failed")); + } + match check_ffmpeg().await { Ok(_) => info!("FFmpeg is working properly"), Err(e) => { @@ -196,7 +203,6 @@ async fn main() -> anyhow::Result<()> { } info!("screenpipe setup complete"); - // TODO: ffmpeg sidecar thing here return Ok(()); } } diff --git a/screenpipe-server/src/video.rs b/screenpipe-server/src/video.rs index 2d69fc4ca..68e1a6c9d 100644 --- a/screenpipe-server/src/video.rs +++ b/screenpipe-server/src/video.rs @@ -168,21 +168,7 @@ pub async fn start_ffmpeg_process(output_file: &str, fps: f64) -> Result