Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(WIP/notgonnamerge/problemsolved) xcap -> scap #220

Closed
wants to merge 1 commit into from
Closed

(WIP/notgonnamerge/problemsolved) xcap -> scap #220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions screenpipe-server/src/bin/screenpipe-server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,9 @@ async fn main() -> anyhow::Result<()> {
}
}

// ! DISGUSTING HACK TEMPORARY FOR MACOS sc
std::env::set_var("SCREENPIPE_FPS", cli.fps.to_string());

let (restart_sender, mut restart_receiver) = channel(10);
let resource_monitor =
ResourceMonitor::new(cli.self_healing, Duration::from_secs(60), 3, restart_sender);
Expand Down
1 change: 1 addition & 0 deletions screenpipe-vision/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ serde_json = "1.0"

# Cross-platform screen capture
xcap = "0.0.12" # This is the latest version as of now
scap = { git = "https://github.com/mediar-ai/scap.git", branch = "main" }

# threadpool
threadpool = "1.8.1"
Expand Down
338 changes: 202 additions & 136 deletions screenpipe-vision/src/capture_screenshot_by_window.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,162 +1,228 @@
use std::sync::Arc;
use image::DynamicImage;
// use image::{DynamicImage, ImageFormat};
use xcap::{Monitor, Window, XCapError};
use log::error;
// use log::{debug, error, info, warn};
use std::time::Duration;
use tokio::time;
use std::error::Error;
use std::fmt;
// use std::fs;
// use std::path::Path;

#[derive(Debug)]
enum CaptureError {
NoWindows,
XCapError(XCapError),
}
#[cfg(not(target_os = "macos"))]
mod non_macos {
use image::DynamicImage;
use log::error;
use std::error::Error;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use tokio::time;
use xcap::{Monitor, Window, XCapError};

#[derive(Debug)]
enum CaptureError {
NoWindows,
XCapError(XCapError),
}

impl fmt::Display for CaptureError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
CaptureError::NoWindows => write!(f, "No windows found"),
CaptureError::XCapError(e) => write!(f, "XCap error: {}", e),
impl fmt::Display for CaptureError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
CaptureError::NoWindows => write!(f, "No windows found"),
CaptureError::XCapError(e) => write!(f, "XCap error: {}", e),
}
}
}
}

impl Error for CaptureError {}
impl Error for CaptureError {}

impl From<XCapError> for CaptureError {
fn from(error: XCapError) -> Self {
error!("XCap error occurred: {}", error);
CaptureError::XCapError(error)
impl From<XCapError> for CaptureError {
fn from(error: XCapError) -> Self {
error!("XCap error occurred: {}", error);
CaptureError::XCapError(error)
}
}
}

pub async fn capture_all_visible_windows() -> Result<Vec<(DynamicImage, String, String, bool)>, Box<dyn Error>> {
let monitors = Monitor::all()?;
// info!("Found {} monitors", monitors.len());
pub async fn capture_all_visible_windows(
) -> Result<Vec<(DynamicImage, String, String, bool)>, Box<dyn Error>> {
let monitors = Monitor::all()?;
let mut all_captured_images = Vec::new();

for monitor in monitors {
let windows = retry_with_backoff(
|| {
let windows = Window::all()?;
if windows.is_empty() {
Err(CaptureError::NoWindows)
} else {
Ok(windows)
}
},
3,
Duration::from_millis(500),
)
.await?;

let focused_window = get_focused_window(Arc::new(monitor.clone())).await;

for window in windows {
if is_valid_window(&window, &monitor) {
let app_name = window.app_name();
let window_name = window.title();
let is_focused = focused_window
.as_ref()
.map_or(false, |fw| fw.id() == window.id());

match window.capture_image() {
Ok(buffer) => {
let image = DynamicImage::ImageRgba8(
image::ImageBuffer::from_raw(
buffer.width() as u32,
buffer.height() as u32,
buffer.into_raw(),
)
.unwrap(),
);

all_captured_images.push((
image,
app_name.to_string(),
window_name.to_string(),
is_focused,
));
}
Err(e) => error!(
"Failed to capture image for window {} on monitor {}: {}",
window_name,
monitor.name(),
e
),
}
}
}
}

let mut all_captured_images = Vec::new();
Ok(all_captured_images)
}

for monitor in monitors {
// debug!("Capturing windows for monitor: {:?}", monitor);
fn is_valid_window(window: &Window, monitor: &Monitor) -> bool {
let monitor_match = window.current_monitor().id() == monitor.id();
let not_minimized = !window.is_minimized();
let not_window_server = window.app_name() != "Window Server";
let not_contexts = window.app_name() != "Contexts";
let has_title = !window.title().is_empty();

// info!("Attempting to get all windows for monitor {} with retry mechanism", monitor.name());
let windows = retry_with_backoff(|| {
let windows = Window::all()?;
if windows.is_empty() {
Err(CaptureError::NoWindows)
} else {
Ok(windows)
}
}, 3, Duration::from_millis(500)).await?;

// let windows_count = windows.len();
// info!("Successfully retrieved {} windows for monitor {}", windows_count, monitor.name());

// if windows_count == 0 {
// warn!("No windows were retrieved for monitor {}. This might indicate an issue.", monitor.name());
// }

let focused_window = get_focused_window(Arc::new(monitor.clone())).await;

// Create 'last_screenshots' directory if it doesn't exist
// let screenshots_dir = Path::new("last_screenshots");
// fs::create_dir_all(screenshots_dir)?;

for (_index, window) in windows.into_iter().enumerate() {
if is_valid_window(&window, &monitor) {
let app_name = window.app_name();
let window_name = window.title();
let is_focused = focused_window.as_ref().map_or(false, |fw| fw.id() == window.id());

match window.capture_image() {
Ok(buffer) => {
let image = DynamicImage::ImageRgba8(image::ImageBuffer::from_raw(
buffer.width() as u32,
buffer.height() as u32,
buffer.into_raw(),
).unwrap());

// Save the image to the 'last_screenshots' directory
// let file_name = format!("monitor_{}_window_{:03}_{}.png",
// sanitize_filename(&monitor.name()), index, sanitize_filename(&window_name));
// let file_path = screenshots_dir.join(file_name);
// image.save_with_format(&file_path, ImageFormat::Png)?;
// info!("Saved screenshot: {:?}", file_path);

all_captured_images.push((image, app_name.to_string(), window_name.to_string(), is_focused));
},
Err(e) => error!("Failed to capture image for window {} on monitor {}: {}", window_name, monitor.name(), e),
monitor_match && not_minimized && not_window_server && not_contexts && has_title
}

async fn retry_with_backoff<F, T, E>(
mut f: F,
max_retries: u32,
initial_delay: Duration,
) -> Result<T, E>
where
F: FnMut() -> Result<T, E>,
E: Error + 'static,
{
let mut delay = initial_delay;
for attempt in 1..=max_retries {
match f() {
Ok(result) => return Ok(result),
Err(e) => {
if attempt == max_retries {
error!("All {} attempts failed. Last error: {}", max_retries, e);
return Err(e);
}
time::sleep(delay).await;
delay *= 2;
}
} else {
// debug!("Skipped invalid window: {} on monitor {}", window.title(), monitor.name());
}
}

// info!("Captured {} valid windows out of {} total windows for monitor {}",
// all_captured_images.len(), windows_count, monitor.name());
unreachable!()
}

Ok(all_captured_images)
async fn get_focused_window(monitor: Arc<Monitor>) -> Option<Window> {
retry_with_backoff(
|| -> Result<Option<Window>, CaptureError> {
let windows = Window::all()?;
Ok(windows.into_iter().find(|w| is_valid_window(w, &monitor)))
},
3,
Duration::from_millis(500),
)
.await
.ok()
.flatten()
}
}

// fn sanitize_filename(name: &str) -> String {
// name.chars()
// .map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' })
// .collect()
// }

fn is_valid_window(window: &Window, monitor: &Monitor) -> bool {
let monitor_match = window.current_monitor().id() == monitor.id();
let not_minimized = !window.is_minimized();
let not_window_server = window.app_name() != "Window Server";
let not_contexts = window.app_name() != "Contexts";
let has_title = !window.title().is_empty();
#[cfg(target_os = "macos")]
mod macos {
use image::{DynamicImage, ImageBuffer};
use scap::{
capturer::{Capturer, Options},
frame::Frame,
Target,
};
use std::error::Error;

pub async fn capture_all_visible_windows(
) -> Result<Vec<(DynamicImage, String, String, bool)>, Box<dyn Error>> {
if !scap::is_supported() || !scap::has_permission() {
return Err("Platform not supported or permission not granted".into());
}

let valid = monitor_match && not_minimized && not_window_server && not_contexts && has_title;
let targets = scap::get_all_targets();

// if !valid {
// debug!("Invalid window on monitor {}: {} (app: {}). Reasons: monitor_match={}, not_minimized={}, not_window_server={}, not_contexts={}, has_title={}",
// monitor.name(), window.title(), window.app_name(), monitor_match, not_minimized, not_window_server, not_contexts, has_title);
// }
// ! DISGUSTING HACK until we stabilise macos screencap
let fps = std::env::var("SCREENPIPE_FPS")
.unwrap_or("1".to_string())
.parse::<u32>()
.unwrap();
let mut captured_windows = Vec::new();

valid
}

async fn retry_with_backoff<F, T, E>(mut f: F, max_retries: u32, initial_delay: Duration) -> Result<T, E>
where
F: FnMut() -> Result<T, E>,
E: Error + 'static,
{
let mut delay = initial_delay;
for attempt in 1..=max_retries {
// info!("Attempt {} to execute function", attempt);
match f() {
Ok(result) => {
// info!("Function executed successfully on attempt {}", attempt);
return Ok(result);
},
Err(e) => {
if attempt == max_retries {
error!("All {} attempts failed. Last error: {}", max_retries, e);
return Err(e);
for target in targets {
if let Target::Window(window) = target {
if !window.is_on_screen {
continue;
}
let options = Options {
fps,
show_cursor: true,
show_highlight: false,
output_type: scap::frame::FrameType::BGRAFrame,
target: Some(Target::Window(window.clone())),
output_resolution: scap::capturer::Resolution::_1080p,
crop_area: None,
..Default::default()
};

let mut capturer = Capturer::new(options);
capturer.start_capture();

if let Ok(frame) = capturer.get_next_frame() {
if let Frame::BGRA(bgra_frame) = frame {
let image = frame_to_dynamic_image(bgra_frame);
captured_windows.push((
image,
window.owning_application.unwrap_or_default(),
window.title,
window.is_active,
));
}
}
// warn!("Attempt {} failed: {}. Retrying in {:?}", attempt, e, delay);
time::sleep(delay).await;
delay *= 2;

capturer.stop_capture();
}
}

Ok(captured_windows)
}

fn frame_to_dynamic_image(frame: scap::frame::BGRAFrame) -> DynamicImage {
let width = frame.width as u32;
let height = frame.height as u32;
let buffer = frame.data;

let image_buffer =
ImageBuffer::from_raw(width, height, buffer).expect("Failed to create image buffer");

DynamicImage::ImageRgba8(image_buffer)
}
unreachable!()
}

async fn get_focused_window(monitor: Arc<Monitor>) -> Option<Window> {
retry_with_backoff(|| -> Result<Option<Window>, CaptureError> {
let windows = Window::all()?;
Ok(windows.into_iter().find(|w| is_valid_window(w, &monitor)))
}, 3, Duration::from_millis(500)).await.ok().flatten()
}
#[cfg(target_os = "macos")]
pub use macos::capture_all_visible_windows;

#[cfg(not(target_os = "macos"))]
pub use non_macos::capture_all_visible_windows;
Loading