From 530cb3d6e1632a86b7d5ed47b650725324d21abb Mon Sep 17 00:00:00 2001 From: Louis Beaumont Date: Fri, 2 Aug 2024 16:19:25 +0200 Subject: [PATCH] feat: add app_name tracking + filtering --- .github/workflows/release-app.yml | 6 +- screenpipe-server/src/core.rs | 3 +- screenpipe-server/src/db.rs | 378 +++++++++--------- .../migrations/20240716120004_app_name.sql | 4 + screenpipe-server/src/server.rs | 26 +- screenpipe-vision/src/core.rs | 76 ++-- 6 files changed, 264 insertions(+), 229 deletions(-) create mode 100644 screenpipe-server/src/migrations/20240716120004_app_name.sql diff --git a/.github/workflows/release-app.yml b/.github/workflows/release-app.yml index b17a0bb1a..d6e8bbc60 100644 --- a/.github/workflows/release-app.yml +++ b/.github/workflows/release-app.yml @@ -7,9 +7,9 @@ name: Release App on: - push: - tags: - - "v*" + # push: + # tags: + # - "v*" workflow_dispatch: jobs: diff --git a/screenpipe-server/src/core.rs b/screenpipe-server/src/core.rs index b38b6e1cb..a30d727eb 100644 --- a/screenpipe-server/src/core.rs +++ b/screenpipe-server/src/core.rs @@ -127,7 +127,7 @@ async fn record_video( while is_running.load(Ordering::SeqCst) { if let Some(frame) = video_capture.ocr_frame_queue.lock().await.pop_front() { - match db.insert_frame().await { + match db.insert_frame(&frame.app_name).await { Ok(frame_id) => { let text_json = serde_json::to_string(&frame.text_json).unwrap_or_default(); let new_text_json_vs_previous_frame = @@ -144,6 +144,7 @@ async fn record_video( &text_json, &new_text_json_vs_previous_frame, &raw_data_output_from_ocr, + &frame.app_name ) .await { diff --git a/screenpipe-server/src/db.rs b/screenpipe-server/src/db.rs index 19b1cf345..9dd4482e1 100644 --- a/screenpipe-server/src/db.rs +++ b/screenpipe-server/src/db.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Utc}; +use log::{debug, error, info, warn}; use serde::{Deserialize, Serialize}; use sqlx::migrate::MigrateDatabase; use sqlx::{ @@ -6,7 +7,6 @@ use sqlx::{ FromRow, }; use std::time::Duration; -use log::{debug, error, info, warn}; use tokio::time::{timeout, Duration as TokioDuration}; #[derive(Debug, Serialize)] @@ -19,12 +19,13 @@ pub enum SearchResult { pub struct OCRResult { pub frame_id: i64, pub ocr_text: String, - pub text_json: String, // Store as JSON string + pub text_json: String, // Store as JSON string pub new_text_json_vs_previous_frame: String, // Store as JSON string - pub raw_data_output_from_ocr: String, // Store as JSON string + pub raw_data_output_from_ocr: String, // Store as JSON string pub timestamp: DateTime, pub file_path: String, pub offset_index: i64, + pub app_name: String, } #[derive(Debug, Deserialize, PartialEq, Default, Clone, Copy)] @@ -51,8 +52,10 @@ pub struct DatabaseManager { impl DatabaseManager { pub async fn new(database_path: &str) -> Result { - - debug!("Initializing DatabaseManager with database path: {}", database_path); + debug!( + "Initializing DatabaseManager with database path: {}", + database_path + ); let connection_string = format!("sqlite:{}", database_path); // Create the database if it doesn't exist @@ -62,7 +65,7 @@ impl DatabaseManager { let pool = SqlitePoolOptions::new() .max_connections(10) - .min_connections(3) // Minimum number of idle connections + .min_connections(3) // Minimum number of idle connections .acquire_timeout(Duration::from_secs(10)) .connect(&connection_string) .await?; @@ -127,7 +130,7 @@ impl DatabaseManager { Ok(id) } - pub async fn insert_frame(&self) -> Result { + pub async fn insert_frame(&self, app_name: &str) -> Result { // debug!("Starting insert_frame"); let mut tx = self.pool.begin().await?; @@ -161,11 +164,12 @@ impl DatabaseManager { // Insert the new frame let id = sqlx::query( - "INSERT INTO frames (video_chunk_id, offset_index, timestamp) VALUES (?1, ?2, ?3)", + "INSERT INTO frames (video_chunk_id, offset_index, timestamp, app_name) VALUES (?1, ?2, ?3, ?4)", ) .bind(video_chunk_id) .bind(offset_index) .bind(Utc::now()) + .bind(app_name) .execute(&mut *tx) .await? .last_insert_rowid(); @@ -185,43 +189,65 @@ impl DatabaseManager { text_json: &str, new_text_json_vs_previous_frame: &str, raw_data_output_from_ocr: &str, + app_name: &str, ) -> Result<(), sqlx::Error> { const MAX_RETRIES: u32 = 3; const TIMEOUT_DURATION: TokioDuration = TokioDuration::from_secs(10); - // debug!("Starting insert_ocr_text for frame_id: {}", frame_id); - for attempt in 1..=MAX_RETRIES { // debug!("Attempt {} for frame_id: {}", attempt, frame_id); - match timeout(TIMEOUT_DURATION, self.insert_ocr_text_old( - frame_id, - text, - text_json, - new_text_json_vs_previous_frame, - raw_data_output_from_ocr, - )).await { + match timeout( + TIMEOUT_DURATION, + self.insert_ocr_text_old( + frame_id, + text, + text_json, + new_text_json_vs_previous_frame, + raw_data_output_from_ocr, + app_name, + ), + ) + .await + { Ok(Ok(())) => { // Log successful insertion - debug!("Successfully inserted OCR text for frame_id: {} on attempt {}", frame_id, attempt); + debug!( + "Successfully inserted OCR text for frame_id: {} on attempt {}", + frame_id, attempt + ); return Ok(()); } Ok(Err(e)) => { error!("Failed to insert OCR text on attempt {}: {}", attempt, e); } Err(_) => { - warn!("Timeout occurred on attempt {} while inserting OCR text for frame_id: {}", attempt, frame_id); + warn!( + "Timeout occurred on attempt {} while inserting OCR text for frame_id: {}", + attempt, frame_id + ); } } if attempt < MAX_RETRIES { - warn!("Retrying to insert OCR text for frame_id: {} (attempt {}/{})", frame_id, attempt + 1, MAX_RETRIES); + warn!( + "Retrying to insert OCR text for frame_id: {} (attempt {}/{})", + frame_id, + attempt + 1, + MAX_RETRIES + ); } else { - error!("Failed to insert OCR text for frame_id: {} after {} attempts", frame_id, MAX_RETRIES); + error!( + "Failed to insert OCR text for frame_id: {} after {} attempts", + frame_id, MAX_RETRIES + ); return Err(sqlx::Error::PoolTimedOut); // Return error after max retries } } - debug!("Exiting insert_ocr_text for frame_id: {} with PoolTimedOut error", frame_id); + debug!( + "Exiting insert_ocr_text for frame_id: {} with PoolTimedOut error", + frame_id + ); Err(sqlx::Error::PoolTimedOut) } @@ -232,6 +258,7 @@ impl DatabaseManager { text_json: &str, new_text_json_vs_previous_frame: &str, raw_data_output_from_ocr: &str, + app_name: &str, ) -> Result<(), sqlx::Error> { // debug!("Starting insert_ocr_text_old for frame_id: {}", frame_id); @@ -240,24 +267,25 @@ impl DatabaseManager { "Inserting OCR text with frame_id: {}, text: {}{}", frame_id, text.chars().take(100).collect::(), - if text.chars().count() > 100 { "..." } else { "" } + if text.chars().count() > 100 { + "..." + } else { + "" + } ); let mut tx = self.pool.begin().await?; - sqlx::query("INSERT INTO ocr_text (frame_id, text, text_json, new_text_json_vs_previous_frame, raw_data_output_from_OCR) VALUES (?1, ?2, ?3, ?4, ?5)") + sqlx::query("INSERT INTO ocr_text (frame_id, text, text_json, new_text_json_vs_previous_frame, raw_data_output_from_OCR, app_name) VALUES (?1, ?2, ?3, ?4, ?5, ?6)") .bind(frame_id) .bind(text) .bind(text_json) .bind(new_text_json_vs_previous_frame) .bind(raw_data_output_from_ocr) + .bind(app_name) .execute(&mut *tx) .await?; - - // Log successful insertion - // debug!("Successfully inserted OCR text for frame_id: {}", frame_id); - + tx.commit().await?; - // debug!("Transaction committed for frame_id: {}", frame_id); Ok(()) } @@ -269,17 +297,31 @@ impl DatabaseManager { offset: u32, start_time: Option>, end_time: Option>, + app_name: Option<&str>, ) -> Result, sqlx::Error> { let mut results = Vec::new(); - if content_type == ContentType::All || content_type == ContentType::OCR { - let ocr_results = self.search_ocr(query, limit, offset, start_time, end_time).await?; + // If app_name is specified, only search OCR content + if app_name.is_some() { + let ocr_results = self + .search_ocr(query, limit, offset, start_time, end_time, app_name) + .await?; results.extend(ocr_results.into_iter().map(SearchResult::OCR)); - } + } else { + // If no app_name is specified, proceed with normal search + if content_type == ContentType::All || content_type == ContentType::OCR { + let ocr_results = self + .search_ocr(query, limit, offset, start_time, end_time, None) + .await?; + results.extend(ocr_results.into_iter().map(SearchResult::OCR)); + } - if content_type == ContentType::All || content_type == ContentType::Audio { - let audio_results = self.search_audio(query, limit, offset, start_time, end_time).await?; - results.extend(audio_results.into_iter().map(SearchResult::Audio)); + if content_type == ContentType::All || content_type == ContentType::Audio { + let audio_results = self + .search_audio(query, limit, offset, start_time, end_time) + .await?; + results.extend(audio_results.into_iter().map(SearchResult::Audio)); + } } // Sort results by timestamp in descending order @@ -301,6 +343,7 @@ impl DatabaseManager { Ok(results) } + // Update the search_ocr method async fn search_ocr( &self, query: &str, @@ -308,9 +351,9 @@ impl DatabaseManager { offset: u32, start_time: Option>, end_time: Option>, + app_name: Option<&str>, // Add this parameter ) -> Result, sqlx::Error> { - sqlx::query_as::<_, OCRResult>( - r#" + let mut sql = r#" SELECT ocr_text.frame_id, ocr_text.text as ocr_text, @@ -319,7 +362,8 @@ impl DatabaseManager { ocr_text.raw_data_output_from_OCR, frames.timestamp, video_chunks.file_path, - frames.offset_index + frames.offset_index, + frames.app_name FROM ocr_text JOIN @@ -330,18 +374,33 @@ impl DatabaseManager { ocr_text.text LIKE '%' || ?1 || '%' AND (?2 IS NULL OR frames.timestamp >= ?2) AND (?3 IS NULL OR frames.timestamp <= ?3) + "# + .to_string(); + + if app_name.is_some() { + sql.push_str(" AND frames.app_name = ?6"); + } + + sql.push_str( + r#" ORDER BY frames.timestamp DESC LIMIT ?4 OFFSET ?5 "#, - ) - .bind(query) - .bind(start_time) - .bind(end_time) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await + ); + + let mut query = sqlx::query_as::<_, OCRResult>(&sql) + .bind(query) + .bind(start_time) + .bind(end_time) + .bind(limit) + .bind(offset); + + if let Some(app_name) = app_name { + query = query.bind(app_name); + } + + query.fetch_all(&self.pool).await } async fn search_audio( @@ -401,142 +460,38 @@ impl DatabaseManager { .await } - pub async fn get_recent_results( - &self, - limit: u32, - offset: u32, - start_timestamp: Option>, - end_timestamp: Option>, - ) -> Result, sqlx::Error> { - let mut results = Vec::new(); - - let ocr_query = r#" - SELECT - ocr_text.frame_id, - ocr_text.text as ocr_text, - ocr_text.text_json, - ocr_text.new_text_json_vs_previous_frame, - ocr_text.raw_data_output_from_OCR, - frames.timestamp, - video_chunks.file_path, - frames.offset_index - FROM - ocr_text - JOIN - frames ON ocr_text.frame_id = frames.id - JOIN - video_chunks ON frames.video_chunk_id = video_chunks.id - WHERE - 1=1 - AND (?1 IS NULL OR frames.timestamp >= ?1) - AND (?2 IS NULL OR frames.timestamp <= ?2) - ORDER BY - frames.timestamp DESC - LIMIT ?3 OFFSET ?4 - "#; - - let ocr_results = sqlx::query_as::<_, OCRResult>(ocr_query) - .bind(start_timestamp) - .bind(end_timestamp) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; - - results.extend(ocr_results.into_iter().map(SearchResult::OCR)); - - let audio_query = r#" - SELECT - audio_transcriptions.audio_chunk_id, - audio_transcriptions.transcription, - audio_transcriptions.timestamp, - audio_chunks.file_path, - audio_transcriptions.offset_index - FROM - audio_transcriptions - JOIN - audio_chunks ON audio_transcriptions.audio_chunk_id = audio_chunks.id - WHERE - 1=1 - AND (?1 IS NULL OR audio_transcriptions.timestamp >= ?1) - AND (?2 IS NULL OR audio_transcriptions.timestamp <= ?2) - ORDER BY - audio_transcriptions.timestamp DESC - LIMIT ?3 OFFSET ?4 - "#; - - let audio_results = sqlx::query_as::<_, AudioResult>(audio_query) - .bind(start_timestamp) - .bind(end_timestamp) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; - - results.extend(audio_results.into_iter().map(SearchResult::Audio)); - - // Sort combined results by timestamp - results.sort_by(|a, b| { - let timestamp_a = match a { - SearchResult::OCR(ocr) => ocr.timestamp, - SearchResult::Audio(audio) => audio.timestamp, - }; - let timestamp_b = match b { - SearchResult::OCR(ocr) => ocr.timestamp, - SearchResult::Audio(audio) => audio.timestamp, - }; - timestamp_b.cmp(×tamp_a) - }); - - // Limit the final combined results - results.truncate(limit as usize); - - Ok(results) - } + // Update the count_search_results method pub async fn count_search_results( &self, query: &str, content_type: ContentType, start_time: Option>, end_time: Option>, + app_name: Option<&str>, ) -> Result { let mut total_count = 0; - if content_type == ContentType::All || content_type == ContentType::OCR { - let ocr_count: (i64,) = sqlx::query_as( - r#" - SELECT COUNT(*) - FROM ocr_text - JOIN frames ON ocr_text.frame_id = frames.id - WHERE text LIKE '%' || ?1 || '%' - AND (?2 IS NULL OR frames.timestamp >= ?2) - AND (?3 IS NULL OR frames.timestamp <= ?3) - "#, - ) - .bind(query) - .bind(start_time) - .bind(end_time) - .fetch_one(&self.pool) - .await?; - total_count += ocr_count.0 as usize; - } + // If app_name is specified, only count OCR results + if app_name.is_some() { + let ocr_count = self + .count_ocr_results(query, start_time, end_time, app_name) + .await?; + total_count += ocr_count; + } else { + // If no app_name is specified, proceed with normal counting + if content_type == ContentType::All || content_type == ContentType::OCR { + let ocr_count = self + .count_ocr_results(query, start_time, end_time, None) + .await?; + total_count += ocr_count; + } - if content_type == ContentType::All || content_type == ContentType::Audio { - let audio_count: (i64,) = sqlx::query_as( - r#" - SELECT COUNT(*) - FROM audio_transcriptions - WHERE transcription LIKE '%' || ?1 || '%' - AND (?2 IS NULL OR timestamp >= ?2) - AND (?3 IS NULL OR timestamp <= ?3) - "#, - ) - .bind(query) - .bind(start_time) - .bind(end_time) - .fetch_one(&self.pool) - .await?; - total_count += audio_count.0 as usize; + if content_type == ContentType::All || content_type == ContentType::Audio { + let audio_count = self + .count_audio_results(query, start_time, end_time) + .await?; + total_count += audio_count; + } } Ok(total_count) @@ -583,24 +538,76 @@ impl DatabaseManager { Ok(total_count) } + async fn count_ocr_results( + &self, + query: &str, + start_time: Option>, + end_time: Option>, + app_name: Option<&str>, + ) -> Result { + let mut sql = r#" + SELECT COUNT(*) + FROM ocr_text + JOIN frames ON ocr_text.frame_id = frames.id + WHERE text LIKE '%' || ?1 || '%' + AND (?2 IS NULL OR frames.timestamp >= ?2) + AND (?3 IS NULL OR frames.timestamp <= ?3) + "# + .to_string(); - pub async fn get_latest_timestamps(&self) -> Result<(Option>, Option>), sqlx::Error> { - let latest_frame: Option<(DateTime,)> = sqlx::query_as( - "SELECT timestamp FROM frames ORDER BY timestamp DESC LIMIT 1" - ) - .fetch_optional(&self.pool) - .await?; + if app_name.is_some() { + sql.push_str(" AND frames.app_name = ?4"); + } + + let mut query = sqlx::query_as::<_, (i64,)>(&sql) + .bind(query) + .bind(start_time) + .bind(end_time); - let latest_audio: Option<(DateTime,)> = sqlx::query_as( - "SELECT timestamp FROM audio_chunks ORDER BY timestamp DESC LIMIT 1" + if let Some(app_name) = app_name { + query = query.bind(app_name); + } + + let (count,) = query.fetch_one(&self.pool).await?; + Ok(count as usize) + } + async fn count_audio_results( + &self, + query: &str, + start_time: Option>, + end_time: Option>, + ) -> Result { + let (count,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(*) + FROM audio_transcriptions + WHERE transcription LIKE '%' || ?1 || '%' + AND (?2 IS NULL OR timestamp >= ?2) + AND (?3 IS NULL OR timestamp <= ?3) + "#, ) - .fetch_optional(&self.pool) + .bind(query) + .bind(start_time) + .bind(end_time) + .fetch_one(&self.pool) .await?; - Ok(( - latest_frame.map(|f| f.0), - latest_audio.map(|a| a.0) - )) + Ok(count as usize) + } + pub async fn get_latest_timestamps( + &self, + ) -> Result<(Option>, Option>), sqlx::Error> { + let latest_frame: Option<(DateTime,)> = + sqlx::query_as("SELECT timestamp FROM frames ORDER BY timestamp DESC LIMIT 1") + .fetch_optional(&self.pool) + .await?; + + let latest_audio: Option<(DateTime,)> = + sqlx::query_as("SELECT timestamp FROM audio_chunks ORDER BY timestamp DESC LIMIT 1") + .fetch_optional(&self.pool) + .await?; + + Ok((latest_frame.map(|f| f.0), latest_audio.map(|a| a.0))) } } @@ -611,4 +618,3 @@ impl Clone for DatabaseManager { } } } - diff --git a/screenpipe-server/src/migrations/20240716120004_app_name.sql b/screenpipe-server/src/migrations/20240716120004_app_name.sql new file mode 100644 index 000000000..ca8d3c88b --- /dev/null +++ b/screenpipe-server/src/migrations/20240716120004_app_name.sql @@ -0,0 +1,4 @@ +-- Add migration script here +-- Add app_name column to frames table +ALTER TABLE frames ADD COLUMN app_name TEXT NOT NULL DEFAULT ''; +ALTER TABLE ocr_text ADD COLUMN app_name TEXT NOT NULL DEFAULT ''; \ No newline at end of file diff --git a/screenpipe-server/src/server.rs b/screenpipe-server/src/server.rs index a00ecc677..6229e39e2 100644 --- a/screenpipe-server/src/server.rs +++ b/screenpipe-server/src/server.rs @@ -46,7 +46,7 @@ pub(crate) struct DeviceRequest { device_id: String, } -// Request structs +// Update the SearchQuery struct #[derive(Deserialize)] pub(crate) struct SearchQuery { q: Option, @@ -58,6 +58,8 @@ pub(crate) struct SearchQuery { start_time: Option>, #[serde(default)] end_time: Option>, + #[serde(default)] + app_name: Option, // Add this line } #[derive(Deserialize)] @@ -117,6 +119,7 @@ pub(crate) struct OCRContent { timestamp: DateTime, file_path: String, offset_index: i64, + app_name: String, // Add this line } #[derive(Serialize)] @@ -163,25 +166,35 @@ pub(crate) async fn search( (StatusCode, JsonResponse), > { info!( - "Received search request: query='{}', content_type={:?}, limit={}, offset={}, start_time={:?}, end_time={:?}", + "Received search request: query='{}', content_type={:?}, limit={}, offset={}, start_time={:?}, end_time={:?}, app_name={:?}", query.q.as_deref().unwrap_or(""), query.content_type, query.pagination.limit, query.pagination.offset, query.start_time, - query.end_time + query.end_time, + query.app_name ); let query_str = query.q.as_deref().unwrap_or(""); + + // If app_name is specified, force content_type to OCR + let content_type = if query.app_name.is_some() { + ContentType::OCR + } else { + query.content_type + }; + let results = state .db .search( query_str, - query.content_type, + content_type, query.pagination.limit, query.pagination.offset, query.start_time, query.end_time, + query.app_name.as_deref(), ) .await .map_err(|e| { @@ -196,9 +209,10 @@ pub(crate) async fn search( .db .count_search_results( query_str, - query.content_type, + content_type, query.start_time, query.end_time, + query.app_name.as_deref(), ) .await .map_err(|e| { @@ -434,6 +448,7 @@ fn into_content_item(result: SearchResult) -> ContentItem { timestamp: ocr.timestamp, file_path: ocr.file_path, offset_index: ocr.offset_index, + app_name: ocr.app_name, // Add this line }), SearchResult::Audio(audio) => ContentItem::Audio(AudioContent { chunk_id: audio.audio_chunk_id, @@ -631,5 +646,6 @@ curl "http://localhost:3030/search?limit=5&offset=0&content_type=ocr&start_time= # Search for audio content with a keyword from the beginning of the current month curl "http://localhost:3030/search?q=libmp3&limit=5&offset=0&content_type=audio&start_time=$(date -u -v1d -v0H -v0M -v0S +%Y-%m-01T%H:%M:%SZ)" | jq +curl "http://localhost:3030/search?app_name=cursor" */ diff --git a/screenpipe-vision/src/core.rs b/screenpipe-vision/src/core.rs index ba11961c1..d36baccfc 100644 --- a/screenpipe-vision/src/core.rs +++ b/screenpipe-vision/src/core.rs @@ -12,10 +12,13 @@ use tokio::sync::{ mpsc::{Receiver, Sender}, Mutex, }; // Corrected import for Mutex -use xcap::Monitor; +use xcap::{Monitor, Window}; -use crate::utils::{capture_screenshot, compare_with_previous_image, perform_ocr, perform_ocr_cloud, save_text_files}; -use rusty_tesseract::{DataOutput, Data}; // Add this import +use crate::utils::{ + capture_screenshot, compare_with_previous_image, perform_ocr, perform_ocr_cloud, + save_text_files, +}; +use rusty_tesseract::{Data, DataOutput}; // Add this import pub enum ControlMessage { Pause, @@ -47,10 +50,11 @@ pub struct CaptureResult { pub image: Arc, pub text: String, pub text_json: Vec>, - pub new_text_json: Vec>, + pub new_text_json: Vec>, pub frame_number: u64, pub timestamp: Instant, pub data_output: DataOutput, + pub app_name: String, } impl Clone for CaptureResult { @@ -64,21 +68,27 @@ impl Clone for CaptureResult { timestamp: self.timestamp, data_output: DataOutput { output: self.data_output.output.clone(), - data: self.data_output.data.iter().map(|d| Data { - level: d.level, - page_num: d.page_num, - block_num: d.block_num, - par_num: d.par_num, - line_num: d.line_num, - word_num: d.word_num, - left: d.left, - top: d.top, - width: d.width, - height: d.height, - conf: d.conf, - text: d.text.clone(), - }).collect(), + data: self + .data_output + .data + .iter() + .map(|d| Data { + level: d.level, + page_num: d.page_num, + block_num: d.block_num, + par_num: d.par_num, + line_num: d.line_num, + word_num: d.word_num, + left: d.left, + top: d.top, + width: d.width, + height: d.height, + conf: d.conf, + text: d.text.clone(), + }) + .collect(), }, + app_name: self.app_name.clone(), } } } @@ -98,6 +108,7 @@ pub async fn continuous_capture( cloud_ocr: bool, // Add this parameter ) { let monitor = Monitor::all().unwrap().first().unwrap().clone(); // Simplified monitor retrieval + debug!("continuous_capture: Starting using monitor: {:?}", monitor); let previous_text_json = Arc::new(Mutex::new(None)); let ocr_task_running = Arc::new(AtomicBool::new(false)); @@ -149,6 +160,8 @@ pub async fn continuous_capture( ocr_task_running.store(true, Ordering::SeqCst); // debug!("ocr_task_running {}", ocr_task_running.load(Ordering::SeqCst)); tokio::spawn(async move { + let w = Window::all().unwrap().first().unwrap().clone(); + let app_name = w.app_name(); if let Err(e) = process_ocr_task( ocr_task_data.image, ocr_task_data.frame_number, @@ -156,33 +169,23 @@ pub async fn continuous_capture( ocr_task_data.result_tx, &previous_text_json_clone, save_text_files_flag, // Pass the flag here - cloud_ocr, // Pass the cloud_ocr flag here + cloud_ocr, // Pass the cloud_ocr flag here + app_name.to_string().to_lowercase(), ) .await { error!("Error processing OCR task: {}", e); } ocr_task_running_clone.store(false, Ordering::SeqCst); - // debug!("ocr_task_running_clone {}", ocr_task_running_clone.load(Ordering::SeqCst)); }); // Reset max_average and max_avg_value after spawning the OCR task max_avg_value = 0.0; - // debug!("max_avg_value {}", max_avg_value); } } frame_counter += 1; - // debug!("frame_counter triggered to {}, interval is {:?}", frame_counter, interval); - // let total_duration = start_time.elapsed(); - // info!( - // "Capture completed. Total frames: {}, Total time: {:.1?}, Avg FPS: {:.2}", - // frame_counter, - // total_duration, - // frame_counter as f64 / total_duration.as_secs_f64() - // ); tokio::time::sleep(interval).await; - // debug!("paseed tokio::time::sleep"); } } @@ -202,12 +205,16 @@ async fn process_ocr_task( result_tx: Sender, previous_text_json: &Arc>>>>, save_text_files_flag: bool, // Add this parameter - cloud_ocr: bool, // Add this parameter + cloud_ocr: bool, // Add this parameter + app_name: String, ) -> Result<(), std::io::Error> { let start_time = Instant::now(); - // not to confuse with frame id which is wholly different thing - debug!("Performing OCR for frame number since beginning of program {}", frame_number); + // not to confuse with frame id which is wholly different thing + debug!( + "Performing OCR for frame number since beginning of program {}", + frame_number + ); let (text, data_output, json_output) = if cloud_ocr { debug!("Cloud Unstructured.io OCR"); perform_ocr_cloud(&image_arc).await @@ -266,10 +273,11 @@ async fn process_ocr_task( image: image_arc.into(), text: text.clone(), text_json: current_text_json, - new_text_json, + new_text_json, frame_number, timestamp, data_output, + app_name, }) .await {