Skip to content

Commit

Permalink
Chrome in docker (#462)
Browse files Browse the repository at this point in the history
* feat: integr chrome works with new config and in docker isolation

* fix: repair chrome tool

---------

Co-authored-by: Oleg Klimov <[email protected]>
  • Loading branch information
humbertoyusta and olegklimov authored Dec 6, 2024
1 parent a5e549a commit 4e483bd
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 88 deletions.
162 changes: 77 additions & 85 deletions src/integrations/integr_chrome.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use std::any::Any;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::collections::HashMap;
use std::future::Future;
use std::time::Duration;
Expand All @@ -15,7 +15,7 @@ use crate::call_validation::{ChatContent, ChatMessage};
use crate::scratchpads::multimodality::MultimodalElement;
use crate::postprocessing::pp_command_output::{CmdlineOutputFilter, output_mini_postprocessing};
use crate::tools::tools_description::{Tool, ToolDesc, ToolParam};
use crate::integrations::integr_abstract::Integration;
use crate::integrations::integr_abstract::IntegrationTrait;

use tokio::time::sleep;
use chrono::DateTime;
Expand All @@ -35,7 +35,7 @@ use image::{ImageFormat, ImageReader};


#[derive(Clone, Serialize, Deserialize, Debug, Default)]
pub struct IntegrationChrome {
pub struct SettingsChrome {
pub chrome_path: Option<String>,
pub window_width: Option<u32>,
pub window_height: Option<u32>,
Expand All @@ -48,7 +48,7 @@ fn default_headless() -> bool { true }

#[derive(Debug, Default)]
pub struct ToolChrome {
pub integration_chrome: IntegrationChrome,
pub settings_chrome: SettingsChrome,
pub supports_clicks: bool,
}

Expand All @@ -58,8 +58,8 @@ enum DeviceType {
MOBILE,
}

impl fmt::Display for DeviceType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
impl std::fmt::Display for DeviceType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
DeviceType::DESKTOP => write!(f, "desktop"),
DeviceType::MOBILE => write!(f, "mobile"),
Expand Down Expand Up @@ -122,35 +122,33 @@ impl IntegrationSession for ChromeSession
}
}

impl Integration for ToolChrome {
impl IntegrationTrait for ToolChrome {
fn integr_settings_apply(&mut self, value: &Value) -> Result<(), String> {
let integration_github = serde_json::from_value::<IntegrationChrome>(value.clone())
.map_err(|e|e.to_string())?;
self.integration_chrome = integration_github;
match serde_json::from_value::<SettingsChrome>(value.clone()) {
Ok(settings_chrome) => self.settings_chrome = settings_chrome,
Err(e) => {
tracing::error!("Failed to apply settings: {}\n{:?}", e, value);
return Err(e.to_string());
}
}
Ok(())
}

fn integr_yaml2json(&self, value: &serde_yaml::Value) -> Result<Value, String> {
let integration_github = serde_yaml::from_value::<IntegrationChrome>(value.clone()).map_err(|e| {
let location = e.location().map(|loc| format!(" at line {}, column {}", loc.line(), loc.column())).unwrap_or_default();
format!("{}{}", e.to_string(), location)
})?;
serde_json::to_value(&integration_github).map_err(|e| e.to_string())
fn integr_settings_as_json(&self) -> Value {
serde_json::to_value(&self.settings_chrome).unwrap()
}

fn integr_upgrade_to_tool(&self, integr_name: &String) -> Box<dyn Tool + Send> {
fn integr_upgrade_to_tool(&self, _integr_name: &String) -> Box<dyn Tool + Send> {
Box::new(ToolChrome {
integration_chrome: self.integration_chrome.clone(),
supports_clicks: false}
) as Box<dyn Tool + Send>
settings_chrome: self.settings_chrome.clone(),
supports_clicks: false,
}) as Box<dyn Tool + Send>
}

fn integr_settings_as_json(&self) -> Result<Value, String> {
serde_json::to_value(&self.integration_chrome).map_err(|e| e.to_string())
fn integr_schema(&self) -> &str
{
CHROME_INTEGRATION_SCHEMA
}

fn integr_settings_default(&self) -> String { DEFAULT_CHROME_INTEGRATION_YAML.to_string() }
fn icon_link(&self) -> String { "https://cdn-icons-png.flaticon.com/512/732/732205.png".to_string() }
}

#[async_trait]
Expand All @@ -175,7 +173,7 @@ impl Tool for ToolChrome {
};

let session_hashmap_key = get_session_hashmap_key("chrome", &chat_id);
let mut tool_log = setup_chrome_session(gcx.clone(), &self.integration_chrome, &session_hashmap_key).await?;
let mut tool_log = setup_chrome_session(gcx.clone(), &self.settings_chrome, &session_hashmap_key).await?;

let command_session = {
let gcx_locked = gcx.read().await;
Expand Down Expand Up @@ -259,7 +257,7 @@ impl Tool for ToolChrome {

async fn setup_chrome_session(
gcx: Arc<ARwLock<GlobalContext>>,
args: &IntegrationChrome,
args: &SettingsChrome,
session_hashmap_key: &String,
) -> Result<Vec<String>, String> {
let mut setup_log = vec![];
Expand Down Expand Up @@ -693,8 +691,8 @@ enum Key {
END,
}

impl fmt::Display for Key {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
impl std::fmt::Display for Key {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Key::ENTER => write!(f, "Enter"),
Key::ESC => write!(f, "Escape"),
Expand Down Expand Up @@ -840,60 +838,54 @@ fn parse_single_command(command: &String) -> Result<Command, String> {
_ => Err(format!("Unknown command: {:?}.", command_name)),
}
}
<<<<<<< HEAD

async fn interact_with_chrome(
gcx: Arc<ARwLock<GlobalContext>>,
chat_id: &String,
integration_chrome: &IntegrationChrome,
command: &Command,
) -> Result<Vec<MultimodalElement>, String> {
let session_hashmap_key = get_session_hashmap_key("chrome", &chat_id);
let setup_log = setup_chrome_session(gcx.clone(), &integration_chrome, &session_hashmap_key).await?;

let command_session = {
let gcx_locked = gcx.read().await;
gcx_locked.integration_sessions.get(&session_hashmap_key)
.ok_or(format!("Error getting chrome session for chat: {}", chat_id))?
.clone()
};
let mut command_session_locked = command_session.lock().await;
let chrome_session = command_session_locked.as_any_mut().downcast_mut::<ChromeSession>().ok_or("Failed to downcast to ChromeSession")?;

let (execute_log, mut multimodal_els) = command.execute(chrome_session).await?;

let tool_log = setup_log.iter().chain(execute_log.iter()).map(|s| s.clone()).collect::<Vec<_>>();
multimodal_els.push(MultimodalElement::new(
"text".to_string(), tool_log.join("\n")
)?);

Ok(multimodal_els)
}

async fn screenshot_jpeg_base64(tab: &Arc<Tab>, capture_beyond_viewport: bool) -> Result<MultimodalElement, String> {
let jpeg_data = tab.call_method(Page::CaptureScreenshot {
format: Some(Page::CaptureScreenshotFormatOption::Jpeg),
clip: None,
quality: Some(75),
from_surface: Some(true),
capture_beyond_viewport: Some(capture_beyond_viewport),
}).map_err(|e| e.to_string())?.data;

MultimodalElement::new("image/jpeg".to_string(), jpeg_data)
}

const DEFAULT_CHROME_INTEGRATION_YAML: &str = r#"
# Chrome integration
# This can be path to your chrome binary. You can install with "npx @puppeteer/browsers install chrome@stable", read
# more here https://developer.chrome.com/blog/chrome-for-testing/?utm_source=Fibery&utm_medium=iframely
#chrome_path: "/Users/me/my_path/chrome/mac_arm-130.0.6723.69/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing"
# Or you can give it ws:// path, read more here https://developer.chrome.com/docs/devtools/remote-debugging/local-server/
# In that case start chrome with --remote-debugging-port
# chrome_path: "ws://127.0.0.1:6006/"
# window_width: 1024
# window_height: 768
# idle_browser_timeout: 600
"#;
=======
>>>>>>> main
const CHROME_INTEGRATION_SCHEMA: &str = r#"
fields:
chrome_path:
f_type: string_long
f_desc: "Path to your chrome binary. You can install with \"npx @puppeteer/browsers install chrome@stable\", read more here https://developer.chrome.com/blog/chrome-for-testing. You can also give it ws:// path, in that case start chrome with --remote-debugging-port, read more here https://developer.chrome.com/docs/devtools/remote-debugging/local-server"
f_placeholder: "/Users/me/my_path/chrome/mac_arm-130.0.6723.69/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing or ws://127.0.0.1:4444/"
window_width:
f_type: string_short
f_desc: "Width of the browser window."
f_default: "1024"
window_height:
f_type: string_short
f_desc: "Height of the browser window."
f_default: "768"
idle_browser_timeout:
f_type: string_short
f_desc: "Idle timeout for the browser in seconds."
f_default: "600"
headless:
f_type: bool
f_desc: "Run Chrome in headless mode."
f_default: "true"
available:
on_your_laptop_possible: true
when_isolated_possible: true
smartlinks:
- sl_label: "Test"
sl_chat:
- role: "user"
content: |
🔧 The chrome tool should be visible now. To test the tool, navigate to a website, take a screenshot, and express satisfaction and relief if it works. If it doesn't work or the tool isn't available, go through the usual plan in the system prompt. The current config file is %CURRENT_CONFIG%.
docker:
filter_label: ""
filter_image: "standalone-chrome"
new_container_default:
image: "selenium/standalone-chrome:latest"
environment: {}
smartlinks:
- sl_label: "Add Chrome Container"
sl_chat:
- role: "user"
content: |
🔧 Your job is to create a chrome container, using the image and environment from new_container_default section in the current config file: %CURRENT_CONFIG%. Follow the system prompt.
smartlinks_for_each_container:
- sl_label: "Use for integration"
sl_chat:
- role: "user"
content: |
🔧 Your job is to modify chrome config in the current file to connect through websockets to the container, use docker tool to inspect the container if needed. Current config file: %CURRENT_CONFIG%.
"#;
6 changes: 3 additions & 3 deletions src/integrations/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pub mod integr_abstract;
// pub mod integr_github;
// pub mod integr_gitlab;
// pub mod integr_pdb;
// pub mod integr_chrome;
pub mod integr_chrome;
pub mod integr_postgres;
pub mod integr_cmdline;
pub mod integr_cmdline_service;
Expand All @@ -35,7 +35,7 @@ pub fn integration_from_name(n: &str) -> Result<Box<dyn IntegrationTrait + Send
// "gitlab" => Ok(Box::new(ToolGitlab { ..Default::default() }) as Box<dyn IntegrationTrait + Send + Sync>),
// "pdb" => Ok(Box::new(ToolPdb { ..Default::default() }) as Box<dyn IntegrationTrait + Send + Sync>),
"postgres" => Ok(Box::new(integr_postgres::ToolPostgres { ..Default::default() }) as Box<dyn IntegrationTrait + Send + Sync>),
// "chrome" => Ok(Box::new(ToolChrome { ..Default::default() }) as Box<dyn IntegrationTrait + Send + Sync>),
"chrome" => Ok(Box::new(integr_chrome::ToolChrome { ..Default::default() }) as Box<dyn IntegrationTrait + Send + Sync>),
"docker" => Ok(Box::new(docker::integr_docker::ToolDocker {..Default::default() }) as Box<dyn IntegrationTrait + Send + Sync>),
cmdline if cmdline.starts_with("cmdline_") => {
// let tool_name = cmdline.strip_prefix("cmdline_").unwrap();
Expand Down Expand Up @@ -68,9 +68,9 @@ pub fn integrations_list() -> Vec<&'static str> {
// "gitlab",
// "pdb",
"postgres",
"chrome",
"cmdline_TEMPLATE",
"service_TEMPLATE",
// "chrome",
"docker"
]
}
Expand Down

0 comments on commit 4e483bd

Please sign in to comment.