From 52ecbc3fbbf02a6b6e8a775eef29876d71dcb217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Humberto=20Yusta=20G=C3=B3mez?= Date: Tue, 3 Dec 2024 10:13:20 +0100 Subject: [PATCH 1/3] feat: add smartlinks for each container --- src/integrations/integr_postgres.rs | 8 +++++++- src/integrations/yaml_schema.rs | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/integrations/integr_postgres.rs b/src/integrations/integr_postgres.rs index 6eebcc524..409a2c04f 100644 --- a/src/integrations/integr_postgres.rs +++ b/src/integrations/integr_postgres.rs @@ -233,7 +233,13 @@ docker: sl_chat: - role: "user" content: | - 🔧 Your job is to create a new section under "docker" that will define a new postgres container, inside the current config file %CURRENT_CONFIG%. Follow the system prompt. + 🔧 Your job is to create a postgres container, using the image and environment from new_container_default section in the current config file: %CURRENT_CONFIG%. Follow the system prompt. + smartlinks_for_each_container: + - sl_label: "Use for integration" + sl_chat: + - role: "user" + content: | + 🔧 Your job is to modify postgres connection config in the current file to match the variables from the container, use docker tool to inspect the container if needed. Current config file: %CURRENT_CONFIG%. "#; diff --git a/src/integrations/yaml_schema.rs b/src/integrations/yaml_schema.rs index 23d4fd6a0..0e439c1dd 100644 --- a/src/integrations/yaml_schema.rs +++ b/src/integrations/yaml_schema.rs @@ -48,6 +48,7 @@ pub struct ISchemaDocker { pub filter_image: String, pub new_container_default: DockerService, pub smartlinks: Vec, + pub smartlinks_for_each_container: Vec, } #[derive(Serialize, Deserialize, Debug, Default)] From 1bf4b2f927aa49416e957627a9bf12f5420a6afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Humberto=20Yusta=20G=C3=B3mez?= Date: Tue, 3 Dec 2024 10:34:34 +0100 Subject: [PATCH 2/3] fix: serialize list of ports as comma separated published:target notation --- .../docker/docker_container_manager.rs | 15 -------------- src/integrations/docker/integr_docker.rs | 20 +++++++++++++++++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/integrations/docker/docker_container_manager.rs b/src/integrations/docker/docker_container_manager.rs index 2b1ece356..5559c5c88 100644 --- a/src/integrations/docker/docker_container_manager.rs +++ b/src/integrations/docker/docker_container_manager.rs @@ -31,21 +31,6 @@ pub struct Port { pub target: String, } -impl<'de> Deserialize<'de> for Port { - fn deserialize>(deserializer: D) -> Result { - let s = String::deserialize(deserializer)?; - let (published, target) = s.split_once(':') - .ok_or_else(|| serde::de::Error::custom("expected format '8080:3000'"))?; - Ok(Port { published: published.to_string(), target: target.to_string() }) - } -} - -impl Serialize for Port { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(&format!("{}:{}", self.published, self.target)) - } -} - pub struct DockerContainerSession { container_id: String, connection: DockerContainerConnectionEnum, diff --git a/src/integrations/docker/integr_docker.rs b/src/integrations/docker/integr_docker.rs index f6fcc30bf..80d30665d 100644 --- a/src/integrations/docker/integr_docker.rs +++ b/src/integrations/docker/integr_docker.rs @@ -33,6 +33,7 @@ pub struct SettingsDocker { pub command: String, #[serde(serialize_with = "serialize_num_to_str", deserialize_with = "deserialize_str_to_num")] pub keep_containers_alive_for_x_minutes: u64, + #[serde(serialize_with = "serialize_ports", deserialize_with = "deserialize_ports")] pub ports: Vec, } @@ -46,6 +47,21 @@ where String::deserialize(deserializer)?.parse().map_err(serde::de::Error::custom) } +fn serialize_ports(ports: &Vec, serializer: S) -> Result { + let ports_str = ports.iter().map(|port| format!("{}:{}", port.published, port.target)) + .collect::>().join(","); + serializer.serialize_str(&ports_str) +} + +fn deserialize_ports<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let ports_str = String::deserialize(deserializer)?; + ports_str.split(',').filter(|s| !s.is_empty()).map(|port_str| { + let (published, target) = port_str.split_once(':') + .ok_or_else(|| serde::de::Error::custom("expected format 'published:target'"))?; + Ok(Port { published: published.to_string(), target: target.to_string() }) + }).collect() +} + impl SettingsDocker { pub fn get_ssh_config(&self) -> Option { if self.remote_docker { @@ -334,8 +350,8 @@ fields: f_desc: "How long to keep containers alive in minutes." f_default: "60" ports: - f_type: array - f_desc: "Ports to expose." + f_type: string_long + f_desc: "Comma separated published:target notation for ports to publish, example '8080:3000,5000:5432'" available: on_your_laptop_possible: true when_isolated_possible: false From 23b1c256cb72b85ce2e53b6e44efa851808577a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Humberto=20Yusta=20G=C3=B3mez?= Date: Tue, 3 Dec 2024 11:46:29 +0100 Subject: [PATCH 3/3] fix: shorten docker error for integration use and docker handlers --- src/http/routers/v1/docker.rs | 6 ++-- .../docker/docker_container_manager.rs | 30 +++++++++---------- src/integrations/docker/integr_docker.rs | 11 +++++-- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/http/routers/v1/docker.rs b/src/http/routers/v1/docker.rs index 94f66ad2b..6ad626bd7 100644 --- a/src/http/routers/v1/docker.rs +++ b/src/http/routers/v1/docker.rs @@ -64,7 +64,7 @@ pub async fn handle_v1_docker_container_action( DockerAction::Remove => format!("container remove --volumes {}", post.container), DockerAction::Stop => format!("container stop {}", post.container), }; - let (output, _) = docker.command_execute(&docker_command, gcx.clone(), true).await + let (output, _) = docker.command_execute(&docker_command, gcx.clone(), true, false).await .map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("Command {} failed: {}", docker_command, e)))?; Ok(Response::builder().status(StatusCode::OK).body(Body::from( @@ -87,7 +87,7 @@ pub async fn handle_v1_docker_container_list( None => "container list --all --no-trunc --format json".to_string(), }; - let (unparsed_output, _) = docker.command_execute(&docker_command, gcx.clone(), true).await + let (unparsed_output, _) = docker.command_execute(&docker_command, gcx.clone(), true, false).await .map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("Command {} failed: {}", docker_command, e)))?; let mut output: Vec = unparsed_output.lines().map(|line| serde_json::from_str(line)).collect::, _>>() @@ -113,7 +113,7 @@ pub async fn handle_v1_docker_container_list( } let inspect_command = format!("container inspect --format json {}", container_ids.join(" ")); - let (inspect_unparsed_output, _) = docker.command_execute(&inspect_command, gcx.clone(), true).await + let (inspect_unparsed_output, _) = docker.command_execute(&inspect_command, gcx.clone(), true, false).await .map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("Command {} failed: {}", inspect_command, e)))?; let inspect_output = serde_json::from_str::>(&inspect_unparsed_output) diff --git a/src/integrations/docker/docker_container_manager.rs b/src/integrations/docker/docker_container_manager.rs index 5559c5c88..c77208d0b 100644 --- a/src/integrations/docker/docker_container_manager.rs +++ b/src/integrations/docker/docker_container_manager.rs @@ -156,7 +156,7 @@ pub async fn docker_container_check_status_or_start( if !docker.settings_docker.command.is_empty() { let cmd_to_execute = format!("exec --detach {} {}", container_id, docker.settings_docker.command); - match docker.command_execute(&cmd_to_execute, gcx.clone(), false).await { + match docker.command_execute(&cmd_to_execute, gcx.clone(), false, true).await { Ok((cmd_stdout, cmd_stderr)) => { info!("Command executed: {cmd_stdout}\n{cmd_stderr}") }, Err(e) => { error!("Command execution failed: {}", e) }, }; @@ -242,7 +242,7 @@ async fn docker_container_create( ); info!("Executing docker command: {}", &run_command); - let (run_output, _) = docker.command_execute(&run_command, gcx.clone(), true).await?; + let (run_output, _) = docker.command_execute(&run_command, gcx.clone(), true, true).await?; let container_id = run_output.trim(); if container_id.len() < 12 { @@ -267,8 +267,8 @@ async fn docker_container_sync_yaml_configs( let temp_dir = tempfile::Builder::new().tempdir() .map_err(|e| format!("Error creating temporary directory: {}", e))?; let temp_dir_path = temp_dir.path().to_string_lossy().to_string(); - docker.command_execute(&format!("container cp {temp_dir_path} {container_id}:{container_home_dir}/.cache/"), gcx.clone(), true).await?; - docker.command_execute(&format!("container cp {temp_dir_path} {container_id}:{container_home_dir}/.cache/refact"), gcx.clone(), true).await?; + docker.command_execute(&format!("container cp {temp_dir_path} {container_id}:{container_home_dir}/.cache/"), gcx.clone(), true, true).await?; + docker.command_execute(&format!("container cp {temp_dir_path} {container_id}:{container_home_dir}/.cache/refact"), gcx.clone(), true, true).await?; let config_files_to_sync = ["privacy.yaml", "integrations.yaml", "bring-your-own-key.yaml", "competency.yaml"]; let remote_integrations_path = { @@ -282,13 +282,13 @@ async fn docker_container_sync_yaml_configs( _ => cache_dir.join(file).to_string_lossy().to_string(), }; let container_path = format!("{container_id}:{container_home_dir}/.cache/refact/{file}"); - docker.command_execute(&format!("container cp {local_path} {container_path}"), gcx.clone(), true).await?; + docker.command_execute(&format!("container cp {local_path} {container_path}"), gcx.clone(), true, true).await?; } // Copying config folder let config_dir_string = config_dir.to_string_lossy().to_string(); - docker.command_execute(&format!("container cp {temp_dir_path} {container_id}:{container_home_dir}/.config/"), gcx.clone(), true).await?; - docker.command_execute(&format!("container cp {config_dir_string} {container_id}:{container_home_dir}/.config/refact"), gcx.clone(), true).await?; + docker.command_execute(&format!("container cp {temp_dir_path} {container_id}:{container_home_dir}/.config/"), gcx.clone(), true, true).await?; + docker.command_execute(&format!("container cp {config_dir_string} {container_id}:{container_home_dir}/.config/refact"), gcx.clone(), true, true).await?; Ok(()) } @@ -299,7 +299,7 @@ async fn docker_container_get_home_dir( gcx: Arc>, ) -> Result { let inspect_config_command = "container inspect --format '{{json .Config}}' ".to_string() + &container_id; - let (inspect_config_output, _) = docker.command_execute(&inspect_config_command, gcx.clone(), true).await?; + let (inspect_config_output, _) = docker.command_execute(&inspect_config_command, gcx.clone(), true, true).await?; let config_json: serde_json::Value = serde_json::from_str(&inspect_config_output) .map_err(|e| format!("Error parsing docker config: {}", e))?; @@ -319,13 +319,13 @@ async fn docker_container_start( container_id: &str, ) -> Result<(), String> { let start_command = "container start ".to_string() + &container_id; - docker.command_execute(&start_command, gcx.clone(), true).await?; + docker.command_execute(&start_command, gcx.clone(), true, true).await?; // If docker container is not running, print last lines of logs. let inspect_command = "container inspect --format '{{json .State.Running}}' ".to_string() + &container_id; - let (inspect_output, _) = docker.command_execute(&inspect_command, gcx.clone(), true).await?; + let (inspect_output, _) = docker.command_execute(&inspect_command, gcx.clone(), true, true).await?; if inspect_output.trim() != "true" { - let (logs_output, _) = docker.command_execute(&format!("container logs --tail 10 {container_id}"), gcx.clone(), true).await?; + let (logs_output, _) = docker.command_execute(&format!("container logs --tail 10 {container_id}"), gcx.clone(), true, true).await?; return Err(format!("Docker container is not running: \n{logs_output}")); } @@ -372,7 +372,7 @@ async fn docker_container_sync_workspace( tar_builder.finish().await.map_err(|e| format!("Error finishing tar archive: {}", e))?; let cp_command = format!("container cp {} {}:{}", temp_tar_file.to_string_lossy(), container_id, container_workspace_folder.to_string_lossy()); - docker.command_execute(&cp_command, gcx.clone(), true).await?; + docker.command_execute(&cp_command, gcx.clone(), true, true).await?; let sync_files_post = SyncFilesExtractTarPost { tar_path: container_workspace_folder.join(&tar_file_name).to_string_lossy().to_string(), @@ -424,7 +424,7 @@ async fn docker_container_get_exposed_ports( gcx: Arc>, ) -> Result, String> { let inspect_command = "inspect --format '{{json .NetworkSettings.Ports}}' ".to_string() + &container_id; - let (inspect_output, _) = docker.command_execute(&inspect_command, gcx.clone(), true).await?; + let (inspect_output, _) = docker.command_execute(&inspect_command, gcx.clone(), true, true).await?; tracing::info!("{}:\n{}", inspect_command, inspect_output); let inspect_data: serde_json::Value = serde_json::from_str(&inspect_output) @@ -446,9 +446,9 @@ async fn docker_container_kill( ) -> Result<(), String> { let docker = docker_tool_load(gcx.clone()).await?; - docker.command_execute(&format!("container stop {container_id}"), gcx.clone(), true).await?; + docker.command_execute(&format!("container stop {container_id}"), gcx.clone(), true, true).await?; info!("Stopped docker container {container_id}."); - docker.command_execute(&format!("container remove {container_id}"), gcx.clone(), true).await?; + docker.command_execute(&format!("container remove {container_id}"), gcx.clone(), true, true).await?; info!("Removed docker container {container_id}."); Ok(()) } \ No newline at end of file diff --git a/src/integrations/docker/integr_docker.rs b/src/integrations/docker/integr_docker.rs index 80d30665d..42b6ed980 100644 --- a/src/integrations/docker/integr_docker.rs +++ b/src/integrations/docker/integr_docker.rs @@ -115,7 +115,7 @@ impl IntegrationTrait for ToolDocker { } impl ToolDocker { - pub async fn command_execute(&self, command: &str, gcx: Arc>, fail_if_stderr_is_not_empty: bool) -> Result<(String, String), String> + pub async fn command_execute(&self, command: &str, gcx: Arc>, fail_if_stderr_is_not_empty: bool, verbose_error: bool) -> Result<(String, String), String> { let mut command_args = split_command(&command)?; @@ -137,7 +137,12 @@ impl ToolDocker { let stderr = String::from_utf8_lossy(&output.stderr).to_string(); if fail_if_stderr_is_not_empty && !stderr.is_empty() { - return Err(format!("Error executing command {command}: \n{stderr}")); + let error_message = if verbose_error { + format!("Command `{}` failed: {}", command, stderr) + } else { + stderr + }; + return Err(error_message); } Ok((stdout, stderr)) @@ -173,7 +178,7 @@ impl Tool for ToolDocker { ccx_locked.global_context.clone() }; - let (stdout, _) = self.command_execute(&command, gcx.clone(), true).await?; + let (stdout, _) = self.command_execute(&command, gcx.clone(), true, false).await?; Ok((false, vec![ ContextEnum::ChatMessage(ChatMessage {