From fee299330fd53ea7635b2ca8eb26537635252833 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Sun, 13 Oct 2024 19:33:28 +0300 Subject: [PATCH 1/8] Fix --use-tls option Signed-off-by: Alexander V. Nikolaev --- src/bin/givc-admin.rs | 2 +- src/bin/givc-agent.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/givc-admin.rs b/src/bin/givc-admin.rs index 09a7668..caf578b 100644 --- a/src/bin/givc-admin.rs +++ b/src/bin/givc-admin.rs @@ -23,7 +23,7 @@ struct Cli { #[arg(long, help = "Additionally listen Vsock socket (cid:port format)")] vsock: Option, - #[arg(long, env = "TLS", default_missing_value = "false")] + #[arg(long, env = "TLS")] use_tls: bool, #[arg(long, env = "CA_CERT")] diff --git a/src/bin/givc-agent.rs b/src/bin/givc-agent.rs index 8b91898..4b7da6b 100644 --- a/src/bin/givc-agent.rs +++ b/src/bin/givc-agent.rs @@ -24,7 +24,7 @@ struct Cli { #[arg(long, env = "PORT", default_missing_value = "9001", value_parser = clap::value_parser!(u16).range(1..))] port: u16, - #[arg(long, env = "TLS", default_missing_value = "false")] + #[arg(long, env = "TLS")] use_tls: bool, #[arg(long, env = "TYPE")] From ef5e7937909e807ae176c4640f9627561860a1fa Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Sun, 13 Oct 2024 19:36:18 +0300 Subject: [PATCH 2/8] Update tonic to 0.12.3 (fix TLS issues) Signed-off-by: Alexander V. Nikolaev --- Cargo.lock | 49 ++++++++++++++++------------------------------- Cargo.toml | 6 +++--- client/Cargo.toml | 4 ++-- common/Cargo.toml | 6 +++--- 4 files changed, 24 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3932928..ff4b3b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -883,15 +883,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -1223,20 +1214,20 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302" dependencies = [ "bytes", "heck 0.5.0", - "itertools 0.12.1", + "itertools", "log", "multimap", "once_cell", "petgraph", "prettyplease", - "prost 0.12.6", - "prost-types 0.12.6", + "prost 0.13.2", + "prost-types", "regex", "syn", "tempfile", @@ -1249,7 +1240,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools", "proc-macro2", "quote", "syn", @@ -1262,21 +1253,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools", "proc-macro2", "quote", "syn", ] -[[package]] -name = "prost-types" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" -dependencies = [ - "prost 0.12.6", -] - [[package]] name = "prost-types" version = "0.13.2" @@ -1805,9 +1787,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", @@ -1837,13 +1819,14 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ "prettyplease", "proc-macro2", "prost-build", + "prost-types", "quote", "syn", ] @@ -1855,7 +1838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b56b874eedb04f89907573b408eab1e87c1c1dce43aac6ad63742f57faa99ff" dependencies = [ "prost 0.13.2", - "prost-types 0.13.2", + "prost-types", "tokio", "tokio-stream", "tonic", @@ -1863,12 +1846,12 @@ dependencies = [ [[package]] name = "tonic-types" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d967793411bc1a5392accf4731114295f0fd122865d22cde46a8584b03402b2" +checksum = "0081d8ee0847d01271392a5aebe960a4600f5d4da6c67648a6382a0940f8b367" dependencies = [ "prost 0.13.2", - "prost-types 0.13.2", + "prost-types", "tonic", ] diff --git a/Cargo.toml b/Cargo.toml index 1af528a..0b5915e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,9 +28,9 @@ regex = "1.11" tokio = {version = "1.0", features = ["rt-multi-thread", "time", "macros", "fs"]} tokio-stream = "0.1" tokio-vsock = "0.5" -tonic = {version="0.12.2", features = ["tls"]} -tonic-types = {version="0.12.2"} -tonic-reflection = {version="0.12.2"} +tonic = {version="0.12", features = ["tls"]} +tonic-types = {version="0.12"} +tonic-reflection = {version="0.12"} tower = {version = "0.4"} tracing = "0.1" tracing-subscriber = {version = "0.3", features = ["env-filter", "tracing-log", "time", "local-time"]} diff --git a/client/Cargo.toml b/client/Cargo.toml index 2e3a520..a6b138c 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -16,8 +16,8 @@ hyper-util = { version = "0.1.4"} tokio = {version = "1.0", features = ["rt-multi-thread", "time", "macros"]} tokio-stream = "0.1" tokio-vsock = "*" -tonic = {version="0.12.2", features = ["tls"]} -tonic-types = {version="0.12.2"} +tonic = {version="0.12", features = ["tls"]} +tonic-types = {version="0.12"} tower = {version = "0.4"} tracing = "0.1" serde = { version = "1.0.202", features = ["derive"]} diff --git a/common/Cargo.toml b/common/Cargo.toml index 5c176c3..529dde3 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -15,12 +15,12 @@ prost = "0.13" tokio = {version = "1.0", features = ["rt-multi-thread", "time", "macros"]} tokio-stream = "0.1" tokio-vsock = "*" -tonic = {version="0.12.2", features = ["tls"]} -tonic-types = {version="0.12.2"} +tonic = {version="0.12", features = ["tls"]} +tonic-types = {version="0.12"} tracing = "0.1" tracing-subscriber = {version = "0.3"} serde = { version = "1.0.202", features = ["derive"]} strum = { version = "0.26", features = ["derive"] } [build-dependencies] -tonic-build = {version = "0.11.0", features = ["prost"]} +tonic-build = {version = "0.12", features = ["prost"]} From 1c488a0ffc278e99093387739dd124864d1d7b21 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Mon, 14 Oct 2024 18:12:24 +0300 Subject: [PATCH 3/8] Refactor agent lookup Signed-off-by: Alexander V. Nikolaev --- src/admin/entry.rs | 4 ++-- src/admin/server.rs | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/admin/entry.rs b/src/admin/entry.rs index 26f5277..068fa9b 100644 --- a/src/admin/entry.rs +++ b/src/admin/entry.rs @@ -26,8 +26,8 @@ pub struct RegistryEntry { } impl RegistryEntry { - pub fn agent(self) -> anyhow::Result { - match self.placement { + pub fn agent(&self) -> anyhow::Result<&EndpointEntry> { + match &self.placement { Placement::Endpoint(endpoint) => Ok(endpoint), Placement::Managed(by) => Err(anyhow!( "Agent endpoint {} is managed by {}!", diff --git a/src/admin/server.rs b/src/admin/server.rs index e0bd989..c245ff1 100644 --- a/src/admin/server.rs +++ b/src/admin/server.rs @@ -104,18 +104,18 @@ impl AdminServiceImpl { vm: VmType::Host, service: ServiceType::Mgr, })?; - self.endpoint(host_mgr).context("Resolving host agent") + self.endpoint(&host_mgr).context("Resolving host agent") } - pub fn endpoint(&self, reentry: RegistryEntry) -> anyhow::Result { + pub fn endpoint(&self, reentry: &RegistryEntry) -> anyhow::Result { Ok(EndpointConfig { - transport: reentry.agent()?, + transport: reentry.agent()?.to_owned(), tls: self.tls_config.clone(), }) } pub fn agent_endpoint(&self, name: &str) -> anyhow::Result { let reentry = self.registry.by_name(name)?; - self.endpoint(reentry) + self.endpoint(&reentry) } pub fn app_entries(&self, name: String) -> anyhow::Result> { @@ -134,13 +134,14 @@ impl AdminServiceImpl { let transport = match &entry.placement { Placement::Managed(parent) => { let parent = self.registry.by_name(parent)?; - parent.agent()? // Fail, if parent also `Managed` + parent.agent()?.to_owned() // Fail, if parent also `Managed` } Placement::Endpoint(endpoint) => endpoint.clone(), // FIXME: avoid clone! }; let tls_name = transport.tls_name.clone(); + info!("Get remote status for {tls_name}!"); let endpoint = EndpointConfig { - transport, + transport: transport.to_owned(), tls: self.tls_config.clone().map(|mut tls| { tls.tls_name = Some(tls_name); tls @@ -537,7 +538,7 @@ impl pb::admin_service_server::AdminService for AdminService { let managers = self.inner.registry.find_map(|re| { (re.r#type.service == ServiceType::Mgr) .then_some(()) - .and_then(|_| self.inner.endpoint(re.clone()).ok()) + .and_then(|_| self.inner.endpoint(re).ok()) }); let locale = req.locale.clone(); tokio::spawn(async move { @@ -571,7 +572,7 @@ impl pb::admin_service_server::AdminService for AdminService { let managers = self.inner.registry.find_map(|re| { (re.r#type.service == ServiceType::Mgr) .then_some(()) - .and_then(|_| self.inner.endpoint(re.clone()).ok()) + .and_then(|_| self.inner.endpoint(re).ok()) }); let timezone = req.timezone.clone(); tokio::spawn(async move { From 1c6fee5c59cc016d23be25cb003db566ca199f37 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Tue, 15 Oct 2024 15:13:04 +0300 Subject: [PATCH 4/8] More verbose outgoing connection Signed-off-by: Alexander V. Nikolaev --- client/src/endpoint.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/client/src/endpoint.rs b/client/src/endpoint.rs index 0bf3a40..ba8b32a 100644 --- a/client/src/endpoint.rs +++ b/client/src/endpoint.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use hyper_util::rt::TokioIo; use tokio::net::UnixStream; use tokio_vsock::{VsockAddr, VsockStream}; @@ -43,6 +43,7 @@ impl TlsConfig { .tls_name .as_deref() .ok_or_else(|| anyhow!("Missing TLS name"))?; + info!("Using TLS name: {tls_name}"); Ok(ClientTlsConfig::new() .ca_certificate(ca) .domain_name(tls_name) @@ -93,14 +94,17 @@ impl EndpointConfig { pub async fn connect(&self) -> anyhow::Result { let url = transport_config_to_url(&self.transport.address, self.tls.is_some()); info!("Connecting to {url}, TLS name {:?}", &self.tls); - let mut endpoint = Endpoint::try_from(url)? + let mut endpoint = Endpoint::try_from(url.clone())? .timeout(Duration::from_secs(5)) .concurrency_limit(30); if let Some(tls) = &self.tls { endpoint = endpoint.tls_config(tls.client_config()?)?; }; let channel = match &self.transport.address { - EndpointAddress::Tcp { .. } => endpoint.connect().await?, + EndpointAddress::Tcp { .. } => endpoint + .connect() + .await + .with_context(|| format!("Connecting TCP {url} with {:?}", self.tls))?, EndpointAddress::Unix(unix) => connect_unix_socket(endpoint, unix).await?, EndpointAddress::Abstract(abs) => connect_unix_socket(endpoint, abs).await?, EndpointAddress::Vsock(vs) => connect_vsock_socket(endpoint, *vs).await?, From c6fcc9b7647f3186cfd823743159d0b9e5582035 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Tue, 15 Oct 2024 15:31:23 +0300 Subject: [PATCH 5/8] Wrong tls name in test Signed-off-by: Alexander V. Nikolaev --- nixos/tests/admin.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/tests/admin.nix b/nixos/tests/admin.nix index 06ebb50..b029862 100644 --- a/nixos/tests/admin.nix +++ b/nixos/tests/admin.nix @@ -69,7 +69,7 @@ in addr = addrs.host; port = "9000"; admin = { - name = "admin"; + name = "admin-vm"; addr = addrs.adminvm; port = "9001"; protocol = "tcp"; # go version expect word "tcp" here, but it unused From 13a26209d6a607270fb4364a0ce703de6d3d2835 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Tue, 15 Oct 2024 16:04:40 +0300 Subject: [PATCH 6/8] Inject tls name to all endpoints Signed-off-by: Alexander V. Nikolaev --- src/admin/server.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/admin/server.rs b/src/admin/server.rs index c245ff1..1c3cfe6 100644 --- a/src/admin/server.rs +++ b/src/admin/server.rs @@ -108,9 +108,14 @@ impl AdminServiceImpl { } pub fn endpoint(&self, reentry: &RegistryEntry) -> anyhow::Result { + let transport = reentry.agent()?.to_owned(); + let tls_name = transport.tls_name.clone(); Ok(EndpointConfig { - transport: reentry.agent()?.to_owned(), - tls: self.tls_config.clone(), + transport, + tls: self.tls_config.clone().map(|mut tls| { + tls.tls_name = Some(tls_name); + tls + }), }) } pub fn agent_endpoint(&self, name: &str) -> anyhow::Result { From 17272c17d00d530dd253e247591e00c2fdb58869 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Tue, 15 Oct 2024 16:05:30 +0300 Subject: [PATCH 7/8] Refactor memory ownership (part 2) Signed-off-by: Alexander V. Nikolaev --- src/admin/server.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/admin/server.rs b/src/admin/server.rs index 1c3cfe6..2b32fda 100644 --- a/src/admin/server.rs +++ b/src/admin/server.rs @@ -123,12 +123,12 @@ impl AdminServiceImpl { self.endpoint(&reentry) } - pub fn app_entries(&self, name: String) -> anyhow::Result> { + pub fn app_entries(&self, name: &str) -> anyhow::Result> { if name.contains('@') { - let list = self.registry.find_names(&name)?; + let list = self.registry.find_names(name)?; Ok(list) } else { - Ok(vec![name]) + Ok(vec![name.to_owned()]) } } @@ -423,10 +423,10 @@ impl pb::admin_service_server::AdminService for AdminService { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status> { - escalate(request, |req| async { + escalate(request, |req| async move { let agent = self.inner.agent_endpoint(&req.app_name)?; let client = SystemDClient::new(agent); - for each in self.inner.app_entries(req.app_name)? { + for each in self.inner.app_entries(&req.app_name)? { _ = client.pause_remote(each).await? } app_success() @@ -437,10 +437,10 @@ impl pb::admin_service_server::AdminService for AdminService { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status> { - escalate(request, |req| async { + escalate(request, |req| async move { let agent = self.inner.agent_endpoint(&req.app_name)?; let client = SystemDClient::new(agent); - for each in self.inner.app_entries(req.app_name)? { + for each in self.inner.app_entries(&req.app_name)? { _ = client.resume_remote(each).await? } app_success() @@ -451,10 +451,10 @@ impl pb::admin_service_server::AdminService for AdminService { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status> { - escalate(request, |req| async { + escalate(request, |req| async move { let agent = self.inner.agent_endpoint(&req.app_name)?; let client = SystemDClient::new(agent); - for each in self.inner.app_entries(req.app_name)? { + for each in self.inner.app_entries(&req.app_name)? { _ = client.stop_remote(each).await? } app_success() From 020f0b2728e3cddd99a5da236ff32460b0b6b4d0 Mon Sep 17 00:00:00 2001 From: "Alexander V. Nikolaev" Date: Tue, 15 Oct 2024 19:11:31 +0300 Subject: [PATCH 8/8] Fix and enable TLS tests Signed-off-by: Alexander V. Nikolaev --- nixos/tests/admin.nix | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nixos/tests/admin.nix b/nixos/tests/admin.nix index b029862..21a9b98 100644 --- a/nixos/tests/admin.nix +++ b/nixos/tests/admin.nix @@ -5,7 +5,7 @@ ... }: let - tls = false; + tls = true; snakeoil = ./snakeoil; addrs = { host = "192.168.101.10"; @@ -232,7 +232,8 @@ in services.openssh.enable = true; givc.appvm = { enable = true; - name = "foot-vm"; + debug = true; + name = "chromium-vm"; addr = addrs.appvm; admin = adminSettings; tls = mkTls "chromium-vm"; @@ -339,7 +340,7 @@ in adminvm.wait_for_file("/etc/timezone.conf") with subtest("Clean run"): - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start foot")) + print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm foot")) time.sleep(10) # Give few seconds to application to spin up wait_for_window("ghaf@appvm") @@ -348,13 +349,13 @@ in appvm.succeed("pkill foot") time.sleep(10) # .. then ask to restart - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start foot")) + print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm foot")) wait_for_window("ghaf@appvm") with subtest("clear exit and restart"): - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm foot-vm clearexit")) + print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm clearexit")) time.sleep(20) # Give few seconds to application to spin up, exit, then start it again - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm foot-vm clearexit")) + print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm clearexit")) with subtest("suspend system"): print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} suspend"))