code clean & shutdown in hooks

Signed-off-by: Ziy1-Tan <[email protected]>

vmm/common/Cargo.toml

@@ -19,7 +19,8 @@ async-trait = "0.1"
 regex = "1.5.6"
 
 tracing = "0.1.40"
-tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] }
+tracing-opentelemetry = "0.21.0"
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 
 opentelemetry = { version = "0.20.0", features = ["rt-tokio"] }
 opentelemetry-otlp = "0.13.0"

vmm/common/src/tracer.rs

@@ -14,8 +14,40 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-use opentelemetry::sdk::{trace, trace::Tracer, Resource};
-use tracing_subscriber::EnvFilter;
+use anyhow::anyhow;
+use opentelemetry::{
+    global,
+    sdk::{trace, trace::Tracer, Resource},
+};
+use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer, Registry};
+
+pub fn setup_tracing(
+    log_level: &str,
+    enable_tracing: bool,
+    otlp_service_name: &str,
+) -> anyhow::Result<()> {
+    let env_filter = init_logger_filter(log_level)
+        .map_err(|e| anyhow!("failed to init logger filter: {}", e))?;
+
+    let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
+    if enable_tracing {
+        let tracer = init_otlp_tracer(otlp_service_name)
+            .map_err(|e| anyhow!("failed to init otlp tracer: {}", e))?;
+        layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
+    }
+
+    let subscriber = Registry::default().with(env_filter).with(layers);
+    tracing::subscriber::set_global_default(subscriber)
+        .map_err(|e| anyhow!("failed to set global subscriber: {}", e))?;
+    Ok(())
+}
+
+fn init_logger_filter(log_level: &str) -> anyhow::Result<EnvFilter> {
+    let filter = EnvFilter::from_default_env()
+        .add_directive(format!("containerd_sandbox={}", log_level).parse()?)
+        .add_directive(format!("vmm_sandboxer={}", log_level).parse()?);
+    Ok(filter)
+}
 
 pub fn init_otlp_tracer(otlp_service_name: &str) -> anyhow::Result<Tracer> {
     let tracer = opentelemetry_otlp::new_pipeline()
@@ -28,9 +60,6 @@ pub fn init_otlp_tracer(otlp_service_name: &str) -> anyhow::Result<Tracer> {
     Ok(tracer)
 }
 
-pub fn init_logger_filter(log_level: &str) -> anyhow::Result<EnvFilter> {
-    let filter = EnvFilter::from_default_env()
-        .add_directive(format!("containerd_sandbox={}", log_level).parse()?)
-        .add_directive(format!("vmm_sandboxer={}", log_level).parse()?);
-    Ok(filter)
+pub fn shutdown_tracing() {
+    global::shutdown_tracer_provider();
 }

vmm/sandbox/Cargo.toml

@@ -51,10 +51,6 @@ hostname = "0.3"
 path-clean = "1.0.1"
 
 tracing = "0.1.40"
-tracing-opentelemetry = "0.21.0"
-tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] }
-
-opentelemetry = { version = "0.20.0", features = ["rt-tokio"] }
 
 [[bin]]
 name = "qemu"

vmm/sandbox/src/bin/cloud_hypervisor/main.rs

@@ -15,10 +15,7 @@ limitations under the License.
 */
 
 use clap::Parser;
-use opentelemetry::global;
-use tracing::info_span;
-use tracing_subscriber::{layer::SubscriberExt, Layer, Registry};
-use vmm_common::tracer::{init_logger_filter, init_otlp_tracer};
+use vmm_common::tracer;
 use vmm_sandboxer::{
     args,
     cloud_hypervisor::{factory::CloudHypervisorVMFactory, hooks::CloudHypervisorHooks},
@@ -38,26 +35,19 @@ async fn main() {
     let config = Config::load_config(&args.config).await.unwrap();
 
     // Update args log level if it not presents args but in config.
-    let env_filter = init_logger_filter(&args.log_level.unwrap_or(config.sandbox.log_level()))
-        .expect("failed to init logger filter");
-
-    let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
-    if config.sandbox.enable_tracing {
-        let tracer = init_otlp_tracer("kuasar-vmm-sandboxer-clh-tracing-service")
-            .expect("failed to init otlp tracer");
-        layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
-    }
-
-    let subscriber = Registry::default().with(env_filter).with(layers);
-    tracing::subscriber::set_global_default(subscriber).expect("unable to set global subscriber");
-
-    let root_span = info_span!("kuasar-vmm-sandboxer-clh-root").entered();
+    let enable_tracing = config.sandbox.enable_tracing;
+    tracer::setup_tracing(
+        &args.log_level.unwrap_or(config.sandbox.log_level()),
+        enable_tracing,
+        "kuasar-vmm-sandboxer-clh-otlp-service",
+    )
+    .unwrap();
 
     let mut sandboxer: KuasarSandboxer<CloudHypervisorVMFactory, CloudHypervisorHooks> =
         KuasarSandboxer::new(
             config.sandbox,
             config.hypervisor,
-            CloudHypervisorHooks::default(),
+            CloudHypervisorHooks::new(enable_tracing),
         );
 
     // Do recovery job
@@ -72,7 +62,4 @@ async fn main() {
     )
     .await
     .unwrap();
-
-    root_span.exit();
-    global::shutdown_tracer_provider();
 }

vmm/sandbox/src/bin/qemu/main.rs

@@ -15,10 +15,7 @@ limitations under the License.
 */
 
 use clap::Parser;
-use opentelemetry::global;
-use tracing::{info, info_span};
-use tracing_subscriber::{layer::SubscriberExt, Layer, Registry};
-use vmm_common::tracer::{init_logger_filter, init_otlp_tracer};
+use vmm_common::tracer;
 use vmm_sandboxer::{
     args,
     config::Config,
@@ -54,29 +51,20 @@ async fn main() {
         Config::load_config(&args.config).await.unwrap()
     };
 
-    // Initialize log filter
-    let env_filter =
-        init_logger_filter(&config.sandbox.log_level()).expect("failed to init logger filter");
-
-    let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
-    if config.sandbox.enable_tracing {
-        let tracer = init_otlp_tracer("kuasar-vmm-sandboxer-qemu-tracing-service")
-            .expect("failed to init otlp tracer");
-        layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
-    }
-
-    let subscriber = Registry::default().with(env_filter).with(layers);
-    tracing::subscriber::set_global_default(subscriber).expect("unable to set global subscriber");
-
-    let root_span = info_span!("kuasar-vmm-sandboxer-qemu-root").entered();
+    let enable_tracing = config.sandbox.enable_tracing;
+    tracer::setup_tracing(
+        &config.sandbox.log_level(),
+        enable_tracing,
+        "kuasar-vmm-sandboxer-qemu-otlp-service",
+    )
+    .unwrap();
 
     let sandboxer: KuasarSandboxer<QemuVMFactory, QemuHooks> = KuasarSandboxer::new(
         config.sandbox,
         config.hypervisor.clone(),
-        QemuHooks::new(config.hypervisor),
+        QemuHooks::new(config.hypervisor, enable_tracing),
     );
 
-    info!("Kuasar vmm sandboxer clh is started");
     // Run the sandboxer
     containerd_sandbox::run(
         "kuasar-vmm-sandboxer-qemu",
@@ -86,9 +74,4 @@ async fn main() {
     )
     .await
     .unwrap();
-
-    info!("Kuasar vmm sandboxer clh is exited");
-
-    root_span.exit();
-    global::shutdown_tracer_provider();
 }

vmm/sandbox/src/bin/stratovirt/main.rs

@@ -15,10 +15,7 @@ limitations under the License.
 */
 
 use clap::Parser;
-use opentelemetry::global;
-use tracing::{info, info_span};
-use tracing_subscriber::{layer::SubscriberExt, Layer, Registry};
-use vmm_common::tracer::{init_logger_filter, init_otlp_tracer};
+use vmm_common::tracer;
 use vmm_sandboxer::{
     args,
     config::Config,
@@ -40,31 +37,23 @@ async fn main() {
     let config: Config<StratoVirtVMConfig> = Config::load_config(&args.config).await.unwrap();
 
     // Update args log level if it not presents args but in config.
-    let env_filter = init_logger_filter(&args.log_level.unwrap_or(config.sandbox.log_level()))
-        .expect("failed to init logger filter");
-
-    let mut layers = vec![tracing_subscriber::fmt::layer().boxed()];
-    if config.sandbox.enable_tracing {
-        let tracer = init_otlp_tracer("kuasar-vmm-sandboxer-stratovirt-otlp-service")
-            .expect("failed to init otlp tracer");
-        layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
-    }
-
-    let subscriber = Registry::default().with(env_filter).with(layers);
-    tracing::subscriber::set_global_default(subscriber).expect("unable to set global subscriber");
-
-    let root_span = info_span!("kuasar-vmm-sandboxer-stratovirt-root").entered();
+    let enable_tracing = config.sandbox.enable_tracing;
+    tracer::setup_tracing(
+        &config.sandbox.log_level(),
+        enable_tracing,
+        "kuasar-vmm-sandboxer-stratovirt-otlp-service",
+    )
+    .unwrap();
 
     let mut sandboxer: KuasarSandboxer<StratoVirtVMFactory, StratoVirtHooks> = KuasarSandboxer::new(
         config.sandbox,
         config.hypervisor.clone(),
-        StratoVirtHooks::new(config.hypervisor),
+        StratoVirtHooks::new(config.hypervisor, enable_tracing),
     );
 
     // Do recovery job
     sandboxer.recover(&args.dir).await;
 
-    info!("Kuasar vmm sandboxer stratovirt is started");
     // Run the sandboxer
     containerd_sandbox::run(
         "kuasar-vmm-sandboxer-stratovirt",
@@ -74,9 +63,4 @@ async fn main() {
     )
     .await
     .unwrap();
-
-    info!("Kuasar vmm sandboxer stratovirt is exited");
-
-    root_span.exit();
-    global::shutdown_tracer_provider();
 }

vmm/sandbox/src/cloud_hypervisor/hooks.rs

@@ -15,13 +15,21 @@ limitations under the License.
 */
 
 use containerd_sandbox::error::Result;
+use vmm_common::tracer;
 
 use crate::{
     cloud_hypervisor::CloudHypervisorVM, sandbox::KuasarSandbox, utils::get_resources, vm::Hooks,
 };
 
-#[derive(Default)]
-pub struct CloudHypervisorHooks {}
+pub struct CloudHypervisorHooks {
+    enable_tracing: bool,
+}
+
+impl CloudHypervisorHooks {
+    pub fn new(enable_tracing: bool) -> Self {
+        Self { enable_tracing }
+    }
+}
 
 #[async_trait::async_trait]
 impl Hooks<CloudHypervisorVM> for CloudHypervisorHooks {
@@ -37,6 +45,13 @@ impl Hooks<CloudHypervisorVM> for CloudHypervisorHooks {
         sandbox.sync_clock().await;
         Ok(())
     }
+
+    async fn post_stop(&self, _sandbox: &mut KuasarSandbox<CloudHypervisorVM>) -> Result<()> {
+        if self.enable_tracing {
+            tracer::shutdown_tracing();
+        }
+        Ok(())
+    }
 }
 
 async fn process_annotation(_sandbox: &mut KuasarSandbox<CloudHypervisorVM>) -> Result<()> {

vmm/sandbox/src/qemu/hooks.rs

@@ -16,6 +16,7 @@ limitations under the License.
 
 use async_trait::async_trait;
 use containerd_sandbox::error::Result;
+use vmm_common::tracer;
 
 use crate::{
     qemu::{config::QemuVMConfig, QemuVM},
@@ -27,11 +28,15 @@ use crate::{
 pub struct QemuHooks {
     #[allow(dead_code)]
     config: QemuVMConfig,
+    enable_tracing: bool,
 }
 
 impl QemuHooks {
-    pub fn new(config: QemuVMConfig) -> Self {
-        Self { config }
+    pub fn new(config: QemuVMConfig, enable_tracing: bool) -> Self {
+        Self {
+            config,
+            enable_tracing,
+        }
     }
 }
 
@@ -49,6 +54,13 @@ impl Hooks<QemuVM> for QemuHooks {
         sandbox.sync_clock().await;
         Ok(())
     }
+
+    async fn post_stop(&self, _sandbox: &mut KuasarSandbox<QemuVM>) -> Result<()> {
+        if self.enable_tracing {
+            tracer::shutdown_tracing();
+        }
+        Ok(())
+    }
 }
 
 async fn process_annotation(_sandbox: &KuasarSandbox<QemuVM>) -> Result<()> {