[feature] #4083: Add pprof-rs support (#4250)

Signed-off-by: Shanin Roman <[email protected]>

.github/workflows/iroha2-profiling-image.yml

@@ -16,7 +16,13 @@ on:
         default: profiling
       IROHA2_RUSTFLAGS:
         required: false
-        default: -C force-frame-pointers=on --cfg wasm_profiling
+        default: -C force-frame-pointers=on
+      IROHA2_FEATURES:
+        required: false
+        default: profiling
+      IROHA2_CARGOFLAGS:
+        required: false
+        default: -Z build-std
 
 jobs:
   registry:
@@ -51,6 +57,8 @@ jobs:
           build-args: |
             "PROFILE=${{ github.event.inputs.IROHA2_PROFILE }}"
             "RUSTFLAGS=${{ github.event.inputs.IROHA2_RUSTFLAGS }}"
+            "FEATURES=${{ github.event.inputs.IROHA2_FEATURES }}"
+            "CARGOFLAGS=${{ github.event.inputs.IROHA2_CARGOFLAGS }}"
           file: ${{ github.event.inputs.IROHA2_DOCKERFILE }}
           # This context specification is required
           context: .

CONTRIBUTING.md

@@ -273,18 +273,18 @@ tokio-console http://127.0.0.1:5555
 
 To optimize performance it's useful to profile iroha.
 
-To do that you should compile iroha with `profiling` profile and with `wasm_profiling` compilation flag: 
+To do that you should compile iroha with `profiling` profile and with `profiling` feature: 
 
 ```bash
-RUSTFLAGS="-C force-frame-pointers=on --cfg wasm_profiling" cargo build --profile profiling
+RUSTFLAGS="-C force-frame-pointers=on" cargo +nightly -Z build-std build --target your-desired-target --profile profiling --features profiling
 ```
 
 Then start iroha and attach profiler of your choice to the iroha pid.
 
 Alternatively it's possible to build iroha inside docker with profiler support and profile iroha this way.
 
 ```bash
-docker build -f Dockerfile.glibc --build-arg="PROFILE=profiling" --build-arg='RUSTFLAGS=-C force-frame-pointers=on --cfg wasm_profiling' -t iroha2:profiling .
+docker build -f Dockerfile.glibc --build-arg="PROFILE=profiling" --build-arg='RUSTFLAGS=-C force-frame-pointers=on' --build-arg='FEATURES=profiling' --build-arg='CARGOFLAGS=-Z build-std' -t iroha2:profiling .
 ```
 
 E.g. using perf (available only on linux):
@@ -304,6 +304,15 @@ It can be done by running:
 cargo run --bin iroha_wasm_builder_cli -- build ./path/to/executor --outfile executor.wasm
 ```
 
+With profiling feature enabled iroha exposes endpoint to scrap pprof profiles:
+
+```bash
+# profile iroha for 30 seconds and get protobuf profile
+curl host:port/debug/pprof/profile?seconds=30 -o profile.pb
+# analyze profile in browser (required installed go)
+go tool pprof -web profile.pb
+```
+
 </details>
 
 ## Style Guides

Cargo.toml

@@ -264,4 +264,4 @@ lto = true
 
 [profile.profiling]
 inherits = "release"
-debug = "line-tables-only"
+debug = "limited"

Dockerfile.glibc

@@ -18,7 +18,9 @@ WORKDIR /iroha
 COPY . .
 ARG PROFILE="deploy"
 ARG RUSTFLAGS=""
-RUN RUSTFLAGS="${RUSTFLAGS}" mold --run cargo build --target x86_64-unknown-linux-gnu --profile "${PROFILE}"
+ARG FEATURES=""
+ARG CARGOFLAGS=""
+RUN RUSTFLAGS="${RUSTFLAGS}" mold --run cargo ${CARGOFLAGS} build --target x86_64-unknown-linux-gnu --profile "${PROFILE}" --features "${FEATURES}"
 
 # final image
 FROM debian:bookworm-slim

cli/Cargo.toml

@@ -19,6 +19,8 @@ workspace = true
 
 [features]
 default = ["bridge", "telemetry", "schema-endpoint"]
+# Enables profiling endpoint
+profiling = ["pprof"]
 
 # Support interoperability with popular blockchain networks (Substrate, Ether, etc.)
 bridge = ["iroha_core/bridge"]
@@ -80,6 +82,9 @@ dashmap = { workspace = true }
 thread-local-panic-hook = { version = "0.1.0", optional = true }
 uuid = { version = "1.4.1", features = ["v4"] }
 
+# TODO: switch to original crate once fix is merged (https://github.com/tikv/pprof-rs/pull/241)
+pprof = { git = " https://github.com/Erigara/pprof-rs", branch = "fix_pointer_align", optional = true, default-features = false, features = ["protobuf-codec", "frame-pointer", "cpp"] }
+
 [dev-dependencies]
 serial_test = "2.0.0"
 

cli/src/torii/mod.rs

@@ -107,6 +107,9 @@ pub enum Error {
     #[cfg(feature = "telemetry")]
     /// Error while getting Prometheus metrics
     Prometheus(#[source] eyre::Report),
+    #[cfg(feature = "profiling")]
+    /// Failed to get pprof profile
+    Pprof(#[source] eyre::Report),
     /// Error while resuming cursor
     UnknownCursor,
 }
@@ -166,6 +169,8 @@ impl Error {
             },
             #[cfg(feature = "telemetry")]
             Prometheus(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            #[cfg(feature = "profiling")]
+            Pprof(_) => StatusCode::INTERNAL_SERVER_ERROR,
         }
     }
 

cli/src/torii/routing.rs

@@ -525,6 +525,30 @@ impl Torii {
             .or(get_router_metrics)
             .or(get_api_version));
 
+        #[cfg(feature = "profiling")]
+        let get_router = {
+            // `warp` panics if there is `/` in the string given to the `warp::path` filter
+            // Path filter has to be boxed to have a single uniform type during iteration
+            let profile_router_path = uri::PROFILE
+                .split('/')
+                .skip_while(|p| p.is_empty())
+                .fold(warp::any().boxed(), |path_filter, path| {
+                    path_filter.and(warp::path(path)).boxed()
+                });
+
+            let profiling_lock = std::sync::Arc::new(tokio::sync::Mutex::new(()));
+            get_router.or(profile_router_path
+                .and(warp::query::<routing::profiling::ProfileParams>())
+                .and_then(move |params| {
+                    let profiling_lock = Arc::clone(&profiling_lock);
+                    async move {
+                        Ok::<_, Infallible>(
+                            routing::profiling::handle_profile(params, profiling_lock).await,
+                        )
+                    }
+                }))
+        };
+
         #[cfg(feature = "schema-endpoint")]
         let get_router = get_router.or(warp::path(uri::SCHEMA)
             .and_then(|| async { Ok::<_, Infallible>(handle_schema().await) }));
@@ -672,3 +696,81 @@ impl Torii {
         Ok(())
     }
 }
+
+#[cfg(feature = "profiling")]
+pub mod profiling {
+    use std::num::{NonZeroU16, NonZeroU64};
+
+    use pprof::protos::Message;
+    use serde::{Deserialize, Serialize};
+
+    use super::*;
+
+    /// Query params used to configure profile gathering
+    #[derive(Serialize, Deserialize, Clone, Copy)]
+    pub struct ProfileParams {
+        /// How often to sample iroha
+        #[serde(default = "ProfileParams::default_frequency")]
+        frequency: NonZeroU16,
+        /// How long to sample iroha
+        #[serde(default = "ProfileParams::default_seconds")]
+        seconds: NonZeroU64,
+    }
+
+    impl ProfileParams {
+        fn default_frequency() -> NonZeroU16 {
+            NonZeroU16::new(99).unwrap()
+        }
+
+        fn default_seconds() -> NonZeroU64 {
+            NonZeroU64::new(10).unwrap()
+        }
+    }
+
+    /// Serve pprof protobuf profiles
+    pub async fn handle_profile(
+        ProfileParams { frequency, seconds }: ProfileParams,
+        profiling_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
+    ) -> Result<Vec<u8>> {
+        match profiling_lock.try_lock() {
+            Ok(_guard) => {
+                let mut body = Vec::new();
+                {
+                    // Create profiler guard
+                    let guard = pprof::ProfilerGuardBuilder::default()
+                        .frequency(frequency.get() as i32)
+                        .blocklist(&["libc", "libgcc", "pthread", "vdso"])
+                        .build()
+                        .map_err(|e| {
+                            Error::Pprof(eyre::eyre!(
+                                "pprof::ProfilerGuardBuilder::build fail: {}",
+                                e
+                            ))
+                        })?;
+
+                    // Collect profiles for seconds
+                    tokio::time::sleep(tokio::time::Duration::from_secs(seconds.get())).await;
+
+                    let report = guard
+                        .report()
+                        .build()
+                        .map_err(|e| Error::Pprof(eyre::eyre!("generate report fail: {}", e)))?;
+
+                    let profile = report.pprof().map_err(|e| {
+                        Error::Pprof(eyre::eyre!("generate pprof from report fail: {}", e))
+                    })?;
+
+                    profile.write_to_vec(&mut body).map_err(|e| {
+                        Error::Pprof(eyre::eyre!("encode pprof into bytes fail: {}", e))
+                    })?;
+                }
+
+                Ok(body)
+            }
+            Err(_) => {
+                // profile already running return error
+                Err(Error::Pprof(eyre::eyre!("profiling already running")))
+            }
+        }
+    }
+}

config/src/torii.rs

@@ -87,6 +87,8 @@ pub mod uri {
     pub const SCHEMA: &str = "schema";
     /// URI for getting the API version currently used
     pub const API_VERSION: &str = "api_version";
+    /// URI for getting cpu profile
+    pub const PROFILE: &str = "debug/pprof/profile";
 }
 
 #[cfg(test)]