diff --git a/Cargo.lock b/Cargo.lock
index 87a54c78ec1f..f727741883db 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1653,6 +1653,20 @@ dependencies = [
"parking_lot_core 0.9.8",
]
+[[package]]
+name = "dashmap"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core 0.9.8",
+]
+
[[package]]
name = "data-encoding"
version = "2.4.0"
@@ -1952,6 +1966,15 @@ dependencies = [
"syn 2.0.90",
]
+[[package]]
+name = "env_filter"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
+dependencies = [
+ "log",
+]
+
[[package]]
name = "env_logger"
version = "0.10.2"
@@ -1965,6 +1988,16 @@ dependencies = [
"termcolor",
]
+[[package]]
+name = "env_logger"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d"
+dependencies = [
+ "env_filter",
+ "log",
+]
+
[[package]]
name = "equator"
version = "0.2.2"
@@ -2948,6 +2981,28 @@ dependencies = [
"str_stack",
]
+[[package]]
+name = "inferno"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75a5d75fee4d36809e6b021e4b96b686e763d365ffdb03af2bd00786353f84fe"
+dependencies = [
+ "ahash",
+ "clap",
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "dashmap 6.1.0",
+ "env_logger 0.11.2",
+ "indexmap 2.0.1",
+ "itoa",
+ "log",
+ "num-format",
+ "once_cell",
+ "quick-xml 0.37.1",
+ "rgb",
+ "str_stack",
+]
+
[[package]]
name = "inotify"
version = "0.9.6"
@@ -3155,7 +3210,7 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2"
dependencies = [
- "dashmap",
+ "dashmap 5.5.0",
"hashbrown 0.13.2",
]
@@ -4421,7 +4476,7 @@ dependencies = [
"bytes",
"crc32c",
"criterion",
- "env_logger",
+ "env_logger 0.10.2",
"log",
"memoffset 0.9.0",
"once_cell",
@@ -4462,7 +4517,7 @@ dependencies = [
"cfg-if",
"criterion",
"findshlibs",
- "inferno",
+ "inferno 0.11.21",
"libc",
"log",
"nix 0.26.4",
@@ -4688,9 +4743,9 @@ dependencies = [
"clap",
"compute_api",
"consumption_metrics",
- "dashmap",
+ "dashmap 5.5.0",
"ecdsa 0.16.9",
- "env_logger",
+ "env_logger 0.10.2",
"fallible-iterator",
"flate2",
"framed-websockets",
@@ -4797,6 +4852,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "quick-xml"
+version = "0.37.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "quote"
version = "1.0.37"
@@ -7319,6 +7383,7 @@ dependencies = [
"hex-literal",
"humantime",
"hyper 0.14.30",
+ "inferno 0.12.0",
"itertools 0.10.5",
"jemalloc_pprof",
"jsonwebtoken",
@@ -7422,7 +7487,7 @@ dependencies = [
"anyhow",
"camino-tempfile",
"clap",
- "env_logger",
+ "env_logger 0.10.2",
"log",
"postgres",
"postgres_ffi",
diff --git a/Cargo.toml b/Cargo.toml
index 55bb9bfe1152..a4e601bb589c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -110,6 +110,7 @@ hyper-util = "0.1"
tokio-tungstenite = "0.21.0"
indexmap = "2"
indoc = "2"
+inferno = "0.12.0"
ipnet = "2.10.0"
itertools = "0.10"
itoa = "1.0.11"
diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml
index 02bf77760a8e..edb451a02cc9 100644
--- a/libs/utils/Cargo.toml
+++ b/libs/utils/Cargo.toml
@@ -26,6 +26,7 @@ git-version.workspace = true
hex = { workspace = true, features = ["serde"] }
humantime.workspace = true
hyper0 = { workspace = true, features = ["full"] }
+inferno.workspace = true
itertools.workspace = true
fail.workspace = true
futures = { workspace = true }
diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs
index 9b37b699398e..4b4aa88d6bf7 100644
--- a/libs/utils/src/http/endpoint.rs
+++ b/libs/utils/src/http/endpoint.rs
@@ -417,6 +417,7 @@ pub async fn profile_heap_handler(req: Request
) -> Result,
enum Format {
Jemalloc,
Pprof,
+ Svg,
}
// Parameters.
@@ -424,9 +425,24 @@ pub async fn profile_heap_handler(req: Request) -> Result,
None => Format::Pprof,
Some("jemalloc") => Format::Jemalloc,
Some("pprof") => Format::Pprof,
+ Some("svg") => Format::Svg,
Some(format) => return Err(ApiError::BadRequest(anyhow!("invalid format {format}"))),
};
+ // Functions and mappings to strip when symbolizing pprof profiles. If true,
+ // also remove child frames.
+ static STRIP_FUNCTIONS: Lazy> = Lazy::new(|| {
+ vec![
+ (Regex::new("^__rust").unwrap(), false),
+ (Regex::new("^_start$").unwrap(), false),
+ (Regex::new("^irallocx_prof").unwrap(), true),
+ (Regex::new("^prof_alloc_prep").unwrap(), true),
+ (Regex::new("^std::rt::lang_start").unwrap(), false),
+ (Regex::new("^std::sys::backtrace::__rust").unwrap(), false),
+ ]
+ });
+ const STRIP_MAPPINGS: &[&str] = &["libc", "libgcc", "pthread", "vdso"];
+
// Obtain profiler handle.
let mut prof_ctl = jemalloc_pprof::PROF_CTL
.as_ref()
@@ -464,24 +480,9 @@ pub async fn profile_heap_handler(req: Request) -> Result,
// Symbolize the profile.
// TODO: consider moving this upstream to jemalloc_pprof and avoiding the
// serialization roundtrip.
- static STRIP_FUNCTIONS: Lazy> = Lazy::new(|| {
- // Functions to strip from profiles. If true, also remove child frames.
- vec![
- (Regex::new("^__rust").unwrap(), false),
- (Regex::new("^_start$").unwrap(), false),
- (Regex::new("^irallocx_prof").unwrap(), true),
- (Regex::new("^prof_alloc_prep").unwrap(), true),
- (Regex::new("^std::rt::lang_start").unwrap(), false),
- (Regex::new("^std::sys::backtrace::__rust").unwrap(), false),
- ]
- });
let profile = pprof::decode(&bytes)?;
let profile = pprof::symbolize(profile)?;
- let profile = pprof::strip_locations(
- profile,
- &["libc", "libgcc", "pthread", "vdso"],
- &STRIP_FUNCTIONS,
- );
+ let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
pprof::encode(&profile)
})
.await
@@ -494,6 +495,27 @@ pub async fn profile_heap_handler(req: Request) -> Result,
.body(Body::from(data))
.map_err(|err| ApiError::InternalServerError(err.into()))
}
+
+ Format::Svg => {
+ let body = tokio::task::spawn_blocking(move || {
+ let bytes = prof_ctl.dump_pprof()?;
+ let profile = pprof::decode(&bytes)?;
+ let profile = pprof::symbolize(profile)?;
+ let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
+ let mut opts = inferno::flamegraph::Options::default();
+ opts.title = "Heap inuse".to_string();
+ opts.count_name = "bytes".to_string();
+ pprof::flamegraph(profile, &mut opts)
+ })
+ .await
+ .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
+ .map_err(ApiError::InternalServerError)?;
+ Response::builder()
+ .status(200)
+ .header(CONTENT_TYPE, "image/svg+xml")
+ .body(Body::from(body))
+ .map_err(|err| ApiError::InternalServerError(err.into()))
+ }
}
}
diff --git a/libs/utils/src/pprof.rs b/libs/utils/src/pprof.rs
index 90910897bf17..dd57f9ed4b87 100644
--- a/libs/utils/src/pprof.rs
+++ b/libs/utils/src/pprof.rs
@@ -1,8 +1,9 @@
+use anyhow::bail;
use flate2::write::{GzDecoder, GzEncoder};
use flate2::Compression;
use itertools::Itertools as _;
use once_cell::sync::Lazy;
-use pprof::protos::{Function, Line, Message as _, Profile};
+use pprof::protos::{Function, Line, Location, Message as _, Profile};
use regex::Regex;
use std::borrow::Cow;
@@ -188,3 +189,59 @@ pub fn strip_locations(
profile
}
+
+/// Generates an SVG flamegraph from a symbolized pprof profile.
+pub fn flamegraph(
+ profile: Profile,
+ opts: &mut inferno::flamegraph::Options,
+) -> anyhow::Result> {
+ if profile.mapping.iter().any(|m| !m.has_functions) {
+ bail!("profile not symbolized");
+ }
+
+ // Index locations, functions, and strings.
+ let locations: HashMap =
+ profile.location.into_iter().map(|l| (l.id, l)).collect();
+ let functions: HashMap =
+ profile.function.into_iter().map(|f| (f.id, f)).collect();
+ let strings = profile.string_table;
+
+ // Resolve stacks as function names, and sum sample values per stack. Also reverse the stack,
+ // since inferno expects it bottom-up.
+ let mut stacks: HashMap, i64> = HashMap::new();
+ for sample in profile.sample {
+ let mut stack = Vec::with_capacity(sample.location_id.len());
+ for location in sample.location_id.into_iter().rev() {
+ let Some(location) = locations.get(&location) else {
+ bail!("missing location {location}");
+ };
+ for line in location.line.iter().rev() {
+ let Some(function) = functions.get(&line.function_id) else {
+ bail!("missing function {}", line.function_id);
+ };
+ let Some(name) = strings.get(function.name as usize) else {
+ bail!("missing string {}", function.name);
+ };
+ stack.push(name.as_str());
+ }
+ }
+ let Some(&value) = sample.value.first() else {
+ bail!("missing value");
+ };
+ *stacks.entry(stack).or_default() += value;
+ }
+
+ // Construct stack lines for inferno.
+ let lines = stacks
+ .into_iter()
+ .map(|(stack, value)| (stack.into_iter().join(";"), value))
+ .map(|(stack, value)| format!("{stack} {value}"))
+ .sorted()
+ .collect_vec();
+
+ // Construct the flamegraph.
+ let mut bytes = Vec::new();
+ let lines = lines.iter().map(|line| line.as_str());
+ inferno::flamegraph::from_lines(opts, lines, &mut bytes)?;
+ Ok(bytes)
+}