diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 52dbc6883..be6988eb5 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -17,8 +17,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Fetch submodules - run: git submodule update --init --recursive - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile @@ -54,14 +52,56 @@ jobs: # Automatically push the benchmark result to gh-pages branch # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details auto-push: ${{ github.event_name == 'schedule' }} + + benchmark-mem: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile + - name: Setup build environment + shell: bash + run: | + os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') + ./scripts/setup-$os.sh + - name: Build and load shims and wasi-demo-app + shell: bash + run: | + make OPT_PROFILE=release build install test-image load test-image/oci load/oci + - name: Run Benchmarks + shell: bash + run: | + set -euxo pipefail + for RUNTIME in wasmtime wasmedge wasmer wamr; do + sudo ./scripts/benchmark-mem.sh $RUNTIME > bench-mem-$RUNTIME.json + done + cat bench-mem-* | jq -s 'flatten(1)' > bench-mem.json + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.20.4 + with: + name: Criterion.rs Benchmark + tool: 'customSmallerIsBetter' + output-file-path: bench-mem.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # my experimental local benchmarking seems to have a 20% margin of error. + # So I set the alert threshold to 130% of the previous benchmark result. + # If the current benchmark result is more than 130% of the previous benchmark result, it will fail. + alert-threshold: '130%' + fail-on-alert: ${{ github.event_name == 'schedule' }} + alert-comment-cc-users: '@runwasi-committers' + # Enable Job Summary + summary-always: true + # Automatically push the benchmark result to gh-pages branch + # See https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#charts-on-github-pages-1 for more details + auto-push: ${{ github.event_name == 'schedule' }} benchmark-http: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Fetch submodules - run: git submodule update --init --recursive - uses: actions-rust-lang/setup-rust-toolchain@v1 with: rustflags: '' #Disable. By default this action sets environment variable is set to -D warnings. We manage this in the Makefile diff --git a/crates/containerd-shim-wasm/src/sandbox/cli.rs b/crates/containerd-shim-wasm/src/sandbox/cli.rs index 63a41a989..9859c8f3a 100644 --- a/crates/containerd-shim-wasm/src/sandbox/cli.rs +++ b/crates/containerd-shim-wasm/src/sandbox/cli.rs @@ -32,6 +32,42 @@ macro_rules! revision { }; } +#[cfg(target_os = "linux")] +fn get_mem(pid: u32) -> (usize, usize) { + let mut rss = 0; + let mut total = 0; + for line in std::fs::read_to_string(format!("/proc/{pid}/status")) + .unwrap() + .lines() + { + let line = line.trim(); + // VmPeak is the maximum total virtual memory used so far. + // VmHWM (high water mark) is the maximum resident set memory used so far. + // See: https://man7.org/linux/man-pages/man5/proc_pid_status.5.html + if let Some(rest) = line.strip_prefix("VmPeak:") { + if let Some(rest) = rest.strip_suffix("kB") { + total = rest.trim().parse().unwrap_or(0); + } + } else if let Some(rest) = line.strip_prefix("VmHWM:") { + if let Some(rest) = rest.strip_suffix("kB") { + rss = rest.trim().parse().unwrap_or(0); + } + } + } + (rss, total) +} + +#[cfg(target_os = "linux")] +fn log_mem() { + let pid = std::process::id(); + let (rss, tot) = get_mem(pid); + log::info!("Shim peak memory usage was: peak resident set {rss} kB, peak total {tot} kB"); + + let pid = zygote::Zygote::global().run(|_| std::process::id(), ()); + let (rss, tot) = get_mem(pid); + log::info!("Zygote peak memory usage was: peak resident set {rss} kB, peak total {tot} kB"); +} + /// Main entry point for the shim. /// /// If the `opentelemetry` feature is enabled, this function will start the shim with OpenTelemetry tracing. @@ -70,6 +106,9 @@ pub fn shim_main<'a, I>( { shim_main_inner::(name, version, revision, shim_version, config); } + + #[cfg(target_os = "linux")] + log_mem(); } #[cfg_attr(feature = "tracing", tracing::instrument(parent = tracing::Span::current(), skip_all, level = "Info"))] diff --git a/scripts/benchmark-mem.sh b/scripts/benchmark-mem.sh new file mode 100755 index 000000000..81e6d1bef --- /dev/null +++ b/scripts/benchmark-mem.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euxo pipefail + +# Parse CLI arguments +RUNTIME=${1:-"wasmtime"}; shift || true +IMAGE=${1:-"ghcr.io/containerd/runwasi/wasi-demo-app:latest"}; shift || true + +if [ $IMAGE == "ghcr.io/containerd/runwasi/wasi-demo-app:latest" ] && [ "$#" == "0" ]; then + set -- /wasi-demo-app.wasm echo 'hello' +fi + +# Run the shim and collect logs +LOG_FILE=$(mktemp) +journalctl -fn 0 -u containerd | timeout -k 16s 15s grep -m 2 'peak memory usage was' > $LOG_FILE & +ctr run --null-io --rm --runtime=io.containerd.$RUNTIME.v1 "$IMAGE" testwasm "$@" + +# Parse the logs +wait +SHIM_MEM=$(cat $LOG_FILE | grep 'Shim peak memory usage was' | sed -E 's/.*peak resident set ([0-9]+) kB.*/\1/') +ZYGOTE_MEM=$(cat $LOG_FILE | grep 'Zygote peak memory usage was' | sed -E 's/.*peak resident set ([0-9]+) kB.*/\1/') +rm $LOG_FILE + +if [ "$SHIM_MEM" == "" ] || [ "$ZYGOTE_MEM" == "" ]; then + exit 1 +fi + +TOTAL_MEM=$(( $SHIM_MEM + $ZYGOTE_MEM )) + +# Print the JSON for the benchmark report +cat <