Skip to content

Migrate C# ModuleDef to V9 #2675

Migrate C# ModuleDef to V9

Migrate C# ModuleDef to V9 #2675

Workflow file for this run

on:
push:
branches:
- master
- jgilles/fix-callgrind-again
workflow_dispatch:
inputs:
pr_number:
description: 'Pull Request Number'
required: false
default: ''
issue_comment:
types: [created]
name: Benchmarks
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
jobs:
benchmark:
name: run criterion benchmarks
runs-on: benchmarks-runner
# filter for a comment containing 'benchmarks please'
if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
env:
PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
steps:
- name: Clear stdb dir
if: always()
run: |
rm -fr /stdb/*
- name: Enable CPU boost
run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
- name: Check membership
if: ${{ github.event_name == 'issue_comment' }}
env:
CONTRIB_ORG: clockworklabs
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
run: |
curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
exit 1
fi
- name: Post initial comment
run: |
if [[ $PR_NUMBER ]]; then
comment_parent=issues/$PR_NUMBER
comment_update=issues/comments
else
comment_parent=commits/$GITHUB_SHA
comment_update=comments
fi
comment_body="Benchmark in progress..."
comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
- name: find PR branch
if: ${{ env.PR_NUMBER }}
run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ env.PR_REF || github.ref }}
# if we're on master we want to know what the sha of HEAD~1 is so
# that we can compare results from it to HEAD (in the "Fetch markdown
# summary PR" step). otherwise, we can use a fully shallow checkout
fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
components: clippy
toolchain: stable
target: wasm32-unknown-unknown
override: true
- name: Install .NET toolchain
uses: actions/setup-dotnet@v3
with:
dotnet-version: "8.x"
env:
DOTNET_INSTALL_DIR: ~/.dotnet
- name: Build
working-directory: crates/bench/
run: |
cargo build --release
- name: Install latest wasm-opt for module optimisations
run: |
curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1
- name: Disable CPU boost
run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost
- name: Branch; run bench
run: |
if [[ $PR_NUMBER ]]; then
BASELINE_NAME=branch
RESULTS_NAME=pr-$PR_NUMBER
BENCH_FILTER='stdb_raw'
echo "Running benchmarks without sqlite"
else
BASELINE_NAME=master
RESULTS_NAME=$GITHUB_SHA
BENCH_FILTER='(stdb_raw|sqlite)'
echo "Running benchmarks with sqlite"
fi
pushd crates/bench
rm -rf .spacetime
cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
# sticker price benchmark
cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" 'stdb_module/.*/disk/update_bulk'
cargo bench --bench special -- --save-baseline "$BASELINE_NAME"
cargo run --bin summarize pack "$BASELINE_NAME"
popd
mkdir criterion-results
[[ ! $PR_NUMBER ]] && cp target/criterion/$BASELINE_NAME.json criterion-results/
cp target/criterion/$BASELINE_NAME.json criterion-results/$RESULTS_NAME.json
# this will work for both PR and master
- name: Upload criterion results to DO spaces
uses: shallwefootball/s3-upload-action@master
with:
aws_key_id: ${{ secrets.AWS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
aws_bucket: "spacetimedb-ci-benchmarks"
source_dir: criterion-results
endpoint: https://nyc3.digitaloceanspaces.com
destination_dir: benchmarks
- name: Fetch markdown summary PR
run: |
if [[ $PR_NUMBER ]]; then
OLD=master
NEW=pr-$PR_NUMBER
else
OLD=$(git rev-parse HEAD~1)
NEW=$GITHUB_SHA
fi
echo "fetching https://benchmarks.spacetimedb.com/compare/$OLD/$NEW"
curl -sS https://benchmarks.spacetimedb.com/compare/$OLD/$NEW > report.md
- name: Post comment
run: |
BODY="<details><summary>Criterion benchmark results</summary>
$(cat report.md)
</details>"
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Post failure comment
if: ${{ failure() && env.COMMENT_UPDATE_URL }}
run: |
BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Clean up
if: always()
run: |
rm -fr /stdb/*
callgrind_benchmark:
name: run callgrind benchmarks
# DON'T run on benchmarks-runner, using docker on a self-hosted runner has
# been broken for 4 years: https://github.com/actions/runner/issues/434 .
# Fortunately, we can run on standard Github Actions infra because we don't care
# about other stuff running on the machine!
# runs-on: benchmarks-runner
runs-on: ubuntu-latest
timeout-minutes: 20 # on a successful run, runs in 8 minutes
container:
image: rust:1.78
options: --privileged
# filter for a comment containing 'benchmarks please'
if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
env:
PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
steps:
- name: Clear stdb dir
if: always()
shell: bash
run: |
rm -fr /stdb/*
- name: Install valgrind & iai-callgrind-runner
run: |
apt-get update
apt-get install -y valgrind protobuf-compiler bash sudo curl gh
cargo install --git https://github.com/clockworklabs/iai-callgrind.git --branch main iai-callgrind-runner
git config --global --add safe.directory /__w/SpacetimeDB/SpacetimeDB
# can't do this off self hosted:
# - name: Enable CPU boost
# shell: bash
# run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
- name: Check membership
if: ${{ github.event_name == 'issue_comment' }}
env:
CONTRIB_ORG: clockworklabs
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
shell: bash
run: |
curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
exit 1
fi
- name: find PR branch
if: ${{ env.PR_NUMBER }}
shell: bash
run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
- name: Checkout sources
uses: actions/checkout@v3
with:
ref: ${{ env.PR_REF || github.ref }}
# if we're on master we want to know what the sha of HEAD~1 is so
# that we can compare results from it to HEAD (in the "Fetch markdown
# summary PR" step). otherwise, we can use a fully shallow checkout
fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
- name: Unbork Github Actions state
shell: bash
run: |
echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
echo "This is necessary because we are running as root inside a docker image"
chmod -R a+rw .
- name: Post initial comment
shell: bash
run: |
set -exo pipefail
if [[ $PR_NUMBER ]]; then
comment_parent=issues/$PR_NUMBER
comment_update=issues/comments
else
comment_parent=commits/$GITHUB_SHA
comment_update=comments
fi
comment_body="Callgrind benchmark in progress..."
comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
- name: Install stable toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
components: clippy
toolchain: stable
target: wasm32-unknown-unknown
override: true
- name: Build
working-directory: crates/bench/
shell: bash
run: |
cargo build --release
- name: Install latest wasm-opt for module optimisations
shell: bash
run: |
curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1
# leave CPU boost on, doesn't affect callgrind!
- name: Branch; run bench
shell: bash
run: |
if [[ $PR_NUMBER ]]; then
BASELINE_NAME=branch
RESULTS_NAME=pr-$PR_NUMBER
BENCH_FILTER='(special|stdb_module|stdb_raw)'
echo "Running branch callgrind benchmarks"
else
BASELINE_NAME=master
RESULTS_NAME=$GITHUB_SHA
BENCH_FILTER='.*'
echo "Running master callgrind benchmarks"
fi
pushd crates/bench
rm -rf .spacetime
cargo bench --bench callgrind -- --save-summary pretty-json
cargo run --bin summarize pack-callgrind "$BASELINE_NAME"
popd
mkdir callgrind-results
[[ ! $PR_NUMBER ]] && cp target/iai/$BASELINE_NAME.json callgrind-results/
cp target/iai/$BASELINE_NAME.json callgrind-results/$RESULTS_NAME.json
# this will work for both PR and master
- name: Upload callgrind results to DO spaces
uses: shallwefootball/s3-upload-action@master
with:
aws_key_id: ${{ secrets.AWS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
aws_bucket: "spacetimedb-ci-benchmarks"
source_dir: callgrind-results
endpoint: https://nyc3.digitaloceanspaces.com
destination_dir: callgrind-benchmarks
- name: Fetch markdown summary PR
shell: bash
run: |
if [[ $PR_NUMBER ]]; then
OLD=master
NEW=pr-$PR_NUMBER
else
OLD=$(git rev-parse HEAD~1)
NEW=$GITHUB_SHA
fi
echo "fetching https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW"
curl -sS https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW > report.md
- name: Post comment
shell: bash
run: |
BODY="<details><summary>Callgrind benchmark results</summary>
$(cat report.md)
</details>"
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Post failure comment
if: ${{ failure() && env.COMMENT_UPDATE_URL }}
shell: bash
run: |
BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Clean up
if: always()
shell: bash
run: |
rm -fr /stdb/*
echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
echo "This is necessary because we are running as root inside a docker image"
chmod -R a+rw .