Skip to content

Commit 559bee2

Browse files
CarlWachterCarlWachter
committed
ci: add continuous benchmarking
1 parent d7203ed commit 559bee2

File tree

2 files changed

+179
-1
lines changed

2 files changed

+179
-1
lines changed

‎.github/benchmarks/bench.json

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
[
2+
{
3+
"name": "1 core",
4+
"command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark",
5+
"external_time": true,
6+
"iterations": 25,
7+
"group": "General",
8+
"plot_group": "Startup Time"
9+
},
10+
{
11+
"name": "2 cores",
12+
"command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark",
13+
"external_time": true,
14+
"iterations": 25,
15+
"group": "General",
16+
"plot_group": "Startup Time"
17+
},
18+
{
19+
"name": "4 cores",
20+
"command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark",
21+
"external_time": true,
22+
"iterations": 25,
23+
"group": "General",
24+
"plot_group": "Startup Time"
25+
},
26+
{
27+
"name": "alloc_benchmarks",
28+
"path": "target/x86_64-unknown-hermit/release/alloc_benchmarks",
29+
"group": "File Size",
30+
"plot_group": "Benchmark File Size"
31+
},
32+
{
33+
"name": "micro_benchmarks",
34+
"path": "target/x86_64-unknown-hermit/release/micro_benchmarks",
35+
"group": "File Size",
36+
"plot_group": "Benchmark File Size"
37+
},
38+
{
39+
"name": "mutex test",
40+
"path": "target/x86_64-unknown-hermit/release/mutex",
41+
"group": "File Size",
42+
"plot_group": "Benchmark File Size"
43+
},
44+
{
45+
"name": "Micro Benchmarks",
46+
"command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks",
47+
"iterations": 20,
48+
"group": "Micro"
49+
},
50+
{
51+
"name": "Allocation Benchmarks",
52+
"command": "sudo qemu-system-x86_64 -display none -smp 1 -m 10G -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks",
53+
"iterations": 30,
54+
"group": "Allocations"
55+
},
56+
{
57+
"name": "Mutex Stress Test",
58+
"command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex",
59+
"iterations": 40,
60+
"group": "Mutex"
61+
},
62+
{
63+
"name": "Pi Multithreaded Benchmark",
64+
"command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark",
65+
"iterations": 15,
66+
"group": "General",
67+
"plot_group": "Multithreaded Pi Efficiency"
68+
},
69+
{
70+
"name": "Netbench TCP BW - Client",
71+
"command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'",
72+
"iterations": 3,
73+
"group": "Networking",
74+
"plot_group": "Netbench Bandwidth - 1000x1MB"
75+
},
76+
{
77+
"name": "Netbench TCP BW - Server",
78+
"command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'",
79+
"iterations": 3,
80+
"group": "Networking",
81+
"plot_group": "Netbench Bandwidth - 1000x1MB"
82+
},
83+
{
84+
"name": "Netbench TCP Latency - Client",
85+
"command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250\"'",
86+
"iterations": 3,
87+
"group": "Networking",
88+
"plot_group": "Netbench Latency - 250x1MB"
89+
},
90+
{
91+
"name": "Netbench TCP Latency - Server",
92+
"command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'",
93+
"iterations": 3,
94+
"group": "Networking",
95+
"plot_group": "Netbench Latency - 250x1MB"
96+
},
97+
{
98+
"name": "Netbench UDP BW - Server",
99+
"command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'",
100+
"iterations": 3,
101+
"group": "Networking",
102+
"plot_group": "Netbench Bandwidth - 1000x1472B"
103+
},
104+
{
105+
106+
"name": "Netbench UDP BW - Client",
107+
"command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000 \"'",
108+
"iterations": 3,
109+
"group": "Networking",
110+
"plot_group": "Netbench Bandwidth - 1000x1472B"
111+
},
112+
{
113+
114+
"name": "Netbench UDP Latency - Client",
115+
"command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000\"'",
116+
"iterations": 3,
117+
"group": "Networking",
118+
"plot_group": "Netbench Latency - 250x1472B"
119+
},
120+
{
121+
"name": "Netbench UDP Latency - Server",
122+
"command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'",
123+
"iterations": 3,
124+
"group": "Networking",
125+
"plot_group": "Netbench Latency - 250x1472B"
126+
}
127+
]

‎.github/workflows/ci.yml

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: CI
33
on:
44
pull_request:
55
merge_group:
6-
6+
77
env:
88
GH_TOKEN: ${{ github.token }}
99
RUSTFLAGS: -Dwarnings
@@ -240,3 +240,54 @@ jobs:
240240
if: matrix.arch == 'x86_64'
241241
- run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} --package thread_test --smp 4 qemu ${{ matrix.flags }}
242242
if: matrix.arch == 'x86_64'
243+
244+
benchmark:
245+
name: Bench
246+
runs-on: [self-hosted]
247+
steps:
248+
- name: Checkout hermit-rs
249+
uses: actions/checkout@v4
250+
with:
251+
repository: hermit-os/hermit-rs
252+
submodules: true
253+
- name: Remove hermit-kernel submodule
254+
run: git rm -r kernel
255+
working-directory: .
256+
- name: Checkout hermit-kernel
257+
uses: actions/checkout@v4
258+
with:
259+
path: kernel
260+
- name: Install QEMU
261+
run: |
262+
sudo apt-get update
263+
sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm
264+
- uses: mkroening/rust-toolchain-toml@main
265+
- name: Download loader
266+
run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64
267+
- name: Install uhyve
268+
run: cargo +stable install --locked uhyve
269+
- name: Check KVM availability
270+
shell: bash
271+
run: |
272+
lscpu
273+
kvm-ok
274+
- name: Add wasm target
275+
run: |
276+
rustup component add llvm-tools
277+
rustup target add wasm32-wasip1
278+
rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
279+
- name: Run benchmarks
280+
uses: hermit-os/hermit-bench@kernel
281+
with:
282+
benchmark-file: /kernel/.github/benchmarks/bench.json
283+
build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release"
284+
- name: Store benchmark result
285+
# Only save the result if the commit is on main
286+
# This is to avoid saving results for PRs
287+
if: (github.event_name == 'push' && github.ref == 'refs/heads/main')
288+
uses: hermit-os/github-action-benchmark@hermit-rs
289+
with:
290+
tool: 'hermit-bench'
291+
output-file-path: results.json
292+
github-token: ${{ secrets.GITHUB_TOKEN }}
293+
auto-push: true

0 commit comments

Comments
 (0)