diff --git a/.github/benchmarks/bench.json b/.github/benchmarks/bench.json new file mode 100644 index 0000000000..2915ff9914 --- /dev/null +++ b/.github/benchmarks/bench.json @@ -0,0 +1,127 @@ +[ + { + "name": "1 core", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "2 cores", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "4 cores", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "alloc_benchmarks", + "path": "target/x86_64-unknown-hermit/release/alloc_benchmarks", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "micro_benchmarks", + "path": "target/x86_64-unknown-hermit/release/micro_benchmarks", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "mutex test", + "path": "target/x86_64-unknown-hermit/release/mutex", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "Micro Benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", + "iterations": 20, + "group": "Micro" + }, + { + "name": "Allocation Benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", + "iterations": 30, + "group": "Allocations" + }, + { + "name": "Mutex Stress Test", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", + "iterations": 40, + "group": "Mutex" + }, + { + "name": "Pi Multithreaded Benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", + "iterations": 15, + "group": "General", + "plot_group": "Multithreaded Pi Efficiency" + }, + { + "name": "Netbench TCP BW - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1MB" + }, + { + "name": "Netbench TCP BW - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1MB" + }, + { + "name": "Netbench TCP Latency - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1MB" + }, + { + "name": "Netbench TCP Latency - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1MB" + }, + { + "name": "Netbench UDP BW - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1472B" + }, + { + + "name": "Netbench UDP BW - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000 \"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1472B" + }, + { + + "name": "Netbench UDP Latency - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1472B" + }, + { + "name": "Netbench UDP Latency - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1472B" + } +] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f58fbdb2b..91d5344c03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,8 @@ name: CI on: pull_request: merge_group: - + + env: GH_TOKEN: ${{ github.token }} RUSTFLAGS: -Dwarnings @@ -240,3 +241,58 @@ jobs: if: matrix.arch == 'x86_64' - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} --package thread_test --smp 4 qemu ${{ matrix.flags }} if: matrix.arch == 'x86_64' + + benchmark: + name: Bench + runs-on: [self-hosted] + steps: + - name: Checkout hermit-rs + uses: actions/checkout@v4 + with: + repository: hermit-os/hermit-rs + submodules: true + - name: Remove hermit-kernel submodule + run: git rm -r kernel + working-directory: . + - name: Checkout hermit-kernel + uses: actions/checkout@v4 + with: + path: kernel + - name: Install QEMU + run: | + sudo apt-get update + sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm + - uses: mkroening/rust-toolchain-toml@main + - name: Install parallel + run: sudo apt install parallel -y + - name: Download loader + run: | + sudo apt install gh -y + gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 + - name: Install uhyve + run: cargo +stable install --locked uhyve + - name: Check KVM availability + shell: bash + run: | + lscpu + kvm-ok + - name: Add wasm target + run: | + rustup component add llvm-tools + rustup target add wasm32-wasip1 + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Run benchmarks + uses: hermit-os/hermit-bench@kernel + with: + benchmark-file: /kernel/.github/benchmarks/bench.json + build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" + - name: Store benchmark result + # Only save the result if the commit is on main + # This is to avoid saving results for PRs + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') + uses: hermit-os/github-action-benchmark@hermit-rs + with: + tool: 'hermit-bench' + output-file-path: results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true