diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d02408bc60..9562fc54a2 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -96,6 +96,7 @@ jobs: run: sudo /root/bin/prep.sh - name: Run cargo bench + if: 0 run: | # Pin all but neqo-bin benchmarks to CPU 0. neqo-bin benchmarks run # both a client and a server, thus benefiting from multiple CPU cores. @@ -122,15 +123,21 @@ jobs: truncate -s "$SIZE" "$TMP/$SIZE" # Define the commands to run for each client and server. declare -A client_cmd=( - ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE" + ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . _flags -Q 1 https://$HOST:$PORT/$SIZE" ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE" - ["gquiche"]="gquiche/bazel-bin/quiche/quic_client --disable_certificate_verification https://$HOST:$PORT/$SIZE > $SIZE || true" + ["gquiche"]="gquiche/bazel-bin/quiche/quic_client --disable_certificate_verification https://$HOST:$PORT/$SIZE > $SIZE" ) declare -A server_cmd=( - ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null" - ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true" + ["neqo"]="target/release/neqo-server _cc _pacing _flags -Q 1 $HOST:$PORT" + ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit" ["gquiche"]="gquiche/bazel-bin/quiche/quic_server --generate_dynamic_responses --port $PORT --certificate_file $TMP/cert --key_file $TMP/key" ) + # Flags to pass to neqo when it runs against another implementation. + declare -A neqo_flags=( + ["neqo"]="" + ["msquic"]="-o -a hq-interop" + ["gquiche"]="" + ) # Replace various placeholders in the commands with the actual values. # Also generate an extension to append to the file name. @@ -138,6 +145,7 @@ jobs: CMD=$1 local cc=$2 local pacing=$3 + local flags=$4 if [ "$cc" != "" ]; then CMD=${CMD//_cc/--cc $cc} EXT="-$cc" @@ -149,6 +157,7 @@ jobs: CMD=${CMD//_pacing/--no-pacing} EXT="$EXT-nopacing" fi + CMD=${CMD//_flags/$flags} } # See https://github.com/microsoft/msquic/issues/4618#issuecomment-2422611592 @@ -156,6 +165,11 @@ jobs: sudo ip link set dev lo mtu "$mtu" for server in gquiche msquic neqo; do for client in gquiche msquic neqo; do + # Do not run msquic against google-quiche; the latter only supports H3. + # Also, we're not really interested in the performance of those combinations. + if [[ "$client" == "gquiche" && "$server" == "msquic" || "$client" == "msquic" && "$server" == "gquiche" ]]; then + continue + fi # gquiche and msquic doesn't let us configure the congestion control or pacing. if [ "$client" != "neqo" ] && [ "$server" != "neqo" ]; then cc_opt=("") @@ -173,18 +187,18 @@ jobs: TAG="$client,$server,$cc,$pacing,$mtu" fi echo "Running benchmarks for $TAG" | tee -a comparison.txt - transmogrify "${server_cmd[$server]}" "$cc" "$pacing" + transmogrify "${server_cmd[$server]}" "$cc" "$pacing" "${neqo_flags[$client]}" echo "$CMD" # shellcheck disable=SC2086 taskset -c 0 nice -n -20 \ perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD & PID=$! - transmogrify "${client_cmd[$client]}" "$cc" "$pacing" + transmogrify "${client_cmd[$client]}" "$cc" "$pacing" "${neqo_flags[$server]}" echo "$CMD" # shellcheck disable=SC2086 taskset -c 1 nice -n -20 \ perf $PERF_OPT -o "$client-$server$EXT.client.perf" \ - hyperfine --ignore-failure -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" | + hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" | tee -a comparison.txt echo >> comparison.txt kill $PID || true