Again

.github/workflows/bench.yml

@@ -96,6 +96,7 @@ jobs:
         run: sudo /root/bin/prep.sh
 
       - name: Run cargo bench
+        if: 0
         run: |
           # Pin all but neqo-bin benchmarks to CPU 0. neqo-bin benchmarks run
           # both a client and a server, thus benefiting from multiple CPU cores.
@@ -122,22 +123,29 @@ jobs:
           truncate -s "$SIZE" "$TMP/$SIZE"
           # Define the commands to run for each client and server.
           declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
+            ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . _flags -Q 1 https://$HOST:$PORT/$SIZE"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
-            ["gquiche"]="gquiche/bazel-bin/quiche/quic_client --disable_certificate_verification https://$HOST:$PORT/$SIZE > $SIZE || true"
+            ["gquiche"]="gquiche/bazel-bin/quiche/quic_client --disable_certificate_verification https://$HOST:$PORT/$SIZE > $SIZE"
           )
           declare -A server_cmd=(
-            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true"
+            ["neqo"]="target/release/neqo-server _cc _pacing _flags -Q 1 $HOST:$PORT"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
             ["gquiche"]="gquiche/bazel-bin/quiche/quic_server --generate_dynamic_responses --port $PORT --certificate_file $TMP/cert --key_file $TMP/key"
           )
+          # Flags to pass to neqo when it runs against another implementation.
+          declare -A neqo_flags=(
+            ["neqo"]=""
+            ["msquic"]="-o -a hq-interop"
+            ["gquiche"]=""
+          )
 
           # Replace various placeholders in the commands with the actual values.
           # Also generate an extension to append to the file name.
           function transmogrify {
             CMD=$1
             local cc=$2
             local pacing=$3
+            local flags=$4
             if [ "$cc" != "" ]; then
               CMD=${CMD//_cc/--cc $cc}
               EXT="-$cc"
@@ -149,13 +157,19 @@ jobs:
               CMD=${CMD//_pacing/--no-pacing}
               EXT="$EXT-nopacing"
             fi
+            CMD=${CMD//_flags/$flags}
           }
 
           # See https://github.com/microsoft/msquic/issues/4618#issuecomment-2422611592
           for mtu in 1504; do # We used to measure 65536 here, too, but that's somewhat unrealistic.
             sudo ip link set dev lo mtu "$mtu"
             for server in gquiche msquic neqo; do
               for client in gquiche msquic neqo; do
+                # Do not run msquic against google-quiche; the latter only supports H3.
+                # Also, we're not really interested in the performance of those combinations.
+                if [[ "$client" == "gquiche" && "$server" == "msquic" || "$client" == "msquic" && "$server" == "gquiche" ]]; then
+                  continue
+                fi
                 # gquiche and msquic doesn't let us configure the congestion control or pacing.
                 if [ "$client" != "neqo" ] && [ "$server" != "neqo" ]; then
                   cc_opt=("")
@@ -173,18 +187,18 @@ jobs:
                       TAG="$client,$server,$cc,$pacing,$mtu"
                     fi
                     echo "Running benchmarks for $TAG" | tee -a comparison.txt
-                    transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
+                    transmogrify "${server_cmd[$server]}" "$cc" "$pacing" "${neqo_flags[$client]}"
                     echo "$CMD"
                     # shellcheck disable=SC2086
                     taskset -c 0 nice -n -20 \
                       perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                     PID=$!
-                    transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
+                    transmogrify "${client_cmd[$client]}" "$cc" "$pacing" "${neqo_flags[$server]}"
                     echo "$CMD"
                     # shellcheck disable=SC2086
                     taskset -c 1 nice -n -20 \
                       perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                        hyperfine --ignore-failure -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                        hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                         tee -a comparison.txt
                     echo >> comparison.txt
                     kill $PID || true