Merge branch 'main' into mtmk-windows-build-tests

# Conflicts:
#	.github/workflows/on-pr-debug.yml

.github/workflows/build-test.yml

@@ -10,12 +10,21 @@ on:
       coverage:
         type: string
         default: "OFF"
-      lib_msg_delivery:
+      dev_mode:
         type: string
         default: "OFF"
-      lib_write_deadline:
+      pool_dispatch:
         type: string
-        default: "OFF"
+        default: "NO-pool"
+      write_deadline:
+        type: string
+        default: "NO-write_deadline"
+      tls:
+        type: string
+        default: "TLS"
+      verify_host:
+        type: string
+        default: "verify_host"
       repeat:
         type: string
         default: "1"
@@ -29,9 +38,6 @@ on:
       streaming:
         type: string
         default: "ON"
-      tls:
-        type: string
-        default: "ON"
       type:
         type: string
         description: "Debug or Release."
@@ -40,6 +46,16 @@ on:
         type: string
         description: "Ubuntu version to use, e.g. '20.04'"
         default: "latest"
+      verbose_test_output:
+        type: string
+        default: "OFF"
+      verbose_make_output:
+        type: string
+        default: "OFF"
+      benchmark:
+        type: string
+        default: "OFF"
+
     secrets:
       CODECOV_TOKEN:
         description: "Codecov repo token"
@@ -72,17 +88,32 @@ jobs:
           flags: -DNATS_BUILD_ARCH=${{ inputs.arch }}
             -DCMAKE_BUILD_TYPE=${{ inputs.type }}
             -DNATS_BUILD_STREAMING=${{ inputs.streaming }}
-            -DNATS_BUILD_WITH_TLS=${{ inputs.tls }}
             -DNATS_PROTOBUF_DIR=${{ github.workspace}}/deps/pbuf
             -DNATS_BUILD_USE_SODIUM=ON
             -DNATS_SODIUM_DIR=${{ github.workspace}}/deps/sodium
         run: |
+          if [[ "${{ inputs.tls }}" == "TLS" ]]; then
+            flags="$flags -DNATS_BUILD_WITH_TLS=ON"
+            if [[ "${{ inputs.verify_host }}" == "verify_host" ]]; then
+              flags="$flags -DNATS_BUILD_TLS_FORCE_HOST_VERIFY=ON"
+            else
+              flags="$flags -DNATS_BUILD_TLS_FORCE_HOST_VERIFY=OFF"
+            fi
+          else
+            flags="$flags -DNATS_BUILD_WITH_TLS=OFF"
+          fi
           if [[ -n "${{ inputs.sanitize }}" ]]; then
             flags="$flags -DNATS_SANITIZE=ON -DCMAKE_C_FLAGS=-fsanitize=${{ inputs.sanitize }}"
           fi
           if [[ "${{ inputs.coverage }}" == "ON" ]]; then
             flags="$flags -DNATS_COVERAGE=ON"
           fi
+          if [[ "${{ inputs.dev_mode }}" == "ON" ]]; then
+            flags="$flags -DDEV_MODE=ON"
+          fi
+          if [[ "${{ inputs.verbose_make_output }}" == "ON" ]]; then
+            flags="$flags -DCMAKE_VERBOSE_MAKEFILE=ON"
+          fi
           echo "flags=$flags" >> $GITHUB_OUTPUT
 
       - id: nats-vars
@@ -91,10 +122,10 @@ jobs:
           if [[ -n "${{ inputs.sanitize }}" ]]; then
             echo "NATS_TEST_VALGRIND=yes" >> $GITHUB_ENV
           fi
-          if [[ "${{ inputs.lib_msg_delivery }}" == "ON" ]]; then
+          if [[ "${{ inputs.pool_dispatch }}" == "pool" ]]; then
             echo "NATS_DEFAULT_TO_LIB_MSG_DELIVERY=yes" >> $GITHUB_ENV
           fi
-          if [[ "${{ inputs.lib_write_deadline }}" == "ON" ]]; then
+          if [[ "${{ inputs.write_deadline }}" == "write_deadline" ]]; then
             echo "NATS_DEFAULT_LIB_WRITE_DEADLINE=2000" >> $GITHUB_ENV
           fi
           echo "CC=${{ inputs.compiler }}" >> $GITHUB_ENV
@@ -114,15 +145,6 @@ jobs:
           cmake .. ${{ steps.cmake-flags.outputs.flags }}
           make rebuild_cache && make
 
-      - name: "Rebuild the list of tests to match the compile flags"
-        working-directory: ./build
-        run: |
-          ./bin/testsuite
-          if [[ $(diff list.txt ../test/list.txt; echo $?) != 0 ]]; then
-            mv list.txt ../test/list.txt
-            make rebuild_cache
-          fi
-
       # testing
 
       - name: "Download nats-server version ${{ inputs.server_version }}"
@@ -153,21 +175,69 @@ jobs:
           fi
 
       - name: "Test"
+        if: inputs.benchmark == 'OFF'
         working-directory: ./build
         run: |
           export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
           export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
           flags=""
-          ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} 2>&1 | tee /tmp/out.txt
-          if [[ $(grep -q 'ThreadSanitizer: ' /tmp/out.txt; echo $?) == 0 ]]; then
-            echo "!!! ThreadSanitizer detected WARNING(s) !!!"
-            exit 1
-          fi
+          ctest -L 'test' --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }}
 
       - name: Upload coverage reports to Codecov
-        if: inputs.coverage == 'ON'
+        # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301
+        # Only upload coverage reports for PRs from the same repo (not forks)
+        if: inputs.coverage == 'ON' && github.event.pull_request.head.repo.full_name == github.repository
         uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
-          # verbose: true
+          # verbose: true
+
+      - name: "Cache for base benchmark for ${{ github.event.pull_request.base.ref }}"
+        id: cache-base-bench
+        if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref
+        uses: actions/cache@v4
+        with:
+          key: bench-${{ github.event.pull_request.base.ref }}
+          path: ./build/bench-${{ github.event.pull_request.base.ref }}.log
+
+      - name: "Benchmark"
+        if: inputs.benchmark == 'ON'
+        working-directory: ./build
+        run: |
+          export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
+          export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
+          flags=""
+          ctest -L 'bench' --timeout 600 -VV | tee bench.log
+
+      - name: "Checkout nats.c for ${{ github.event.pull_request.base.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref && steps.cache-base-bench.outputs.cache-hit != 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+          clean: false
+
+      - name: "Benchmark ${{ github.event.pull_request.base.ref }} for comparison"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref && steps.cache-base-bench.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p build
+          cd build
+          rm -rf CMakeFiles CMakeCache.txt
+          cmake .. ${{ steps.cmake-flags.outputs.flags }}
+          make rebuild_cache && make
+          export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
+          export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
+          flags=""
+          ctest -L 'bench' --timeout 600 -VV | tee bench-${{ github.event.pull_request.base.ref }}.log
+
+      - name: "Checkout HEAD ${{ github.event.pull_request.head.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref && steps.cache-base-bench.outputs.cache-hit != 'true'
+        uses: actions/checkout@v4
+        with:
+          clean: false
+
+      - name: "Compare benchmark to ${{ github.event.pull_request.base.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref
+        run: |
+          cd build
+          go run ../test/diffstat_sub_async.go bench-${{ github.event.pull_request.base.ref }}.log bench.log

.github/workflows/on-pr-debug.yml

@@ -1,4 +1,4 @@
-name: "Debug"
+name: "PR"
 on:
   pull_request:
 
@@ -18,46 +18,85 @@ jobs:
       server_version: main
       type: Debug
 
-  coverage:
-    name: "Coverage"
+  dev-mode:
+    name: "DEV_MODE"
     uses: ./.github/workflows/build-test.yml
     with:
-      coverage: ON
+      dev_mode: ON
       server_version: main
       type: Debug
-    secrets:
-      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      verbose_test_output: ON
+      verbose_make_output: ON
 
-  sanitize-addr:
-    name: "Sanitize address"
+  sanitize:
+    name: "Sanitize"
+    strategy:
+      fail-fast: false
+      matrix:
+        compiler: [gcc, clang]
+        sanitize: [address, thread]
+        pooled_dispatch: [pool, NO-pool]
     uses: ./.github/workflows/build-test.yml
     with:
-      sanitize: address
       server_version: main
-      type: Debug
+      type: RelWithDebInfo
+      compiler: ${{ matrix.compiler }}
+      sanitize: ${{ matrix.sanitize }}
+      pool_dispatch: ${{ matrix.pooled_dispatch }}
 
-  sanitize-addr-lib-msg-delivery:
-    name: "Sanitize address (lib_msg_delivery)"
+  coverage-TLS:
+    name: "Coverage: TLS"
+    strategy:
+      fail-fast: false
+      matrix:
+          pooled_dispatch: [pool, NO-pool]
+          write_deadline: [write_deadline, NO-write_deadline]
     uses: ./.github/workflows/build-test.yml
     with:
-      sanitize: address
+      coverage: ON
+      type: RelWithDebInfo
       server_version: main
-      lib_msg_delivery: ON
+      compiler: gcc
+      tls: TLS
+      verify_host: verify_host
+      pool_dispatch: ${{ matrix.pooled_dispatch }}
+      write_deadline: ${{ matrix.write_deadline }}
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-  san-addr:
-    name: "Sanitize address (lib_write_deadline)"
+  coverage-NO-verify_host:
+    name: "Coverage: NO-verify_host"
     uses: ./.github/workflows/build-test.yml
     with:
-      sanitize: address
+      coverage: ON
+      type: RelWithDebInfo
       server_version: main
-      lib_write_deadline: ON
-
-  san-thread:
-    name: "Sanitize thread"
+      compiler: gcc
+      tls: TLS
+      verify_host: NO-verify_host
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  coverage-NO-TLS:
+    name: "Coverage NO-TLS"
     uses: ./.github/workflows/build-test.yml
     with:
-      sanitize: thread
+      coverage: ON
+      type: RelWithDebInfo
       server_version: main
+      compiler: gcc
+      tls: NO-TLS
+      verify_host: NO-verify_host
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  bench:
+    name: "Benchmark"
+    uses: ./.github/workflows/build-test.yml
+    with:
+      server_version: main
+      benchmark: ON
+      type: Release
 
   Windows:
     name: "Windows"
@@ -77,21 +116,13 @@ jobs:
         env:
           VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
         run: |
-          cmake -B build -S . -DCMAKE_C_FLAGS=/W4
-          cmake --build build --config Debug
+          cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -DNATS_BUILD_STREAMING=OFF
+          cmake --build build
 
       - name: Test
         shell: bash
         run: |
           cd build
-          
-          ./bin/Debug/testsuite
-          mv list.txt ../test/list.txt
-          
-          # refresh the cache
-          cmake -S .. -B .
-          cmake --build . --config Debug
-
           # Download latest nats-server
           rel="latest" # TODO: parameterize
           if [ "$rel" = "latest" ]; then
@@ -117,43 +148,6 @@ jobs:
             mkdir -p ../deps/nats-server
             mv ./nats-server.exe ../deps/nats-server/nats-server.exe
           fi
-          
           export PATH=../deps/nats-server:$PATH
           export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
-
-          run_ctest() {
-            local extra_args="$@"
-            local current_set=$(set +o)
-            set +e
-            echo "Running ctest($extra_args)..."
-
-            ctest -C Debug --timeout 60 --output-on-failure $extra_args
-            
-            local result=$?
-            eval "$current_set"
-            return $result
-          }
-
-          run_ctest
-          result=$?
-      
-          # Retry if failed
-          MAX_RETRIES=10
-          retries=0
-          while [ $result -ne 0 ] && [ $retries -lt $MAX_RETRIES ]; do
-            tasklist | grep nats-server.exe | awk '{print $2}' | xargs -I {} taskkill /PID {} /F /T
-            sleep 3
-            echo "Tests failed. Retrying failed tests... ($((retries + 1))/$MAX_RETRIES)"
-            retries=$((retries + 1))
-            run_ctest --rerun-failed
-            result=$?
-          done
-
-          # Check the final result
-          if [ $result -ne 0 ]; then
-            echo "Tests failed after $MAX_RETRIES retries."
-            exit 1
-          else
-            echo "All tests passed. (retried $retries times)"
-            exit 0
-          fi
+          ctest -L test -C Debug --timeout 60 --output-on-failure --repeat until-pass:3

.github/workflows/on-push-release-extra.yml

@@ -5,7 +5,6 @@ on:
       - main
       - release_*
 
-
 permissions:
   contents: write # required by build-test to comment on coverage but not used here.
 
@@ -67,3 +66,11 @@ jobs:
       type: Debug
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  bench:
+    name: "Benchmark"
+    uses: ./.github/workflows/build-test.yml
+    with:
+      server_version: main
+      benchmark: ON
+      type: Release