From 048621b52448b99f72601a0ac6fe42cb84a7c9a4 Mon Sep 17 00:00:00 2001 From: Lev <1187448+levb@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:13:22 -0700 Subject: [PATCH 01/13] =?UTF-8?q?[CI=20only]=20Skip=20codecov=20for=20fork?= =?UTF-8?q?s=20until=20they=20resolve=20tokenless=20uploads=E2=80=A6=20(#7?= =?UTF-8?q?68)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [CI only] Skip codecov for forks until they resolve tokenless uploads/reports * Update .github/workflows/build-test.yml Co-authored-by: mtmk --------- Co-authored-by: mtmk --- .github/workflows/build-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9a1bf957c..fd6a1b212 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -165,7 +165,9 @@ jobs: fi - name: Upload coverage reports to Codecov - if: inputs.coverage == 'ON' + # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301 + # Only upload coverage reports for PRs from the same repo (not forks) + if: inputs.coverage == 'ON' && github.event.pull_request.head.repo.full_name == github.repository uses: codecov/codecov-action@v4 with: fail_ci_if_error: true From a4892341cd27106b63e26092bcd87e1f5c599327 Mon Sep 17 00:00:00 2001 From: Tristan Ayala Date: Mon, 15 Jul 2024 10:28:54 -0700 Subject: [PATCH 02/13] Change "Ws2_32" to "ws2_32" for MinGW compatibility with no adverse effect for native Windows build. (#763) Co-authored-by: Lev <1187448+levb@users.noreply.github.com> --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d015eb3a..213314243 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,7 +199,7 @@ if(UNIX) elseif(WIN32) set(NATS_LIBDIR "lib") set(NATS_INCLUDE_DIR "include") - set(NATS_EXTRA_LIB "Ws2_32") + set(NATS_EXTRA_LIB "ws2_32") set(NATS_OS "_WIN32") set(NATS_PLATFORM_INCLUDE "win") if(sodium_USE_STATIC_LIBS) From bd700a92681c2f6c440a04eee7daeacfe0eaa232 Mon Sep 17 00:00:00 2001 From: Lev <1187448+levb@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:00:42 -0700 Subject: [PATCH 03/13] [FIXED] cleaning up sanitize=thread found several races (#771) * [TEST+CI only] cleanup of sanitizeThread * Removed repeat 10 * use TSAN_OPTIONS * PR feedback: use natsSock_Shutdown in natsConnection_Reconnect * PR feedback * disabled sanitize with gcc on travis again * Fix data race in _resendSubscriptions: add a delivery worker lock * removed a blank line * removed TSAN extra verbosity * fixed another race, in JetStreamSubscribeIdleHeartbeat * experiment with double-coverage * Fixed test_MicroAsyncErrorHandler_MaxPendingMsgs --- .github/workflows/build-test.yml | 21 +++++++++---- .github/workflows/on-pr-debug.yml | 43 ++++++++++++++++++--------- .github/workflows/on-push-release.yml | 23 ++++++++++++++ src/conn.c | 22 +++++++++++++- src/js.c | 20 +++++++++++++ test/CMakeLists.txt | 7 ++++- test/test.c | 15 ++++++++-- 7 files changed, 128 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index fd6a1b212..0ae656e33 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,6 +10,9 @@ on: coverage: type: string default: "OFF" + dev_mode: + type: string + default: "OFF" lib_msg_delivery: type: string default: "OFF" @@ -40,6 +43,12 @@ on: type: string description: "Ubuntu version to use, e.g. '20.04'" default: "latest" + verbose_test_output: + type: string + default: "OFF" + verbose_make_output: + type: string + default: "ON" secrets: CODECOV_TOKEN: description: "Codecov repo token" @@ -83,6 +92,12 @@ jobs: if [[ "${{ inputs.coverage }}" == "ON" ]]; then flags="$flags -DNATS_COVERAGE=ON" fi + if [[ "${{ inputs.dev_mode }}" == "ON" ]]; then + flags="$flags -DDEV_MODE=ON" + fi + if [[ "${{ inputs.verbose_make_output }}" == "ON" ]]; then + flags="$flags -DCMAKE_VERBOSE_MAKEFILE=ON" + fi echo "flags=$flags" >> $GITHUB_OUTPUT - id: nats-vars @@ -158,11 +173,7 @@ jobs: export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH export NATS_TEST_SERVER_VERSION="$(nats-server -v)" flags="" - ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} 2>&1 | tee /tmp/out.txt - if [[ $(grep -q 'ThreadSanitizer: ' /tmp/out.txt; echo $?) == 0 ]]; then - echo "!!! ThreadSanitizer detected WARNING(s) !!!" - exit 1 - fi + ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} - name: Upload coverage reports to Codecov # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301 diff --git a/.github/workflows/on-pr-debug.yml b/.github/workflows/on-pr-debug.yml index eccc0f9f2..089cd7f58 100644 --- a/.github/workflows/on-pr-debug.yml +++ b/.github/workflows/on-pr-debug.yml @@ -28,36 +28,51 @@ jobs: secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - sanitize-addr: - name: "Sanitize address" + coverage-pooled: + name: "Coverage" uses: ./.github/workflows/build-test.yml with: - sanitize: address + coverage: ON server_version: main type: Debug + lib_msg_delivery: ON + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - sanitize-addr-lib-msg-delivery: - name: "Sanitize address (lib_msg_delivery)" + dev-mode: + name: "DEV_MODE" uses: ./.github/workflows/build-test.yml with: - sanitize: address + dev_mode: ON server_version: main - lib_msg_delivery: ON + type: Debug + verbose_test_output: ON + verbose_make_output: ON - san-addr: - name: "Sanitize address (lib_write_deadline)" + sanitize: + name: "Sanitize" + strategy: + fail-fast: false + matrix: + compiler: [gcc, clang] + sanitize: [address, thread] + pooled_delivery: [ON, OFF] uses: ./.github/workflows/build-test.yml with: - sanitize: address server_version: main - lib_write_deadline: ON + type: Debug + compiler: ${{ matrix.compiler }} + sanitize: ${{ matrix.sanitize }} + lib_msg_delivery: ${{ matrix.pooled_delivery }} - san-thread: - name: "Sanitize thread" + san-addr-deadline: + name: "Sanitize address (lib_write_deadline)" uses: ./.github/workflows/build-test.yml with: - sanitize: thread + type: Debug + sanitize: address server_version: main + lib_write_deadline: ON Windows: name: "Windows" diff --git a/.github/workflows/on-push-release.yml b/.github/workflows/on-push-release.yml index d47c20308..50513575c 100644 --- a/.github/workflows/on-push-release.yml +++ b/.github/workflows/on-push-release.yml @@ -22,3 +22,26 @@ jobs: server_version: main ubuntu_version: ${{ matrix.ubuntu_version }} compiler: ${{ matrix.compiler }} + + dev-mode: + name: "DEV_MODE" + uses: ./.github/workflows/build-test.yml + with: + dev_mode: ON + server_version: main + verbose_test_output: ON + verbose_make_output: ON + + sanitize-addr: + name: "Sanitize address" + uses: ./.github/workflows/build-test.yml + with: + sanitize: address + server_version: main + + san-thread: + name: "Sanitize thread" + uses: ./.github/workflows/build-test.yml + with: + sanitize: thread + server_version: main diff --git a/src/conn.c b/src/conn.c index 1b405c852..7cfe9beb1 100644 --- a/src/conn.c +++ b/src/conn.c @@ -1137,17 +1137,29 @@ _resendSubscriptions(natsConnection *nc) adjustedMax = 0; natsSub_Lock(sub); + if (sub->libDlvWorker != NULL) + { + natsMutex_Lock(sub->libDlvWorker->lock); + } // If JS ordered consumer, trigger a reset. Don't check the error // condition here. If there is a failure, it will be retried // at the next HB interval. if ((sub->jsi != NULL) && (sub->jsi->ordered)) { jsSub_resetOrderedConsumer(sub, sub->jsi->sseq+1); + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); continue; } if (natsSub_drainStarted(sub)) { + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); continue; } @@ -1160,6 +1172,10 @@ _resendSubscriptions(natsConnection *nc) // messages have reached the max, if so, unsubscribe. if (adjustedMax == 0) { + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); s = natsConn_sendUnsubProto(nc, sub->sid, 0); continue; @@ -1172,6 +1188,10 @@ _resendSubscriptions(natsConnection *nc) // Hold the lock up to that point so we are sure not to resend // any SUB/UNSUB for a subscription that is in draining mode. + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); } @@ -3435,7 +3455,7 @@ natsConnection_Reconnect(natsConnection *nc) return nats_setDefaultError(NATS_CONNECTION_CLOSED); } - natsSock_Close(nc->sockCtx.fd); + natsSock_Shutdown(nc->sockCtx.fd); natsConn_Unlock(nc); return NATS_OK; diff --git a/src/js.c b/src/js.c index e31dd2f0a..3d3e4bb18 100644 --- a/src/js.c +++ b/src/js.c @@ -2045,6 +2045,10 @@ _hbTimerFired(natsTimer *timer, void* closure) natsStatus s = NATS_OK; natsSub_Lock(sub); + if (sub->libDlvWorker != NULL) + { + natsMutex_Lock(sub->libDlvWorker->lock); + } alert = !jsi->active; oc = jsi->ordered; jsi->active = false; @@ -2062,10 +2066,18 @@ _hbTimerFired(natsTimer *timer, void* closure) natsCondition_Signal(sub->cond); natsTimer_Stop(timer); } + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); return; } nc = sub->conn; + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); if (!alert) @@ -2075,12 +2087,20 @@ _hbTimerFired(natsTimer *timer, void* closure) if (oc) { natsSub_Lock(sub); + if (sub->libDlvWorker != NULL) + { + natsMutex_Lock(sub->libDlvWorker->lock); + } if (!sub->closed) { // If we fail in that call, we will report to async err callback // (if one is specified). s = jsSub_resetOrderedConsumer(sub, sub->jsi->sseq+1); } + if (sub->libDlvWorker != NULL) + { + natsMutex_Unlock(sub->libDlvWorker->lock); + } natsSub_Unlock(sub); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 58a4aa14f..c15ccfc62 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,7 +35,6 @@ file(STRINGS list.txt listOfTestNames) # For each test name foreach(name ${listOfTestNames}) - # Create a test and pass the index (start and end are the same) # to the testsuite executable add_test(NAME Test_${name} @@ -45,6 +44,12 @@ foreach(name ${listOfTestNames}) # Make sure the test passes set_tests_properties(Test_${name} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED") + # Set TSAN_OPTIONS for the test + if(NATS_SANITIZE) + set_tests_properties(Test_${name} PROPERTIES + ENVIRONMENT "TSAN_OPTIONS=detect_deadlocks=1:second_deadlock_stack=1:halt_on_error=1:report_signal_unsafe=1") + endif(NATS_SANITIZE) + # Bump the test index number math(EXPR testIndex "${testIndex}+1") endforeach() diff --git a/test/test.c b/test/test.c index a2c898038..12a92c4d5 100644 --- a/test/test.c +++ b/test/test.c @@ -5477,6 +5477,7 @@ test_natsSrvVersionAtLeast(void) { s = NATS_ERR; } + natsConn_Unlock(nc); } } testCond(s == NATS_OK); @@ -13337,7 +13338,10 @@ test_ClientAutoUnsubAndReconnect(void) nats_Sleep(10); test("Received no more than max: "); - testCond((s == NATS_OK) && (arg.sum == 10)); + natsMutex_Lock(arg.m); + int sum = arg.sum; + natsMutex_Unlock(arg.m); + testCond((s == NATS_OK) && (sum == 10)); natsSubscription_Destroy(sub); natsConnection_Destroy(nc); @@ -13390,6 +13394,7 @@ test_AutoUnsubNoUnsubOnDestroy(void) natsMutex_Lock(arg.m); while ((s != NATS_TIMEOUT) && !arg.done) s = natsCondition_TimedWait(arg.c, arg.m, 2000); + natsMutex_Unlock(arg.m); testCond(s == NATS_OK); natsConnection_Destroy(nc); @@ -20252,6 +20257,7 @@ test_ForcedReconnect(void) CHECK_SERVER_STARTED(pid); IFOK(s, natsOptions_Create(&opts)); IFOK(s, natsOptions_SetReconnectedCB(opts, _reconnectedCb, &arg)); + IFOK(s, natsOptions_SetReconnectWait(opts, 100)); IFOK(s, natsConnection_Connect(&nc, opts)); IFOK(s, natsConnection_SubscribeSync(&sub, nc, "foo")); testCond(s == NATS_OK); @@ -20800,6 +20806,7 @@ test_EventLoop(void) natsMutex_Lock(arg.m); if (arg.attached != 2 || !arg.detached) s = NATS_ERR; + natsMutex_Unlock(arg.m); testCond(s == NATS_OK); natsSubscription_Destroy(sub); @@ -29485,6 +29492,9 @@ _jsOrderedErrHandler(natsConnection *nc, natsSubscription *subscription, natsSta { struct threadArg *args = (struct threadArg*) closure; + if (err != NATS_MISSED_HEARTBEAT) + return; + natsMutex_Lock(args->m); args->status = err; natsCondition_Signal(args->c); @@ -29824,6 +29834,7 @@ test_JetStreamOrderedConsSrvRestart(void) natsMutex_Lock(args.m); while ((s != NATS_TIMEOUT) && !args.reconnected) s = natsCondition_TimedWait(args.c, args.m, 2000); + natsMutex_Unlock(args.m); testCond(s == NATS_OK); test("Send 1 message: "); @@ -33921,8 +33932,8 @@ test_MicroAsyncErrorHandler_MaxPendingMsgs(void) natsMutex_Lock(arg.m); while ((s != NATS_TIMEOUT) && !arg.closed) s = natsCondition_TimedWait(arg.c, arg.m, 1000); - natsMutex_Unlock(arg.m); testCond((s == NATS_OK) && arg.closed && (arg.status == NATS_SLOW_CONSUMER)); + natsMutex_Unlock(arg.m); microService_Destroy(m); _waitForMicroservicesAllDone(&arg); From d8f52d935b6e83684f9a813bf337504e8745605a Mon Sep 17 00:00:00 2001 From: Lev <1187448+levb@users.noreply.github.com> Date: Sat, 27 Jul 2024 16:01:21 -0700 Subject: [PATCH 04/13] benchmark for SubscribeAsync (#774) * benchmark for SubscribeAsync * properly verbose_make_output default OFF * WIN32 build of bench * WIN32 build of bench, +1 * WIN32 build of bench, +2 * PR feedback: removed irrelevant CMake options and comments * bigger matrix, occasional flush, shorter delay, diffstat * experimental: more flush * refactor test list * moved bench to test refactored bench Added BenchSubscribeAsync_Inject Added BenchSubscribeAsync_InjectSlow * PR feedback: move stopServer before CloseAndWait * Fix Win32 ctest and various warnings * comment --- .github/workflows/build-test.yml | 58 +- .github/workflows/on-pr-debug.yml | 46 +- .github/workflows/on-push-release-extra.yml | 9 +- CMakeLists.txt | 11 + README.md | 37 +- buildOnTravis.sh | 2 +- examples/examples.h | 32 +- examples/micro-func.c | 4 +- examples/micro_args.h | 4 +- examples/stan/pub-async.c | 1 + examples/stan/pub.c | 1 + examples/stan/sub.c | 1 + src/buf.h | 2 + src/crypto.c | 2 +- src/js.c | 4 +- src/js.h | 2 + src/jsm.c | 16 +- src/kv.c | 2 +- src/natsp.h | 2 +- src/sub.c | 2 +- src/util.c | 12 +- test/CMakeLists.txt | 70 +- test/bench_sub_async.c | 501 ++++++ test/list.h | 30 + test/list.txt | 301 ---- test/list_bench.txt | 4 + test/list_stan.txt | 27 + test/list_test.txt | 274 +++ test/test.c | 1669 ++++--------------- test/test.h | 360 ++++ 30 files changed, 1754 insertions(+), 1732 deletions(-) create mode 100644 test/bench_sub_async.c create mode 100644 test/list.h delete mode 100644 test/list.txt create mode 100644 test/list_bench.txt create mode 100644 test/list_stan.txt create mode 100644 test/list_test.txt create mode 100644 test/test.h diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0ae656e33..14ed168c8 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -48,7 +48,11 @@ on: default: "OFF" verbose_make_output: type: string - default: "ON" + default: "OFF" + benchmark: + type: string + default: "OFF" + secrets: CODECOV_TOKEN: description: "Codecov repo token" @@ -129,15 +133,6 @@ jobs: cmake .. ${{ steps.cmake-flags.outputs.flags }} make rebuild_cache && make - - name: "Rebuild the list of tests to match the compile flags" - working-directory: ./build - run: | - ./bin/testsuite - if [[ $(diff list.txt ../test/list.txt; echo $?) != 0 ]]; then - mv list.txt ../test/list.txt - make rebuild_cache - fi - # testing - name: "Download nats-server version ${{ inputs.server_version }}" @@ -168,12 +163,13 @@ jobs: fi - name: "Test" + if: inputs.benchmark == 'OFF' working-directory: ./build run: | export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH export NATS_TEST_SERVER_VERSION="$(nats-server -v)" flags="" - ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} + ctest -L 'test' --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} - name: Upload coverage reports to Codecov # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301 @@ -183,4 +179,42 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} - # verbose: true \ No newline at end of file + # verbose: true + + - name: "Download benchmark results from ${{ github.event.pull_request.base.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref + uses: actions/download-artifact@v2 + with: + name: benchmark_results_${{ github.event.pull_request.base.ref }} + path: ./build/prev_bench.log + continue-on-error: true + + - name: "Benchmark" + if: inputs.benchmark == 'ON' + working-directory: ./build + run: | + export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH + export NATS_TEST_SERVER_VERSION="$(nats-server -v)" + flags="" + ctest -L 'bench' --timeout 600 -VV | tee bench.log + # + # ...coming: compare to base branch + + - name: "Upload benchmark result for PR ${{ github.event.pull_request.head.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref + uses: actions/upload-artifact@v4 + with: + name: benchmark_results_${{ github.event.pull_request.head.ref }} + path: ./build/bench.log + + - name: Extract branch name + if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref + id: extract_branch + run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + + - name: "Upload benchmark result for branch ${{ env.BRANCH_NAME }}" + if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref + uses: actions/upload-artifact@v4 + with: + name: benchmark_results_${{ env.BRANCH_NAME }} + path: ./build/bench.log diff --git a/.github/workflows/on-pr-debug.yml b/.github/workflows/on-pr-debug.yml index 089cd7f58..6ec95f12f 100644 --- a/.github/workflows/on-pr-debug.yml +++ b/.github/workflows/on-pr-debug.yml @@ -1,4 +1,4 @@ -name: "Debug" +name: "PR" on: pull_request: @@ -29,7 +29,7 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} coverage-pooled: - name: "Coverage" + name: "Coverage (pooled delivery)" uses: ./.github/workflows/build-test.yml with: coverage: ON @@ -74,6 +74,14 @@ jobs: server_version: main lib_write_deadline: ON + bench: + name: "Benchmark" + uses: ./.github/workflows/build-test.yml + with: + server_version: main + benchmark: ON + type: Release + Windows: name: "Windows" runs-on: windows-latest @@ -92,10 +100,38 @@ jobs: env: VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" run: | - cmake -B build -S . -DCMAKE_C_FLAGS=/W4 -DNATS_BUILD_STREAMING=OFF - cmake --build build --config Debug + cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -DNATS_BUILD_STREAMING=OFF + cmake --build build - name: Test + shell: bash run: | cd build - ./bin/Debug/testsuite + # Download latest nats-server + rel="latest" # TODO: parameterize + if [ "$rel" = "latest" ]; then + rel=$(curl -s https://api.github.com/repos/nats-io/nats-server/releases/latest | jq -r '.tag_name') + fi + if [ "$rel" != "${rel#v}" ] && wget https://github.com/nats-io/nats-server/releases/download/$rel/nats-server-$rel-windows-amd64.tar.gz; then + tar -xzf nats-server-$rel-linux-amd64.tar.gz + cp nats-server-$rel-windows-amd64/nats-server.exe ../deps/nats-server/nats-server.exe + else + for c in 1 2 3 4 5 + do + echo "Attempt $c to download binary for main" + rm -f ./nats-server + curl -sf "https://binaries.nats.dev/nats-io/nats-server/v2@$rel" | PREFIX=. sh + # We are sometimes getting nats-server of size 0. Make sure we have a + # working nats-server by making sure we get a version number. + mv ./nats-server ./nats-server.exe + v="$(./nats-server.exe -v)" + if [ "$v" != "" ]; then + break + fi + done + mkdir -p ../deps/nats-server + mv ./nats-server.exe ../deps/nats-server/nats-server.exe + fi + export PATH=../deps/nats-server:$PATH + export NATS_TEST_SERVER_VERSION="$(nats-server -v)" + ctest -L test -C Debug --timeout 60 --output-on-failure --repeat until-pass:3 diff --git a/.github/workflows/on-push-release-extra.yml b/.github/workflows/on-push-release-extra.yml index f11aaf4fd..487f5d362 100644 --- a/.github/workflows/on-push-release-extra.yml +++ b/.github/workflows/on-push-release-extra.yml @@ -5,7 +5,6 @@ on: - main - release_* - permissions: contents: write # required by build-test to comment on coverage but not used here. @@ -67,3 +66,11 @@ jobs: type: Debug secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + bench: + name: "Benchmark" + uses: ./.github/workflows/build-test.yml + with: + server_version: main + benchmark: ON + type: Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 213314243..7b87592f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,9 @@ include(FindPackageHandleStandardArgs) # Uncomment to have the build process verbose # set(CMAKE_VERBOSE_MAKEFILE TRUE) +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) + # Set output directories for libraries and executables. # This is important for Windows builds to have the DLLs in the same directory as the executables. set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) @@ -202,6 +205,14 @@ elseif(WIN32) set(NATS_EXTRA_LIB "ws2_32") set(NATS_OS "_WIN32") set(NATS_PLATFORM_INCLUDE "win") + + # Warning control. + add_compile_options(/W4) # Set warning level to maximum, then disable: + add_compile_options(/wd4100) # unreferenced formal parameter + add_compile_options(/wd4200) # nonstandard extension used: zero-sized array in struct/union + add_compile_options(/wd4130) # logical operation on address of string constant + add_compile_options(/wd4127) # conditional expression is constant + if(sodium_USE_STATIC_LIBS) add_definitions( -DSODIUM_STATIC diff --git a/README.md b/README.md index 83badc0c3..fa081026d 100644 --- a/README.md +++ b/README.md @@ -241,38 +241,15 @@ ctest -T memcheck -V -I 1,4 ``` The above command would run the tests with `valgrind` (`-T memcheck`), with verbose output (`-V`), and run the tests from 1 to 4 (`-I 1,4`). -If you add a test to `test/test.c`, you need to add it into the `allTests` array. Each entry contains a name, and the test function. You can add it anywhere into this array. -Build you changes: +If you add a test to `test/test.c`, you need to add it into the `list_test.txt` +file. Each entry contains just the test name, the function must be named +identically, with a `test_` prefix. The list is in alphabetical order, but it +does not need to be, you can add anywhere. -``` -make -[ 44%] Built target nats -[ 88%] Built target nats_static -[ 90%] Built target nats-publisher -[ 92%] Built target nats-queuegroup -[ 94%] Built target nats-replier -[ 96%] Built target nats-requestor -[ 98%] Built target nats-subscriber -Scanning dependencies of target testsuite -[100%] Building C object test/CMakeFiles/testsuite.dir/test.c.o -Linking C executable testsuite -[100%] Built target testsuite -``` - -Now regenerate the list by invoking the test suite without any argument: - -``` -./test/testsuite -Number of tests: 77 -``` - -This list the number of tests added to the file `list.txt`. Move this file to the source's test directory. - -``` -mv list.txt ../test/ -``` +If you are adding a benchmark, it should be added to the `list_bench.txt`. These +tests are labeled differently (`-L 'bench'`) and executed separately on CI. -Then, refresh the build: +You need to re-run `cmake` for the changes to take effect: ``` cmake .. diff --git a/buildOnTravis.sh b/buildOnTravis.sh index f5af110a8..8bc7da678 100755 --- a/buildOnTravis.sh +++ b/buildOnTravis.sh @@ -75,7 +75,7 @@ fi export NATS_TEST_TRAVIS=yes echo "Using NATS server version: $NATS_TEST_SERVER_VERSION" -ctest --timeout 60 --output-on-failure $4 +ctest -L 'test' --timeout 60 --output-on-failure $4 res=$? if [ $res -ne 0 ]; then exit $res diff --git a/examples/examples.h b/examples/examples.h index 643e0e48f..ba30d38ff 100644 --- a/examples/examples.h +++ b/examples/examples.h @@ -45,7 +45,7 @@ volatile int64_t elapsed = 0; bool print = false; int64_t timeout = 10000; // 10 seconds. -natsOptions *opts = NULL; +natsOptions *gOpts = NULL; const char *certFile = NULL; const char *keyFile = NULL; @@ -154,7 +154,7 @@ printUsageAndExit(const char *progName, const char *usage) "%s\n", progName, usage); - natsOptions_Destroy(opts); + natsOptions_Destroy(gOpts); nats_Close(); exit(1); @@ -214,7 +214,7 @@ parseArgs(int argc, char **argv, const char *usage) bool urlsSet = false; int i; - if (natsOptions_Create(&opts) != NATS_OK) + if (natsOptions_Create(&gOpts) != NATS_OK) s = NATS_NO_MEMORY; for (i=1; (idata) #define natsBuf_Capacity(b) ((b)->capacity) #define natsBuf_Len(b) ((b)->len) diff --git a/src/crypto.c b/src/crypto.c index fdebf07e9..04391b220 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -61,7 +61,7 @@ typedef unsigned long long u64; typedef long long i64; typedef i64 gf[16]; -static const gf gf0; +static const gf gf0 = {0}; static const gf gf1 = {1}; static const gf D2 = {0xf159, 0x26b2, 0x9b94, 0xebd6, 0xb156, 0x8283, 0x149a, 0x00e0, 0xd130, 0xeef3, 0x80f2, 0x198e, 0xfce7, 0x56df, 0xd9dc, 0x2406}; static const gf X = {0xd51a, 0x8f25, 0x2d60, 0xc956, 0xa7b2, 0x9525, 0xc760, 0x692c, 0xdc5c, 0xfdd6, 0xe231, 0xc0a4, 0x53fe, 0xcd6e, 0x36d3, 0x2169}; diff --git a/src/js.c b/src/js.c index 3d3e4bb18..221640738 100644 --- a/src/js.c +++ b/src/js.c @@ -497,7 +497,7 @@ js_PublishMsg(jsPubAck **new_puback,jsCtx *js, natsMsg *msg, int64_t ttl = 0; nats_JSON *json = NULL; natsMsg *resp = NULL; - jsApiResponse ar; + jsApiResponse ar = JS_EMPTY_API_RESPONSE; if (errCode != NULL) *errCode = 0; @@ -1807,7 +1807,7 @@ _fetch(natsMsgList *list, natsSubscription *sub, jsFetchRequest *req, bool simpl jsSub *jsi = NULL; natsMsg *mhMsg = NULL; char *reqSubj = NULL; - bool noWait; + bool noWait = false; if (list == NULL) return nats_setDefaultError(NATS_INVALID_ARG); diff --git a/src/js.h b/src/js.h index bc5ec2abe..0503849a7 100644 --- a/src/js.h +++ b/src/js.h @@ -205,6 +205,8 @@ typedef struct __jsApiResponse } jsApiResponse; +#define JS_EMPTY_API_RESPONSE { NULL, { 0, 0, NULL } } + // Sets the options in `resOpts` based on the given `opts` and defaults to the context // own options when some options are not specified. // Returns also the NATS connection to be used to send the request. diff --git a/src/jsm.c b/src/jsm.c index 3c8321491..7c2201231 100644 --- a/src/jsm.c +++ b/src/jsm.c @@ -1517,7 +1517,7 @@ js_GetStreamInfo(jsStreamInfo **new_si, jsCtx *js, const char *stream, jsOptions bool done = false; jsStreamInfo *si = NULL; jsStreamStateSubjects *subjects = NULL; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -1879,7 +1879,7 @@ _getMsg(natsMsg **msg, jsCtx *js, const char *stream, uint64_t seq, const char * bool freePfx = false; jsOptions o; char buffer[64]; - natsBuffer buf; + natsBuffer buf = NATS_EMPTY_BUFFER; if ((msg == NULL) || (js == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); @@ -2029,7 +2029,7 @@ js_DirectGetMsg(natsMsg **msg, jsCtx *js, const char *stream, jsOptions *opts, j bool doLBS = false; jsOptions o; char buffer[64]; - natsBuffer buf; + natsBuffer buf = NATS_EMPTY_BUFFER; if ((msg == NULL) || (js == NULL) || (dgOpts == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); @@ -2108,7 +2108,7 @@ _deleteMsg(jsCtx *js, bool noErase, const char *stream, uint64_t seq, jsOptions bool success = false; jsOptions o; char buffer[64]; - natsBuffer buf; + natsBuffer buf = NATS_EMPTY_BUFFER; if (errCode != NULL) *errCode = 0; @@ -2239,7 +2239,7 @@ js_Streams(jsStreamInfoList **new_list, jsCtx *js, jsOptions *opts, jsErrCode *e natsStrHash *streams= NULL; jsStreamInfoList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -2441,7 +2441,7 @@ js_StreamNames(jsStreamNamesList **new_list, jsCtx *js, jsOptions *opts, jsErrCo natsStrHash *names = NULL; jsStreamNamesList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -3693,7 +3693,7 @@ js_Consumers(jsConsumerInfoList **new_list, jsCtx *js, const char *stream, jsOpt natsStrHash *cons = NULL; jsConsumerInfoList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -3844,7 +3844,7 @@ js_ConsumerNames(jsConsumerNamesList **new_list, jsCtx *js, const char *stream, natsStrHash *names = NULL; jsConsumerNamesList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; diff --git a/src/kv.c b/src/kv.c index 81de4ce56..2b079680c 100644 --- a/src/kv.c +++ b/src/kv.c @@ -1065,7 +1065,7 @@ kvStore_Watch(kvWatcher **new_watcher, kvStore *kv, const char *key, kvWatchOpti natsStatus kvStore_WatchMulti(kvWatcher **new_watcher, kvStore *kv, const char **keys, int numKeys, kvWatchOptions *opts) { - natsStatus s; + natsStatus s = NATS_OK; kvWatcher *w = NULL; jsSubOptions so; char *singleSubject[1]; diff --git a/src/natsp.h b/src/natsp.h index c749b84bb..b9887b5d3 100644 --- a/src/natsp.h +++ b/src/natsp.h @@ -714,7 +714,7 @@ struct __natsConnection // New Request style char respId[NATS_MAX_REQ_ID_LEN+1]; int respIdPos; - int respIdVal; + char respIdVal; char *respSub; // The wildcard subject natsSubscription *respMux; // A single response subscription natsStrHash *respMap; // Request map for the response msg diff --git a/src/sub.c b/src/sub.c index 499909797..3ff9472bc 100644 --- a/src/sub.c +++ b/src/sub.c @@ -451,7 +451,7 @@ natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, sub->msgCb = cb; sub->msgCbClosure = cbClosure; sub->msgsLimit = nc->opts->maxPendingMsgs; - sub->bytesLimit = nc->opts->maxPendingBytes == -1 ? nc->opts->maxPendingMsgs * 1024 : nc->opts->maxPendingBytes;; + sub->bytesLimit = nc->opts->maxPendingBytes == -1 ? nc->opts->maxPendingMsgs * 1024 : (int)nc->opts->maxPendingBytes;; sub->jsi = jsi; sub->subject = NATS_STRDUP(subj); diff --git a/src/util.c b/src/util.c index 4510b4bfe..2d33b3d23 100644 --- a/src/util.c +++ b/src/util.c @@ -1705,7 +1705,7 @@ nats_Base32_Init(void) int i; for (i=0; i<(int)sizeof(base32DecodeMap); i++) - base32DecodeMap[i] = (char) 0xFF; + base32DecodeMap[i] = '\xFF'; for (i=0; i 0) len++; // For the ',' if (strings[i] == NULL) - len += strlen("(null)"); + len += (int)strlen("(null)"); else - len += strlen(strings[i]); + len += (int)strlen(strings[i]); } len++; // For the ']' len++; // For the '\0' diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c15ccfc62..72308f18e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,9 +1,14 @@ +cmake_minimum_required(VERSION 3.10) + +enable_testing() + if(NOT BUILD_TESTING) return() endif() + if(NOT NATS_BUILD_LIB_STATIC) - MESSAGE(FATAL_ERROR - "Building tests require static library, or run CMake with -DBUILD_TESTING=OFF") + MESSAGE(FATAL_ERROR + "Building tests require static library, or run CMake with -DBUILD_TESTING=OFF") return() endif() @@ -13,45 +18,62 @@ endif() # We need this to build the test program include_directories(${PROJECT_SOURCE_DIR}/src) + if(NATS_BUILD_WITH_TLS) include_directories(${OPENSSL_INCLUDE_DIR}) endif(NATS_BUILD_WITH_TLS) + if(NATS_BUILD_STREAMING) include_directories(${NATS_PROTOBUF_INCLUDE_DIRS}) include_directories(${PROJECT_SOURCE_DIR}/src/stan) endif(NATS_BUILD_STREAMING) # Build the test program -add_executable(testsuite test.c) +add_executable(testsuite test.c bench_sub_async.c) # Link statically with the library target_link_libraries(testsuite nats_static ${NATS_EXTRA_LIB}) -# Set the test index to 0 -set(testIndex 0) +set(BENCH_LIST ${PROJECT_SOURCE_DIR}/test/list_bench.txt) +set(STAN_LIST ${PROJECT_SOURCE_DIR}/test/list_stan.txt) -# Read the file 'list.txt' to get all the test names -file(STRINGS list.txt listOfTestNames) +list(APPEND ALL_LISTS ${PROJECT_SOURCE_DIR}/test/list_test.txt) +list(APPEND ALL_LISTS ${BENCH_LIST}) +if(NATS_BUILD_STREAMING) +list(APPEND ALL_LISTS ${STAN_LIST}) +message("Building Streaming tests" ${ALL_LISTS}) +endif() -# For each test name -foreach(name ${listOfTestNames}) - # Create a test and pass the index (start and end are the same) - # to the testsuite executable - add_test(NAME Test_${name} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMAND testsuite ${testIndex} ${testIndex}) +set(TEST_NAMES) - # Make sure the test passes - set_tests_properties(Test_${name} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED") +foreach(LIST_FILE ${ALL_LISTS}) + # Get all the test names + if(EXISTS ${LIST_FILE}) + file(STRINGS ${LIST_FILE} TEST_NAMES) + else() + set(TEST_NAMES) + endif() - # Set TSAN_OPTIONS for the test - if(NATS_SANITIZE) - set_tests_properties(Test_${name} PROPERTIES - ENVIRONMENT "TSAN_OPTIONS=detect_deadlocks=1:second_deadlock_stack=1:halt_on_error=1:report_signal_unsafe=1") - endif(NATS_SANITIZE) + foreach(name ${TEST_NAMES}) + # Remove the _test() prefix + string(REGEX REPLACE "_test\\(([^)]+)\\)" "\\1" TEST_NAME ${name}) + add_test(NAME ${TEST_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND testsuite ${TEST_NAME}) - # Bump the test index number - math(EXPR testIndex "${testIndex}+1") -endforeach() + if(${LIST_FILE} STREQUAL ${BENCH_LIST}) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "bench") + else() + # Make sure the test passes + set_tests_properties(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED") + # Set TSAN_OPTIONS for the test + if(NATS_SANITIZE) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT "TSAN_OPTIONS=detect_deadlocks=1:second_deadlock_stack=1:halt_on_error=1:report_signal_unsafe=1") + endif(NATS_SANITIZE) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "test") + endif() + endforeach() +endforeach() diff --git a/test/bench_sub_async.c b/test/bench_sub_async.c new file mode 100644 index 000000000..bda14a1d1 --- /dev/null +++ b/test/bench_sub_async.c @@ -0,0 +1,501 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test.h" +#include "sub.h" + +#define REPEAT 5 + +typedef struct __env ENV; + +typedef struct +{ + bool useGlobalDelivery; + int max; +} threadConfig; + +typedef struct +{ + natsSubscription *sub; + uint64_t sum; + uint64_t xor ; + uint64_t count; + int64_t closedTimestamp; + + ENV *env; +} subState; + +typedef natsStatus (*publishFunc)(natsConnection *nc, const char *subject, ENV *env); + +struct __env +{ + int numSubs; + threadConfig threads; + int numPubMessages; + + bool progressiveFlush; + publishFunc pubf; + int64_t delayNano; + + subState subs[1000]; // magic number is always enough. +}; + +static void _onMessage(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure); +static void _onError(natsConnection *nc, natsSubscription *sub, natsStatus err, void *closure); +static void _onComplete(void *closure); +static void _benchMatrix(threadConfig *threadsVector, int lent, int *subsVector, int lens, int numMessages, ENV *env); +static natsStatus _bench(ENV *env, int *best, int *avg, int *worst); +static natsStatus _publish(natsConnection *nc, const char *subject, ENV *env); +static natsStatus _inject(natsConnection *nc, const char *subject, ENV *env); +static natsStatus _enqueueToSub(natsSubscription *sub, natsMsg *m); +static uint64_t _expectedSum(int N); +static uint64_t _expectedXOR(int N); + +#define RUN_MATRIX(_threads, _subs, _messages, _env) _benchMatrix(_threads, sizeof(_threads) / sizeof(*_threads), _subs, sizeof(_subs) / sizeof(*_subs), _messages, _env) + +// This benchmark publishes messages ASAP (no rate limiting) and measures +// message delivery to a few subscribers. This approach does not work well for a +// large number of subscribers because the server will be overwhelmed having to +// send too many messages at once. +void test_BenchSubscribeAsync_Small(void) +{ + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 5}, + // These should show no material difference since no extra threads will be spun up + {true, 7}, + }; + + int subs[] = {1, 2, 3, 5}; + + ENV env = { + .pubf = _publish, + .progressiveFlush = false, + }; + RUN_MATRIX(threads, subs, 500 * 1000, &env); +} + +// This benchmark publishes messages, flushing the connection every now and then +// to ensure the server is not overwhelmed, and measures message delivery to a +// few subscribers. This approach works well for a large number of +// subscribers, but can be too slow for a few subscriptions. +void test_BenchSubscribeAsync_Large(void) +{ + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 2}, + {true, 11}, + {true, 163}, // to compare to non-pooled + }; + + int subs[] = {23, 83, 163}; + + ENV env = { + .pubf = _publish, + .progressiveFlush = true, + }; + + RUN_MATRIX(threads, subs, 500 * 1000, &env); +} + +// This benchmark injects the messages directly into the relevant queue for +// delivery, bypassing the publish step. +void test_BenchSubscribeAsync_Inject(void) +{ + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 2}, + {true, 3}, + {true, 7}, + {true, 11}, + {true, 19}, + {true, 163}, + }; + + int subs[] = {1, 8, 23, 83, 163, 499}; + + ENV env = { + .pubf = _inject, + }; + + RUN_MATRIX(threads, subs, 1000 * 1000, &env); +} + +// This benchmark injects the messages directly into the relevant queue for +// delivery, bypassing the publish step. It uses a delay to simulate a slow-ish +// callback. +void test_BenchSubscribeAsync_InjectSlow(void) +{ +#ifdef _WIN32 + // This test relies on nanosleep, not sure what the Windows equivalent is. Skip fr now. + printf("Skipping BenchSubscribeAsync_InjectSlow on Windows\n"); + return; + +#else + + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 2}, + {true, 3}, + {true, 7}, + {true, 11}, + {true, 79}, + {true, 499}, + }; + + int subs[] = {1, 8, 12, 83, 163, 499}; + + ENV env = { + .pubf = _inject, + .delayNano = 10 * 1000, // 10µs + }; + + RUN_MATRIX(threads, subs, 20 * 1000, &env); +#endif // _WIN32 +} + +static void _benchMatrix(threadConfig *threadsVector, int lent, int *subsVector, int lens, int NMessages, ENV *env) +{ + printf("[\n"); + for (int *sv = subsVector; sv < subsVector + lens; sv++) + { + int numSubs = *sv; + bool uselessFromHere = false; + int numPubMessages = NMessages / numSubs; + if (numPubMessages == 0) + numPubMessages = 1; + + for (threadConfig *tv = threadsVector; tv < threadsVector + lent; tv++) + { + natsStatus s = NATS_OK; + threadConfig threads = *tv; + int best = 0, average = 0, worst = 0; + + if (threads.useGlobalDelivery) + { + if (uselessFromHere) + continue; + if (threads.max > numSubs) + uselessFromHere = true; // execute this test, but a larger MaxThreads will not make a difference. + } + + env->numSubs = numSubs; + env->numPubMessages = numPubMessages; + env->threads = threads; + for (int i = 0; i < REPEAT; i++) + { + int b = 0, a = 0, w = 0; + s = _bench(env, &b, &a, &w); + if (s != NATS_OK) + { + fprintf(stderr, "Error: %s\n", natsStatus_GetText(s)); + nats_PrintLastErrorStack(stderr); + exit(1); + } + + if ((b < best) || (best == 0)) + best = b; + if (w > worst) + worst = w; + average += a; + } + average /= REPEAT; + + const char *comma = (sv == subsVector + lens - 1) && (tv == threadsVector + lent - 1) ? "" : ","; + printf("\t{\"subs\":%d, \"threads\":%d, \"messages\":%d, \"best\":%d, \"average\":%d, \"worst\":%d}%s\n", + numSubs, env->threads.useGlobalDelivery ? env->threads.max : 0, numPubMessages * numSubs, best, average, worst, comma); + fflush(stdout); + } + } + printf("]\n"); +} + +static natsStatus _bench(ENV *env, int *best, int *avg, int *worst) +{ + natsConnection *nc = NULL; + natsOptions *opts = NULL; + uint64_t expectedSum = _expectedSum(env->numPubMessages); + uint64_t expectedXOR = _expectedXOR(env->numPubMessages); + char subject[256]; + int64_t start, b, w, a; + + if (env->numSubs > 1000) // magic number check. + return NATS_INVALID_ARG; + memset(env->subs, 0, sizeof(subState) * 1000); + for (int i = 0; i < env->numSubs; i++) + env->subs[i].env = env; // set the environment to access it in the callbacks. + + natsPid pid = _startServer("nats://127.0.0.1:4222", NULL, true); + if (pid == NATS_INVALID_PID) + return NATS_ERR; + + natsStatus s = nats_Open(-1); + IFOK(s, natsNUID_Next(subject, NUID_BUFFER_LEN + 1)); + IFOK(s, natsOptions_Create(&opts)); + IFOK(s, nats_SetMessageDeliveryPoolSize(env->threads.max)); + IFOK(s, natsOptions_SetErrorHandler(opts, _onError, NULL)); + IFOK(s, natsOptions_UseGlobalMessageDelivery(opts, env->threads.useGlobalDelivery)); + + IFOK(s, natsConnection_Connect(&nc, opts)); + + for (int i = 0; i < env->numSubs; i++) + { + IFOK(s, natsConnection_Subscribe(&(env->subs[i].sub), nc, subject, _onMessage, &env->subs[i])); + IFOK(s, natsSubscription_SetPendingLimits(env->subs[i].sub, -1, -1)); + IFOK(s, natsSubscription_AutoUnsubscribe(env->subs[i].sub, env->numPubMessages)); + IFOK(s, natsSubscription_SetOnCompleteCB(env->subs[i].sub, _onComplete, &env->subs[i])); + } + + start = nats_Now(); + + // Publish or inject the messages! + IFOK(s, env->pubf(nc, subject, env)); + + while (s == NATS_OK) + { + bool done = true; + for (int i = 0; i < env->numSubs; i++) + { + // threads don't touch this, should be safe + if (natsSubscription_IsValid(env->subs[i].sub)) + { + done = false; + break; + } + } + + nats_Sleep(10); + if (done) + break; + } + + b = w = a = 0; + if (s == NATS_OK) + { + for (int i = 0; i < env->numSubs; i++) + { + if (env->subs[i].sum != expectedSum) + { + s = NATS_ERR; + fprintf(stderr, "Error: sum is %" PRId64 " for sub %d, expected %" PRId64 "\n", env->subs[i].sum, i, expectedSum); + break; + } + if (env->subs[i].xor != expectedXOR) + { + fprintf(stderr, "Error: xor is %" PRId64 " for sub %d, expected %" PRId64 "\n", env->subs[i].xor, i, expectedXOR); + s = NATS_ERR; + break; + } + if ((int)(env->subs[i].count) != env->numPubMessages) + { + fprintf(stderr, "Error: count is %" PRId64 " for sub %d, expected %d\n", env->subs[i].count, i, env->numPubMessages); + s = NATS_ERR; + break; + } + + int64_t dur = env->subs[i].closedTimestamp - start; + if (dur > w) + w = dur; + if ((dur < b) || (b == 0)) + b = dur; + a += dur; + } + } + + // cleanup + for (int i = 0; i < env->numSubs; i++) + natsSubscription_Destroy(env->subs[i].sub); + natsConnection_Destroy(nc); + natsOptions_Destroy(opts); + _stopServer(pid); + nats_CloseAndWait(0); + + *best = (int)b; + *avg = (int)(a / env->numSubs); + *worst = (int)w; + + return s; +} + +static void _onMessage(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure) +{ + subState *ss = (subState *)closure; + +#ifndef _WIN32 + if (ss->env->delayNano > 0) + { + struct timespec wait = {0, ss->env->delayNano}; + nanosleep(&wait, NULL); + } +#endif + + char buf[32]; + int len = natsMsg_GetDataLength(msg); + if (len > 31) + len = 31; + + strncpy(buf, natsMsg_GetData(msg), len); + buf[len] = '\0'; + int64_t val = atoi(buf); + + ss->sum += val; + ss->xor ^= val; + ss->count++; + + natsMsg_Destroy(msg); +} + +static void _onComplete(void *closure) +{ + subState *ss = (subState *)closure; + ss->closedTimestamp = nats_Now(); +} + +static natsStatus _publish(natsConnection *nc, const char *subject, ENV *env) +{ + natsStatus s = NATS_OK; + char buf[16]; + + int flushAfter = env->progressiveFlush ? env->numPubMessages / (env->numSubs * 2) : // trigger + env->numPubMessages + 1; // do not trigger + for (int i = 0; i < env->numPubMessages; i++) + { + snprintf(buf, sizeof(buf), "%d", i); + IFOK(s, natsConnection_PublishString(nc, subject, buf)); + + if (((i != 0) && ((i % flushAfter) == 0)) || // progressive flush + (i == (env->numPubMessages - 1))) // last message in batch + { + IFOK(s, natsConnection_Flush(nc)); + } + } + + return s; +} + +static natsStatus _inject(natsConnection *nc, const char *subject, ENV *env) +{ + natsStatus s = NATS_OK; + natsMsg *m = NULL; + char buf[16]; + + for (int i = 0; i < env->numPubMessages; i++) + { + for (int n = 0; n < env->numSubs; n++) + { + snprintf(buf, sizeof(buf), "%d", i); + + s = natsMsg_Create(&m, subject, NULL, buf, (int)strlen(buf)); + IFOK(s, _enqueueToSub(env->subs[n].sub, m)); + } + } + + return s; +} + +static natsStatus _enqueueToSub(natsSubscription *sub, natsMsg *m) +{ + natsStatus s = NATS_OK; + natsCondition *cond = NULL; + nats_MsgList *list = NULL; + natsMsgDlvWorker *ldw = NULL; + + // Pick condition variable and list based on if the sub is + // part of a global delivery thread pool or not. + // Note about `list`: this is used only to link messages, but + // sub->msgList needs to be used to update/check number of pending + // messages, since in case of delivery thread pool, `list` will have + // messages from many different subscriptions. + if ((ldw = sub->libDlvWorker) != NULL) + { + cond = ldw->cond; + list = &(ldw->msgList); + } + else + { + cond = sub->cond; + list = &(sub->msgList); + } + + natsSubAndLdw_LockAndRetain(sub); + + sub->msgList.msgs++; + sub->msgList.bytes += natsMsg_GetDataLength(m); + if (((sub->msgsLimit > 0) && (sub->msgList.msgs > sub->msgsLimit)) || ((sub->bytesLimit > 0) && (sub->msgList.bytes > sub->bytesLimit))) + { + natsMsg_Destroy(m); + + sub->dropped++; + sub->slowConsumer = true; + + // Undo stats from above. + sub->msgList.msgs--; + sub->msgList.bytes -= natsMsg_GetDataLength(m); + } + else + { + bool signal = false; + + if (sub->msgList.msgs > sub->msgsMax) + sub->msgsMax = sub->msgList.msgs; + + if (sub->msgList.bytes > sub->bytesMax) + sub->bytesMax = sub->msgList.bytes; + + sub->slowConsumer = false; + + m->sub = sub; + if (list->head == NULL) + { + list->head = m; + signal = true; + } + else + list->tail->next = m; + + list->tail = m; + + if (signal) + natsCondition_Signal(cond); + } + + natsSubAndLdw_UnlockAndRelease(sub); + return s; +} + +static uint64_t _expectedSum(int N) +{ + uint64_t sum = 0; + for (int64_t i = 0; i < N; i++) + sum += i; + return sum; +} + +static uint64_t _expectedXOR(int N) +{ + uint64_t xor = 0; + for (int64_t i = 0; i < N; i++) + xor ^= i; + return xor; +} + +static void _onError(natsConnection *nc, natsSubscription *sub, natsStatus err, void *closure) +{ + int64_t dropped = 0; + natsSubscription_GetDropped(sub, (int64_t *)&dropped); + printf("Async error: sid:%" PRId64 ", dropped:%" PRId64 ": %u - %s\n", sub->sid, dropped, err, natsStatus_GetText(err)); +} diff --git a/test/list.h b/test/list.h new file mode 100644 index 000000000..7b0e368b5 --- /dev/null +++ b/test/list.h @@ -0,0 +1,30 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef _TEST_PROTO +#undef _test +#define _test(name) void test_##name(void); +#endif + +#ifdef _TEST_LIST +#undef _test +#define _test(name) {#name, test_##name}, +#endif + +#include "list_test.txt" +#include "list_bench.txt" +#if defined(NATS_HAS_STREAMING) +#include "list_stan.txt" +#endif + +#undef _test diff --git a/test/list.txt b/test/list.txt deleted file mode 100644 index 60be59288..000000000 --- a/test/list.txt +++ /dev/null @@ -1,301 +0,0 @@ -Version -VersionMatchesTag -OpenCloseAndWait -natsNowAndSleep -natsAllocSprintf -natsStrCaseStr -natsSnprintf -natsBuffer -natsParseInt64 -natsParseControl -natsNormalizeErr -natsMutex -natsThread -natsCondition -natsTimer -natsUrl -natsCreateStringFromBuffer -natsHash -natsHashing -natsStrHash -natsInbox -natsOptions -natsSock_ConnectTcp -natsSock_ShuffleIPs -natsSock_IPOrder -natsSock_ReadLine -natsJSON -natsEncodeTimeUTC -natsErrWithLongText -natsErrStackMoreThanMaxFrames -natsMsg -natsBase32 -natsBase64 -natsCRC16 -natsKeys -natsReadFile -natsGetJWTOrSeed -natsHostIsIP -natsWaitReady -natsSign -HeadersLift -HeadersAPIs -MsgIsJSControl -SrvVersionAtLeast -FormatStringArray -ReconnectServerStats -ParseStateReconnectFunctionality -ServersRandomize -SelectNextServer -ParserPing -ParserErr -ParserOK -ParseINFO -ParserShouldFail -ParserSplitMsg -ProcessMsgArgs -LibMsgDelivery -AsyncINFO -RequestPool -NoFlusherIfSendAsapOption -HeadersAndSubPendingBytes -DefaultConnection -SimplifiedURLs -IPResolutionOrder -UseDefaultURLIfNoServerSpecified -ConnectToWithMultipleURLs -ConnectionWithNULLOptions -ConnectionToWithNullURLs -ConnectionStatus -ConnClosedCB -CloseDisconnectedCB -ServerStopDisconnectedCB -ClosedConnections -ConnectVerboseOption -ReconnectThreadLeak -ReconnectTotalTime -ReconnectDisallowedFlags -ReconnectAllowedFlags -ConnCloseBreaksReconnectLoop -BasicReconnectFunctionality -ExtendedReconnectFunctionality -QueueSubsOnReconnect -IsClosed -IsReconnectingAndStatus -ReconnectBufSize -RetryOnFailedConnect -NoPartialOnReconnect -ReconnectFailsPendingRequests -ForcedReconnect -ErrOnConnectAndDeadlock -ErrOnMaxPayloadLimit -Auth -AuthFailNoDisconnectCB -AuthToken -AuthTokenHandler -PermViolation -AuthViolation -AuthenticationExpired -AuthenticationExpiredReconnect -ConnectedServer -MultipleClose -SimplePublish -SimplePublishNoData -PublishMsg -InvalidSubsArgs -AsyncSubscribe -AsyncSubscribeTimeout -SyncSubscribe -PubSubWithReply -NoResponders -Flush -ConnCloseDoesFlush -QueueSubscriber -ReplyArg -SyncReplyArg -Unsubscribe -DoubleUnsubscribe -SubRemovedWhileProcessingMsg -RequestTimeout -Request -RequestNoBody -RequestMuxWithMappedSubject -OldRequest -SimultaneousRequests -RequestClose -CustomInbox -MessagePadding -FlushInCb -ReleaseFlush -FlushErrOnDisconnect -Inbox -Stats -BadSubject -SubBadSubjectAndQueueNames -ClientAsyncAutoUnsub -ClientSyncAutoUnsub -ClientAutoUnsubAndReconnect -AutoUnsubNoUnsubOnDestroy -NextMsgOnClosedSub -CloseSubRelease -IsValidSubscriber -SlowSubscriber -SlowAsyncSubscriber -SlowConsumerCb -PendingLimitsDeliveredAndDropped -PendingLimitsWithSyncSub -AsyncSubscriptionPending -AsyncSubscriptionPendingDrain -SyncSubscriptionPending -SyncSubscriptionPendingDrain -AsyncErrHandlerMaxPendingMsgs -AsyncErrHandlerMaxPendingBytes -AsyncErrHandlerSubDestroyed -AsyncSubscriberStarvation -AsyncSubscriberOnClose -NextMsgCallOnAsyncSub -SubOnComplete -GetLastError -StaleConnection -ServerErrorClosesConnection -NoEcho -NoEchoOldServer -DrainSub -DrainSubStops -DrainSubRaceOnAutoUnsub -DrainSubNotResentOnReconnect -DrainConn -NoDoubleCloseCbOnDrain -GetClientID -GetClientIP -GetRTT -GetLocalIPAndPort -UserCredsCallbacks -UserCredsFromFiles -UserCredsFromMemory -NKey -NKeyFromSeed -ConnSign -WriteDeadline -HeadersNotSupported -HeadersBasic -MsgsFilter -EventLoop -EventLoopRetryOnFailedConnect -EventLoopTLS -SSLBasic -SSLVerify -SSLCAFromMemory -SSLCertAndKeyFromMemory -SSLVerifyHostname -SSLSkipServerVerification -SSLCiphers -SSLMultithreads -SSLConnectVerboseOption -SSLSocketLeakEventLoop -SSLReconnectWithAuthError -SSLAvailable -ServersOption -AuthServers -AuthFailToReconnect -ReconnectWithTokenHandler -BasicClusterReconnect -HotSpotReconnect -ProperReconnectDelay -ProperFalloutAfterMaxAttempts -StopReconnectAfterTwoAuthErr -TimeoutOnNoServer -PingReconnect -GetServers -GetDiscoveredServers -DiscoveredServersCb -IgnoreDiscoveredServers -INFOAfterFirstPONGisProcessedOK -ServerPoolUpdatedOnClusterUpdate -ReconnectJitter -CustomReconnectDelay -LameDuckMode -ReconnectImplicitUserInfo -JetStreamUnmarshalAccInfo -JetStreamUnmarshalStreamState -JetStreamUnmarshalStreamCfg -JetStreamUnmarshalStreamInfo -JetStreamMarshalStreamCfg -JetStreamUnmarshalConsumerInfo -JetStreamContext -JetStreamDomain -JetStreamMgtStreams -JetStreamMgtConsumers -JetStreamPublish -JetStreamPublishAsync -JetStreamPublishAckHandler -JetStreamSubscribe -JetStreamSubscribeSync -JetStreamSubscribeConfigCheck -JetStreamSubscribeIdleHeartbeat -JetStreamSubscribeFlowControl -JetStreamSubscribePull -JetStreamSubscribeHeadersOnly -JetStreamOrderedCons -JetStreamOrderedConsWithErrors -JetStreamOrderedConsAutoUnsub -JetStreamOrderedConsSrvRestart -JetStreamSubscribeWithFWC -JetStreamStreamsSealAndRollup -JetStreamGetMsgAndLastMsg -JetStreamConvertDirectMsg -JetStreamDirectGetMsg -JetStreamNakWithDelay -JetStreamBackOffRedeliveries -JetStreamInfoWithSubjects -JetStreamInfoAlternates -KeyValueManager -KeyValueBasics -KeyValueWatch -KeyValueWatchMulti -KeyValueHistory -KeyValueKeys -KeyValueDeleteVsPurge -KeyValueDeleteTombstones -KeyValueDeleteMarkerThreshold -KeyValueCrossAccount -KeyValueDiscardOldToNew -KeyValueRePublish -KeyValueMirrorDirectGet -KeyValueMirrorCrossDomains -MicroMatchEndpointSubject -MicroAddService -MicroGroups -MicroBasics -MicroStartStop -MicroServiceStopsOnClosedConn -MicroServiceStopsWhenServerStops -MicroAsyncErrorHandlerMaxPendingMsgs -MicroAsyncErrorHandlerMaxPendingBytes -StanPBufAllocator -StanConnOptions -StanSubOptions -StanMsg -StanServerNotReachable -StanBasicConnect -StanConnectError -StanBasicPublish -StanBasicPublishAsync -StanPublishTimeout -StanPublishMaxAcksInflight -StanBasicSubscription -StanSubscriptionCloseAndUnsub -StanDurableSubscription -StanBasicQueueSubscription -StanDurableQueueSubscription -StanCheckReceivedMsg -StanSubscriptionAckMsg -StanPings -StanPingsNoResponder -StanConnectionLostHandlerNotSet -StanPingsUnblockPublishCalls -StanGetNATSConnection -StanNoRetryOnFailedConnect -StanInternalSubsNotPooled -StanSubOnComplete -StanSubTimeout diff --git a/test/list_bench.txt b/test/list_bench.txt new file mode 100644 index 000000000..2c51bb260 --- /dev/null +++ b/test/list_bench.txt @@ -0,0 +1,4 @@ +_test(BenchSubscribeAsync_Large) +_test(BenchSubscribeAsync_Small) +_test(BenchSubscribeAsync_Inject) +_test(BenchSubscribeAsync_InjectSlow) diff --git a/test/list_stan.txt b/test/list_stan.txt new file mode 100644 index 000000000..b4cd28666 --- /dev/null +++ b/test/list_stan.txt @@ -0,0 +1,27 @@ +_test(StanBasicConnect) +_test(StanBasicPublish) +_test(StanBasicPublishAsync) +_test(StanBasicQueueSubscription) +_test(StanBasicSubscription) +_test(StanCheckReceivedMsg) +_test(StanConnectError) +_test(StanConnectionLostHandlerNotSet) +_test(StanConnOptions) +_test(StanDurableQueueSubscription) +_test(StanDurableSubscription) +_test(StanGetNATSConnection) +_test(StanInternalSubsNotPooled) +_test(StanMsg) +_test(StanNoRetryOnFailedConnect) +_test(StanPBufAllocator) +_test(StanPings) +_test(StanPingsNoResponder) +_test(StanPingsUnblockPubCalls) +_test(StanPublishMaxAcksInflight) +_test(StanPublishTimeout) +_test(StanServerNotReachable) +_test(StanSubOnComplete) +_test(StanSubOptions) +_test(StanSubscriptionAckMsg) +_test(StanSubscriptionCloseAndUnsubscribe) +_test(StanSubTimeout) diff --git a/test/list_test.txt b/test/list_test.txt new file mode 100644 index 000000000..0804b84fb --- /dev/null +++ b/test/list_test.txt @@ -0,0 +1,274 @@ +_test(AsyncErrHandlerMaxPendingBytes) +_test(AsyncErrHandlerMaxPendingMsgs) +_test(AsyncErrHandlerSubDestroyed) +_test(AsyncINFO) +_test(AsyncSubscribe) +_test(AsyncSubscriberOnClose) +_test(AsyncSubscriberStarvation) +_test(AsyncSubscribeTimeout) +_test(AsyncSubscriptionPending) +_test(AsyncSubscriptionPendingDrain) +_test(Auth) +_test(AuthenticationExpired) +_test(AuthenticationExpiredReconnect) +_test(AuthFailNoDisconnectCB) +_test(AuthFailToReconnect) +_test(AuthServers) +_test(AuthToken) +_test(AuthTokenHandler) +_test(AuthViolation) +_test(AutoUnsubNoUnsubOnDestroy) +_test(BadSubject) +_test(BasicClusterReconnect) +_test(BasicReconnectFunctionality) +_test(ClientAsyncAutoUnsub) +_test(ClientAutoUnsubAndReconnect) +_test(ClientSyncAutoUnsub) +_test(ClosedConnections) +_test(CloseDisconnectedCB) +_test(CloseSubRelease) +_test(ConnCloseBreaksReconnectLoop) +_test(ConnClosedCB) +_test(ConnCloseDoesFlush) +_test(ConnectedServer) +_test(ConnectionStatus) +_test(ConnectionToWithNullURLs) +_test(ConnectionWithNullOptions) +_test(ConnectToWithMultipleURLs) +_test(ConnectVerboseOption) +_test(ConnSign) +_test(CustomInbox) +_test(CustomReconnectDelay) +_test(DefaultConnection) +_test(DiscoveredServersCb) +_test(DoubleUnsubscribe) +_test(DrainConn) +_test(DrainSub) +_test(DrainSubNotResentOnReconnect) +_test(DrainSubRaceOnAutoUnsub) +_test(DrainSubStops) +_test(ErrOnConnectAndDeadlock) +_test(ErrOnMaxPayloadLimit) +_test(EventLoop) +_test(EventLoopRetryOnFailedConnect) +_test(EventLoopTLS) +_test(ExtendedReconnectFunctionality) +_test(Flush) +_test(FlushErrOnDisconnect) +_test(FlushInCb) +_test(ForcedReconnect) +_test(GetClientID) +_test(GetClientIP) +_test(GetDiscoveredServers) +_test(GetLastError) +_test(GetLocalIPAndPort) +_test(GetRTT) +_test(GetServers) +_test(HeadersAndSubPendingBytes) +_test(HeadersBasic) +_test(HeadersLift) +_test(HeadersNotSupported) +_test(HotSpotReconnect) +_test(IgnoreDiscoveredServers) +_test(Inbox) +_test(InvalidSubsArgs) +_test(IPResolutionOrder) +_test(IsClosed) +_test(IsReconnectingAndStatus) +_test(IsValidSubscriber) +_test(JetStreamBackOffRedeliveries) +_test(JetStreamContext) +_test(JetStreamContextDomain) +_test(JetStreamConvertDirectMsg) +_test(JetStreamDirectGetMsg) +_test(JetStreamGetMsgAndLastMsg) +_test(JetStreamInfoAlternates) +_test(JetStreamInfoWithSubjects) +_test(JetStreamMarshalStreamConfig) +_test(JetStreamMgtConsumers) +_test(JetStreamMgtStreams) +_test(JetStreamNakWithDelay) +_test(JetStreamOrderedConsSrvRestart) +_test(JetStreamOrderedConsumer) +_test(JetStreamOrderedConsumerWithAutoUnsub) +_test(JetStreamOrderedConsumerWithErrors) +_test(JetStreamPublish) +_test(JetStreamPublishAckHandler) +_test(JetStreamPublishAsync) +_test(JetStreamStreamsSealAndRollup) +_test(JetStreamSubscribe) +_test(JetStreamSubscribeConfigCheck) +_test(JetStreamSubscribeFlowControl) +_test(JetStreamSubscribeHeadersOnly) +_test(JetStreamSubscribeIdleHearbeat) +_test(JetStreamSubscribePull) +_test(JetStreamSubscribeSync) +_test(JetStreamSubscribeWithFWC) +_test(JetStreamUnmarshalAccountInfo) +_test(JetStreamUnmarshalConsumerInfo) +_test(JetStreamUnmarshalStreamConfig) +_test(JetStreamUnmarshalStreamInfo) +_test(JetStreamUnmarshalStreamState) +_test(KeyValueBasics) +_test(KeyValueCrossAccount) +_test(KeyValueDeleteTombstones) +_test(KeyValueDeleteVsPurge) +_test(KeyValueDiscardOldToNew) +_test(KeyValueHistory) +_test(KeyValueKeys) +_test(KeyValueManager) +_test(KeyValueMirrorCrossDomains) +_test(KeyValueMirrorDirectGet) +_test(KeyValuePurgeDeletesMarkerThreshold) +_test(KeyValueRePublish) +_test(KeyValueWatch) +_test(KeyValueWatchMulti) +_test(LameDuckMode) +_test(LibMsgDelivery) +_test(MessageBufferPadding) +_test(MicroAddService) +_test(MicroAsyncErrorHandlerMaxPendingBytes) +_test(MicroAsyncErrorHandlerMaxPendingMsgs) +_test(MicroBasics) +_test(MicroGroups) +_test(MicroMatchEndpointSubject) +_test(MicroServiceStopsOnClosedConn) +_test(MicroServiceStopsWhenServerStops) +_test(MicroStartStop) +_test(MultipleClose) +_test(natsAllocSprintf) +_test(natsBase32Decode) +_test(natsBase64Encode) +_test(natsBuffer) +_test(natsCondition) +_test(natsCRC16) +_test(natsCreateStringFromBuffer) +_test(natsEncodeTimeUTC) +_test(natsErrStackMoreThanMaxFrames) +_test(natsErrWithLongText) +_test(natsFormatStringArray) +_test(natsGetJWTOrSeed) +_test(natsHash) +_test(natsHashing) +_test(natsHostIsIP) +_test(natsInbox) +_test(natsJSON) +_test(natsKeys) +_test(natsMsg) +_test(natsMsgHeaderAPIs) +_test(natsMsgIsJSCtrl) +_test(natsMsgsFilter) +_test(natsMutex) +_test(natsNormalizeErr) +_test(natsNowAndSleep) +_test(natsOptions) +_test(natsParseControl) +_test(natsParseInt64) +_test(natsReadFile) +_test(natsSign) +_test(natsSnprintf) +_test(natsSock_ConnectTcp) +_test(natsSock_IPOrder) +_test(natsSock_ReadLine) +_test(natsSock_ShuffleIPs) +_test(natsSrvVersionAtLeast) +_test(natsStrCaseStr) +_test(natsStrHash) +_test(natsThread) +_test(natsTimer) +_test(natsUrl) +_test(natsWaitReady) +_test(NextMsgCallOnAsyncSub) +_test(NextMsgOnClosedSub) +_test(NKey) +_test(NKeyFromSeed) +_test(NoDoubleConnClosedOnDrain) +_test(NoEcho) +_test(NoEchoOldServer) +_test(NoFlusherIfSendAsap) +_test(NoPartialOnReconnect) +_test(NoResponders) +_test(OldRequest) +_test(OpenCloseAndWait) +_test(ParseINFO) +_test(ParserErr) +_test(ParserOK) +_test(ParserPing) +_test(ParserShouldFail) +_test(ParserSplitMsg) +_test(ParseStateReconnectFunctionality) +_test(PendingLimitsDeliveredAndDropped) +_test(PendingLimitsWithSyncSub) +_test(PermViolation) +_test(PingReconnect) +_test(ProcessMsgArgs) +_test(ProperFalloutAfterMaxAttempts) +_test(ProperReconnectDelay) +_test(PublishMsg) +_test(PubSubWithReply) +_test(QueueSubscriber) +_test(QueueSubsOnReconnect) +_test(ReceiveINFORightAfterFirstPONG) +_test(ReconnectAllowedFlags) +_test(ReconnectBufSize) +_test(ReconnectDisallowedFlags) +_test(ReconnectFailsPendingRequest) +_test(ReconnectImplicitUserInfo) +_test(ReconnectJitter) +_test(ReconnectServerStats) +_test(ReconnectThreadLeak) +_test(ReconnectTotalTime) +_test(ReconnectWithTokenHandler) +_test(ReleaseFlush) +_test(ReplyArg) +_test(Request) +_test(RequestClose) +_test(RequestMuxWithMappedSubject) +_test(RequestNoBody) +_test(RequestPool) +_test(RequestTimeout) +_test(RetryOnFailedConnect) +_test(SelectNextServer) +_test(ServerErrorClosesConnection) +_test(ServerPoolUpdatedOnClusterUpdate) +_test(ServersOption) +_test(ServersRandomize) +_test(ServerStopDisconnectedCB) +_test(SimplePublish) +_test(SimplePublishNoData) +_test(SimplifiedURLs) +_test(SimultaneousRequest) +_test(SlowAsyncSubscriber) +_test(SlowConsumerCB) +_test(SlowSubscriber) +_test(SSLAvailable) +_test(SSLBasic) +_test(SSLCertAndKeyFromMemory) +_test(SSLCiphers) +_test(SSLConnectVerboseOption) +_test(SSLLoadCAFromMemory) +_test(SSLMultithreads) +_test(SSLReconnectWithAuthError) +_test(SSLSkipServerVerification) +_test(SSLSocketLeakWithEventLoop) +_test(SSLVerify) +_test(SSLVerifyHostname) +_test(StaleConnection) +_test(Stats) +_test(StopReconnectAfterTwoAuthErr) +_test(SubBadSubjectAndQueueName) +_test(SubOnComplete) +_test(SubRemovedWhileProcessingMsg) +_test(SyncReplyArg) +_test(SyncSubscribe) +_test(SyncSubscriptionPending) +_test(SyncSubscriptionPendingDrain) +_test(TimeoutOnNoServer) +_test(Unsubscribe) +_test(UseDefaultURLIfNoServerSpecified) +_test(UserCredsCallbacks) +_test(UserCredsFromFiles) +_test(UserCredsFromMemory) +_test(Version) +_test(VersionMatchesTag) +_test(WriteDeadline) diff --git a/test/test.c b/test/test.c index 12a92c4d5..614e8b4da 100644 --- a/test/test.c +++ b/test/test.c @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "natsp.h" +#include "test.h" #include #include @@ -33,51 +33,67 @@ #include "sub.h" #include "msg.h" #include "stats.h" -#include "comsock.h" #include "crypto.h" #include "nkeys.h" #include "parser.h" #include "js.h" #include "kv.h" #include "microp.h" + #if defined(NATS_HAS_STREAMING) + #include "stan/conn.h" #include "stan/pub.h" #include "stan/sub.h" #include "stan/copts.h" #include "stan/sopts.h" -#endif + +static const char *clientName = "client"; + +#endif // NATS_HAS_STREAMING + +typedef void (*testFunc)(void); + +typedef struct __testInfo +{ + const char *name; + testFunc func; +} testInfo; + +#define _TEST_PROTO +#include "list.h" +#undef _TEST_PROTO + +#define _TEST_LIST +testInfo allTests[] = +{ +#include "list.h" +}; +#undef _TEST_LIST + static int tests = 0; -static bool failed = false; +bool failed = false; -static bool keepServerOutput = false; +bool keepServerOutput = false; static bool valgrind = false; static bool runOnTravis = false; -static const char *natsServerExe = "nats-server"; static const char *serverVersion = NULL; static const char *natsStreamingServerExe = "nats-streaming-server"; -static natsMutex *slMu = NULL; -static natsHash *slMap = NULL; +natsMutex *slMu = NULL; +natsHash *slMap = NULL; #define test(s) { printf("#%02d ", ++tests); printf("%s", (s)); fflush(stdout); } #ifdef _WIN32 -#define NATS_INVALID_PID (NULL) #define testCond(c) if(c) { printf("PASSED\n"); fflush(stdout); } else { printf("FAILED\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; return; } #define testCondNoReturn(c) if(c) { printf("PASSED\n"); fflush(stdout); } else { printf("FAILED\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; } -#define LOGFILE_NAME "wserver.log" #else -#define NATS_INVALID_PID (-1) #define testCond(c) if(c) { printf("\033[0;32mPASSED\033[0;0m\n"); fflush(stdout); } else { printf("\033[0;31mFAILED\033[0;0m\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; return; } #define testCondNoReturn(c) if(c) { printf("\033[0;32mPASSED\033[0;0m\n"); fflush(stdout); } else { printf("\033[0;31mFAILED\033[0;0m\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; } -#define LOGFILE_NAME "server.log" #endif -#define FAIL(m) { printf("@@ %s @@\n", (m)); failed=true; return; } - -#define CHECK_SERVER_STARTED(p) if ((p) == NATS_INVALID_PID) FAIL("Unable to start or verify that the server was started!") static const char *testServers[] = {"nats://127.0.0.1:1222", "nats://127.0.0.1:1223", @@ -87,11 +103,6 @@ static const char *testServers[] = {"nats://127.0.0.1:1222", "nats://127.0.0.1:1227", "nats://127.0.0.1:1228"}; -#if defined(NATS_HAS_STREAMING) -static const char *clusterName = "test-cluster"; -static const char *clientName = "client"; -#endif - // Forward declaration static void _startMockupServerThread(void *closure); static void _createConfFile(char *buf, int bufLen, const char *content); @@ -221,8 +232,7 @@ _destroyDefaultThreadArgs(struct threadArg *args) natsCondition_Destroy(args->c); } -static void -test_natsNowAndSleep(void) +void test_natsNowAndSleep(void) { int64_t start; int64_t end; @@ -234,8 +244,7 @@ test_natsNowAndSleep(void) testCond(((end - start) >= 990) && ((end - start) <= 1010)); } -static void -test_natsAllocSprintf(void) +void test_natsAllocSprintf(void) { char smallStr[20]; char mediumStr[256]; // This is the size of the temp buffer in nats_asprintf @@ -277,8 +286,7 @@ test_natsAllocSprintf(void) ptr = NULL; } -static void -test_natsStrCaseStr(void) +void test_natsStrCaseStr(void) { const char *s1 = "Hello World!"; const char *s2 = "wo"; @@ -301,8 +309,7 @@ test_natsStrCaseStr(void) testCond(res == NULL); } -static void -test_natsSnprintf(void) +void test_natsSnprintf(void) { #if _WIN32 // This test is specific to older version of Windows @@ -318,7 +325,7 @@ test_natsSnprintf(void) #endif } -static void test_natsBuffer(void) +void test_natsBuffer(void) { natsStatus s; char backend[10]; @@ -595,8 +602,7 @@ static void test_natsBuffer(void) buf = NULL; } -static void -test_natsParseInt64(void) +void test_natsParseInt64(void) { int64_t n; @@ -657,8 +663,7 @@ test_natsParseInt64(void) testCond(n == -1); } -static void -test_natsParseControl(void) +void test_natsParseControl(void) { natsStatus s; natsControl c; @@ -734,8 +739,7 @@ test_natsParseControl(void) c.args = NULL; } -static void -test_natsNormalizeErr(void) +void test_natsNormalizeErr(void) { char error[1024]; char expected[256]; @@ -783,8 +787,7 @@ test_natsNormalizeErr(void) testCond(error[0] == '\0'); } -static void -test_natsMutex(void) +void test_natsMutex(void) { natsStatus s; natsMutex *m = NULL; @@ -842,8 +845,7 @@ sumThread(void *arg) static int NUM_THREADS = 1000; -static void -test_natsThread(void) +void test_natsThread(void) { natsStatus s = NATS_OK; natsMutex *m = NULL; @@ -967,8 +969,7 @@ _unblockLongWait(void *closure) natsMutex_Unlock(args->m); } -static void -test_natsCondition(void) +void test_natsCondition(void) { natsStatus s; natsMutex *m = NULL; @@ -1184,8 +1185,7 @@ _timerStopCB(natsTimer *timer, void *arg) natsCondition_Wait(tArg.c, tArg.m); \ natsMutex_Unlock(tArg.m) -static void -test_natsTimer(void) +void test_natsTimer(void) { natsStatus s; natsTimer *t = NULL; @@ -1405,8 +1405,7 @@ test_natsTimer(void) testCond(s == NATS_OK); } -static void -test_natsUrl(void) +void test_natsUrl(void) { natsStatus s; natsUrl *u = NULL; @@ -1815,8 +1814,7 @@ test_natsUrl(void) && (strstr(nats_GetLastError(NULL), "invalid port '2147483648'") != NULL)); } -static void -test_natsCreateStringFromBuffer(void) +void test_natsCreateStringFromBuffer(void) { natsStatus s = NATS_OK; natsBuffer buf; @@ -1888,8 +1886,7 @@ _testInbox(void *closure) args->status = s; } -static void -test_natsInbox(void) +void test_natsInbox(void) { natsStatus s = NATS_OK; natsThread *threads[INBOX_THREADS_COUNT]; @@ -1956,8 +1953,7 @@ test_natsInbox(void) static int HASH_ITER = 10000000; -static void -test_natsHashing(void) +void test_natsHashing(void) { const char *keys[] = {"foo", "bar", @@ -2005,8 +2001,7 @@ test_natsHashing(void) testCond((s == NATS_OK) && ((end - start) < 1000)); } -static void -test_natsHash(void) +void test_natsHash(void) { natsStatus s; natsHash *hash = NULL; @@ -2271,8 +2266,7 @@ test_natsHash(void) natsHash_Destroy(hash); } -static void -test_natsStrHash(void) +void test_natsStrHash(void) { natsStatus s; natsStrHash *hash = NULL; @@ -2588,8 +2582,7 @@ _dummySigCb(char **customErrTxt, unsigned char **psig, int *sigLen, const char* return NATS_OK; } -static void -test_natsOptions(void) +void test_natsOptions(void) { natsStatus s; natsOptions *opts = NULL; @@ -3145,8 +3138,7 @@ test_natsOptions(void) natsOptions_Destroy(opts); } -static void -test_natsSock_ReadLine(void) +void test_natsSock_ReadLine(void) { char buffer[20]; natsStatus s; @@ -3183,8 +3175,7 @@ _dummyJSONCb(void *userInfo, const char *fieldName, nats_JSONField *f) return NATS_OK; } -static void -test_natsJSON(void) +void test_natsJSON(void) { natsStatus s; nats_JSON *json = NULL; @@ -4232,8 +4223,7 @@ test_natsJSON(void) } -static void -test_natsEncodeTimeUTC(void) +void test_natsEncodeTimeUTC(void) { natsStatus s; char buf[36] = {'\0'}; @@ -4282,8 +4272,7 @@ test_natsEncodeTimeUTC(void) } } -static void -test_natsErrWithLongText(void) +void test_natsErrWithLongText(void) { natsStatus s; char errTxt[300]; @@ -4321,8 +4310,7 @@ test_natsErrWithLongText(void) nats_clearLastError(); } -static void -test_natsErrStackMoreThanMaxFrames(void) +void test_natsErrStackMoreThanMaxFrames(void) { int i; const int total = MAX_FRAMES+10; @@ -4375,8 +4363,7 @@ test_natsErrStackMoreThanMaxFrames(void) testCond(s == NATS_OK); } -static void -test_natsMsg(void) +void test_natsMsg(void) { natsMsg *msg = NULL; natsStatus s = NATS_OK; @@ -4412,8 +4399,7 @@ test_natsMsg(void) natsMsg_Destroy(msg); } -static void -test_natsBase32Decode(void) +void test_natsBase32Decode(void) { natsStatus s; const char *src = "KRUGS4ZANFZSA5DIMUQHEZLTOVWHIIDPMYQGCIDCMFZWKMZSEBSGKY3PMRUW4ZY"; @@ -4439,8 +4425,7 @@ test_natsBase32Decode(void) && (strstr(nats_GetLastError(NULL), "invalid") != NULL)); } -static void -test_natsBase64Encode(void) +void test_natsBase64Encode(void) { natsStatus s; char *enc = NULL; @@ -4586,8 +4571,7 @@ test_natsBase64Encode(void) free(dec); } -static void -test_natsCRC16(void) +void test_natsCRC16(void) { unsigned char a[] = {153, 209, 36, 74, 103, 32, 65, 34, 111, 68, 104, 156, 50, 14, 164, 140, 144, 230}; uint16_t crc = 0; @@ -4605,8 +4589,7 @@ test_natsCRC16(void) testCond(!nats_CRC16_Validate(a, (int)sizeof(a), expected)); } -static void -test_natsKeys(void) +void test_natsKeys(void) { natsStatus s; unsigned char sig[NATS_CRYPTO_SIGN_BYTES]; @@ -4662,8 +4645,7 @@ test_natsKeys(void) && (memcmp(sig, expected, sizeof(expected)) == 0)); } -static void -test_natsReadFile(void) +void test_natsReadFile(void) { natsStatus s = NATS_OK; natsBuffer *buf = NULL; @@ -4757,8 +4739,7 @@ test_natsReadFile(void) remove(fn); } -static void -test_natsGetJWTOrSeed(void) +void test_natsGetJWTOrSeed(void) { natsStatus s; char *val = NULL; @@ -4804,8 +4785,7 @@ test_natsGetJWTOrSeed(void) } } -static void -test_natsHostIsIP(void) +void test_natsHostIsIP(void) { struct _testHost { const char *host; @@ -4896,8 +4876,7 @@ _testSockShutdownThread(void *closure) natsSock_Shutdown(ctx->fd); } -static void -test_natsWaitReady(void) +void test_natsWaitReady(void) { natsStatus s = NATS_OK; natsThread *t = NULL; @@ -4986,8 +4965,7 @@ test_natsWaitReady(void) _destroyDefaultThreadArgs(&arg); } -static void -test_natsSign(void) +void test_natsSign(void) { unsigned char *sig = NULL; int sigLen = 0; @@ -5076,8 +5054,7 @@ _testStatus(const char *testName, char *buf, const char *expectedStatus, const c natsMsg_Destroy(msg); } -static void -test_natsMsgHeadersLift(void) +void test_HeadersLift(void) { char buf[512]; @@ -5138,8 +5115,7 @@ test_natsMsgHeadersLift(void) _testStatus("Status with description (extra spaces): ", buf, "404", "No Messages"); } -static void -test_natsMsgHeaderAPIs(void) +void test_natsMsgHeaderAPIs(void) { natsStatus s = NATS_OK; natsMsg *msg = NULL; @@ -5367,8 +5343,7 @@ test_natsMsgHeaderAPIs(void) natsMsg_Destroy(msg); } -static void -test_natsMsgIsJSCtrl(void) +void test_natsMsgIsJSCtrl(void) { struct testCase { const char *buf; @@ -5426,8 +5401,7 @@ test_natsMsgIsJSCtrl(void) } } -static void -test_natsSrvVersionAtLeast(void) +void test_natsSrvVersionAtLeast(void) { natsOptions *opts = NULL; natsConnection *nc = NULL; @@ -5497,8 +5471,7 @@ test_natsSrvVersionAtLeast(void) natsConnection_Destroy(nc); } -static void -test_natsFormatStringArray(void) +void test_natsFormatStringArray(void) { natsStatus s; size_t i, N; @@ -5532,331 +5505,13 @@ test_natsFormatStringArray(void) NATS_FREE_STRINGS(out, N); } -static natsStatus -_checkStart(const char *url, int orderIP, int maxAttempts) -{ - natsStatus s = NATS_OK; - natsUrl *nUrl = NULL; - int attempts = 0; - natsSockCtx ctx; - - natsSock_Init(&ctx); - ctx.orderIP = orderIP; - - natsDeadline_Init(&(ctx.writeDeadline), 2000); - - s = natsUrl_Create(&nUrl, url); - if (s == NATS_OK) - { - while (((s = natsSock_ConnectTcp(&ctx, - nUrl->host, nUrl->port)) != NATS_OK) - && (attempts++ < maxAttempts)) - { - nats_Sleep(200); - } - - natsUrl_Destroy(nUrl); - - if (s == NATS_OK) - natsSock_Close(ctx.fd); - else - s = NATS_NO_SERVER; - } - - nats_clearLastError(); - - return s; -} - -static natsStatus -_checkStreamingStart(const char *url, int maxAttempts) -{ - natsStatus s = NATS_NOT_PERMITTED; - -#if defined(NATS_HAS_STREAMING) - - stanConnOptions *opts = NULL; - stanConnection *sc = NULL; - int attempts = 0; - - s = stanConnOptions_Create(&opts); - IFOK(s, stanConnOptions_SetURL(opts, url)); - IFOK(s, stanConnOptions_SetConnectionWait(opts, 250)); - if (s == NATS_OK) - { - while (((s = stanConnection_Connect(&sc, clusterName, "checkStart", opts)) != NATS_OK) - && (attempts++ < maxAttempts)) - { - nats_Sleep(200); - } - } - - stanConnection_Destroy(sc); - stanConnOptions_Destroy(opts); - - if (s != NATS_OK) - nats_clearLastError(); -#else -#endif - return s; -} - -#ifdef _WIN32 - -typedef PROCESS_INFORMATION *natsPid; - -static HANDLE logHandle = NULL; - -static void -_stopServer(natsPid pid) -{ - if (pid == NATS_INVALID_PID) - return; - - TerminateProcess(pid->hProcess, 0); - WaitForSingleObject(pid->hProcess, INFINITE); - - CloseHandle(pid->hProcess); - CloseHandle(pid->hThread); - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Remove(slMap, (int64_t) pid); - natsMutex_Unlock(slMu); - - free(pid); -} - -static natsPid -_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) -{ - SECURITY_ATTRIBUTES sa; - STARTUPINFO si; - HANDLE h; - PROCESS_INFORMATION *pid; - DWORD flags = 0; - BOOL createdOk = FALSE; - BOOL hInheritance = FALSE; - char *exeAndCmdLine = NULL; - int ret; - - pid = calloc(1, sizeof(PROCESS_INFORMATION)); - if (pid == NULL) - return NATS_INVALID_PID; - - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - - ret = nats_asprintf(&exeAndCmdLine, "%s%s%s", serverExe, - (cmdLineOpts != NULL ? " " : ""), - (cmdLineOpts != NULL ? cmdLineOpts : "")); - if (ret < 0) - { - printf("No memory allocating command line string!\n"); - free(pid); - return NATS_INVALID_PID; - } - - if (!keepServerOutput) - { - ZeroMemory(&sa, sizeof(sa)); - sa.nLength = sizeof(sa); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = TRUE; - - h = logHandle; - if (h == NULL) - { - h = CreateFile(LOGFILE_NAME, - GENERIC_WRITE, - FILE_SHARE_WRITE | FILE_SHARE_READ, - &sa, - CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, - NULL); - } - - si.dwFlags |= STARTF_USESTDHANDLES; - si.hStdInput = NULL; - si.hStdError = h; - si.hStdOutput = h; - - hInheritance = TRUE; - flags = CREATE_NO_WINDOW; - - if (logHandle == NULL) - logHandle = h; - } - - // Start the child process. - if (!CreateProcess(NULL, - (LPSTR) exeAndCmdLine, - NULL, // Process handle not inheritable - NULL, // Thread handle not inheritable - hInheritance, // Set handle inheritance - flags, // Creation flags - NULL, // Use parent's environment block - NULL, // Use parent's starting directory - &si, // Pointer to STARTUPINFO structure - pid)) // Pointer to PROCESS_INFORMATION structure - { - - printf("Unable to start '%s': error (%d).\n", - exeAndCmdLine, GetLastError()); - free(exeAndCmdLine); - return NATS_INVALID_PID; - } - - free(exeAndCmdLine); - - if (checkStart) - { - natsStatus s; - - if (strcmp(serverExe, natsServerExe) == 0) - s = _checkStart(url, 46, 10); - else - s = _checkStreamingStart(url, 10); - - if (s != NATS_OK) - { - _stopServer(pid); - return NATS_INVALID_PID; - } - } - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Set(slMap, (int64_t) pid, NULL, NULL); - natsMutex_Unlock(slMu); - - return (natsPid) pid; -} - -#else - -typedef pid_t natsPid; - -static void -_stopServer(natsPid pid) -{ - int status = 0; - - if (pid == NATS_INVALID_PID) - return; - - if (kill(pid, SIGINT) < 0) - { - perror("kill with SIGINT"); - if (kill(pid, SIGKILL) < 0) - { - perror("kill with SIGKILL"); - } - } - - waitpid(pid, &status, 0); - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Remove(slMap, (int64_t) pid); - natsMutex_Unlock(slMu); -} - -static natsPid -_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) -{ - natsPid pid = fork(); - if (pid == -1) - { - perror("fork"); - return NATS_INVALID_PID; - } - - if (pid == 0) - { - char *exeAndCmdLine = NULL; - char *argvPtrs[64]; - char *line = NULL; - int index = 0; - int ret = 0; - bool overrideAddr = false; - - if ((cmdLineOpts == NULL) || (strstr(cmdLineOpts, "-a ") == NULL)) - overrideAddr = true; - - ret = nats_asprintf(&exeAndCmdLine, "%s%s%s%s%s", serverExe, - (cmdLineOpts != NULL ? " " : ""), - (cmdLineOpts != NULL ? cmdLineOpts : ""), - (overrideAddr ? " -a 127.0.0.1" : ""), - (keepServerOutput ? "" : " -l " LOGFILE_NAME)); - if (ret < 0) - { - perror("No memory allocating command line string!\n"); - exit(1); - } - - memset(argvPtrs, 0, sizeof(argvPtrs)); - line = exeAndCmdLine; - - while (*line != '\0') - { - while ((*line == ' ') || (*line == '\t') || (*line == '\n')) - *line++ = '\0'; - - argvPtrs[index++] = line; - while ((*line != '\0') && (*line != ' ') - && (*line != '\t') && (*line != '\n')) - { - line++; - } - } - argvPtrs[index++] = NULL; - - // Child process. Replace with NATS server - execvp(argvPtrs[0], argvPtrs); - perror("Exec failed: "); - exit(1); - } - else if (checkStart) - { - natsStatus s; - - if (strcmp(serverExe, natsServerExe) == 0) - s = _checkStart(url, 46, 10); - else - s = _checkStreamingStart(url, 10); - - if (s != NATS_OK) - { - _stopServer(pid); - return NATS_INVALID_PID; - } - } - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Set(slMap, (int64_t) pid, NULL, NULL); - natsMutex_Unlock(slMu); - - // parent, return the child's PID back. - return pid; -} -#endif - -static natsPid -_startServer(const char *url, const char *cmdLineOpts, bool checkStart) -{ - return _startServerImpl(natsServerExe, url, cmdLineOpts, checkStart); -} - static natsPid _startStreamingServer(const char* url, const char *cmdLineOpts, bool checkStart) { return _startServerImpl(natsStreamingServerExe, url, cmdLineOpts, checkStart); } -static void -test_natsSock_IPOrder(void) +void test_natsSock_IPOrder(void) { natsStatus s; natsPid serverPid; @@ -5914,8 +5569,7 @@ test_natsSock_IPOrder(void) } } -static void -test_natsSock_ConnectTcp(void) +void test_natsSock_ConnectTcp(void) { natsPid serverPid = NATS_INVALID_PID; @@ -5954,8 +5608,7 @@ listOrder(struct addrinfo *head, bool ordered) return true; } -static void -test_natsSock_ShuffleIPs(void) +void test_natsSock_ShuffleIPs(void) { struct addrinfo *tmp[10]; struct addrinfo *head = NULL; @@ -6073,8 +5726,7 @@ _reconnectedCb(natsConnection *nc, void *closure) natsMutex_Unlock(arg->m); } -static void -test_ReconnectServerStats(void) +void test_ReconnectServerStats(void) { natsStatus s; natsConnection *nc = NULL; @@ -6308,8 +5960,7 @@ _waitForConnClosed(struct threadArg *arg) return s; } -static void -test_ParseStateReconnectFunctionality(void) +void test_ParseStateReconnectFunctionality(void) { natsStatus s; natsConnection *nc = NULL; @@ -6396,8 +6047,7 @@ test_ParseStateReconnectFunctionality(void) _stopServer(serverPid); } -static void -test_ServersRandomize(void) +void test_ServersRandomize(void) { natsStatus s; natsOptions *opts = NULL; @@ -6531,8 +6181,7 @@ test_ServersRandomize(void) _stopServer(pid); } -static void -test_SelectNextServer(void) +void test_SelectNextServer(void) { natsStatus s; natsOptions *opts = NULL; @@ -6659,8 +6308,7 @@ parserNegTest(int lineNum) #define PARSER_START_TEST parserNegTest(__LINE__) -static void -test_ParserPing(void) +void test_ParserPing(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6725,8 +6373,7 @@ test_ParserPing(void) natsConnection_Destroy(nc); } -static void -test_ParserErr(void) +void test_ParserErr(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6812,8 +6459,7 @@ test_ParserErr(void) natsConnection_Destroy(nc); } -static void -test_ParserOK(void) +void test_ParserOK(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6850,8 +6496,7 @@ test_ParserOK(void) natsConnection_Destroy(nc); } -static void -test_ParseINFO(void) +void test_ParseINFO(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6882,8 +6527,7 @@ test_ParseINFO(void) natsConnection_Destroy(nc); } -static void -test_ParserShouldFail(void) +void test_ParserShouldFail(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7040,8 +6684,7 @@ test_ParserShouldFail(void) natsConnection_Destroy(nc); } -static void -test_ParserSplitMsg(void) +void test_ParserSplitMsg(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7208,8 +6851,7 @@ test_ParserSplitMsg(void) if (s != NATS_OK) \ FAIL("Unable to setup test"); \ -static void -test_ProcessMsgArgs(void) +void test_ProcessMsgArgs(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7582,8 +7224,7 @@ checkNewURLsAddedRandomly(natsConnection *nc, char **urlsAfterPoolSetup, int ini return s; } -static void -test_AsyncINFO(void) +void test_AsyncINFO(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7820,8 +7461,7 @@ _parallelRequests(void *closure) natsConnection_RequestString(&msg, nc, "foo", "test", 500); } -static void -test_RequestPool(void) +void test_RequestPool(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -7884,8 +7524,7 @@ test_RequestPool(void) _stopServer(pid); } -static void -test_NoFlusherIfSendAsap(void) +void test_NoFlusherIfSendAsap(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -7953,8 +7592,7 @@ test_NoFlusherIfSendAsap(void) _stopServer(pid); } -static void -test_HeadersAndSubPendingBytes(void) +void test_HeadersAndSubPendingBytes(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -8033,8 +7671,7 @@ _dummyMsgHandler(natsConnection *nc, natsSubscription *sub, natsMsg *msg, natsMsg_Destroy(msg); } -static void -test_LibMsgDelivery(void) +void test_LibMsgDelivery(void) { natsStatus s; natsPid serverPid = NATS_INVALID_PID; @@ -8178,8 +7815,7 @@ test_LibMsgDelivery(void) nats_Open(-1); } -static void -test_DefaultConnection(void) +void test_DefaultConnection(void) { natsStatus s; natsConnection *nc = NULL; @@ -8226,8 +7862,7 @@ test_DefaultConnection(void) _stopServer(serverPid); } -static void -test_SimplifiedURLs(void) +void test_SimplifiedURLs(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -8288,8 +7923,7 @@ test_SimplifiedURLs(void) #endif } -static void -test_IPResolutionOrder(void) +void test_IPResolutionOrder(void) { natsStatus s; natsConnection *nc = NULL; @@ -8445,8 +8079,7 @@ test_IPResolutionOrder(void) natsOptions_Destroy(opts); } -static void -test_UseDefaultURLIfNoServerSpecified(void) +void test_UseDefaultURLIfNoServerSpecified(void) { natsStatus s; natsOptions *opts = NULL; @@ -8470,8 +8103,7 @@ test_UseDefaultURLIfNoServerSpecified(void) _stopServer(serverPid); } -static void -test_ConnectToWithMultipleURLs(void) +void test_ConnectToWithMultipleURLs(void) { natsStatus s; natsConnection *nc = NULL; @@ -8502,8 +8134,7 @@ test_ConnectToWithMultipleURLs(void) _stopServer(serverPid); } -static void -test_ConnectionToWithNullURLs(void) +void test_ConnectionToWithNullURLs(void) { natsStatus s; natsConnection *nc = NULL; @@ -8524,8 +8155,7 @@ test_ConnectionToWithNullURLs(void) _stopServer(serverPid); } -static void -test_ConnectionWithNullOptions(void) +void test_ConnectionWithNullOptions(void) { natsStatus s; natsConnection *nc = NULL; @@ -8543,8 +8173,7 @@ test_ConnectionWithNullOptions(void) _stopServer(serverPid); } -static void -test_ConnectionStatus(void) +void test_ConnectionStatus(void) { natsStatus s; natsConnection *nc = NULL; @@ -8570,8 +8199,7 @@ test_ConnectionStatus(void) _stopServer(serverPid); } -static void -test_ConnClosedCB(void) +void test_ConnClosedCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -8615,8 +8243,7 @@ test_ConnClosedCB(void) _stopServer(serverPid); } -static void -test_CloseDisconnectedCB(void) +void test_CloseDisconnectedCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -8661,8 +8288,7 @@ test_CloseDisconnectedCB(void) _stopServer(serverPid); } -static void -test_ServerStopDisconnectedCB(void) +void test_ServerStopDisconnectedCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -8705,8 +8331,7 @@ test_ServerStopDisconnectedCB(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ClosedConnections(void) +void test_ClosedConnections(void) { natsStatus s; natsConnection *nc = NULL; @@ -8775,8 +8400,7 @@ test_ClosedConnections(void) _stopServer(serverPid); } -static void -test_ConnectVerboseOption(void) +void test_ConnectVerboseOption(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -8829,8 +8453,7 @@ test_ConnectVerboseOption(void) _stopServer(serverPid); } -static void -test_ReconnectThreadLeak(void) +void test_ReconnectThreadLeak(void) { natsStatus s; natsOptions *opts = NULL; @@ -8891,8 +8514,7 @@ test_ReconnectThreadLeak(void) _stopServer(serverPid); } -static void -test_ReconnectTotalTime(void) +void test_ReconnectTotalTime(void) { natsStatus s; natsOptions *opts = NULL; @@ -8905,8 +8527,7 @@ test_ReconnectTotalTime(void) natsOptions_Destroy(opts); } -static void -test_ReconnectDisallowedFlags(void) +void test_ReconnectDisallowedFlags(void) { natsStatus s; natsConnection *nc = NULL; @@ -8941,8 +8562,7 @@ test_ReconnectDisallowedFlags(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ReconnectAllowedFlags(void) +void test_ReconnectAllowedFlags(void) { natsStatus s; natsConnection *nc = NULL; @@ -8999,8 +8619,7 @@ _closeConn(void *arg) natsConnection_Close(nc); } -static void -test_ConnCloseBreaksReconnectLoop(void) +void test_ConnCloseBreaksReconnectLoop(void) { natsStatus s; natsConnection *nc = NULL; @@ -9068,8 +8687,7 @@ test_ConnCloseBreaksReconnectLoop(void) _destroyDefaultThreadArgs(&arg); } -static void -test_BasicReconnectFunctionality(void) +void test_BasicReconnectFunctionality(void) { natsStatus s; natsConnection *nc = NULL; @@ -9155,8 +8773,7 @@ _doneCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure) natsMsg_Destroy(msg); } -static void -test_ExtendedReconnectFunctionality(void) +void test_ExtendedReconnectFunctionality(void) { natsStatus s; natsConnection *nc = NULL; @@ -9261,8 +8878,7 @@ test_ExtendedReconnectFunctionality(void) _stopServer(serverPid); } -static void -test_QueueSubsOnReconnect(void) +void test_QueueSubsOnReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -9373,8 +8989,7 @@ test_QueueSubsOnReconnect(void) _stopServer(serverPid); } -static void -test_IsClosed(void) +void test_IsClosed(void) { natsStatus s; natsConnection *nc = NULL; @@ -9408,8 +9023,7 @@ test_IsClosed(void) _stopServer(serverPid); } -static void -test_IsReconnectingAndStatus(void) +void test_IsReconnectingAndStatus(void) { natsStatus s; natsConnection *nc = NULL; @@ -9495,8 +9109,7 @@ test_IsReconnectingAndStatus(void) _stopServer(serverPid); } -static void -test_ReconnectBufSize(void) +void test_ReconnectBufSize(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -9603,8 +9216,7 @@ _testCustomReconnectDelayOnInitialConnect(natsConnection *nc, int attempts, void return 50; } -static void -test_RetryOnFailedConnect(void) +void test_RetryOnFailedConnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -9971,8 +9583,7 @@ _startMockupServer(natsSock *serverSock, const char *host, const char *port) return s; } -static void -test_ErrOnConnectAndDeadlock(void) +void test_ErrOnConnectAndDeadlock(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -10037,8 +9648,7 @@ test_ErrOnConnectAndDeadlock(void) natsSock_Close(sock); } -static void -test_ErrOnMaxPayloadLimit(void) +void test_ErrOnMaxPayloadLimit(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -10115,8 +9725,7 @@ test_ErrOnMaxPayloadLimit(void) _destroyDefaultThreadArgs(&arg); } -static void -test_Auth(void) +void test_Auth(void) { natsStatus s; natsConnection *nc = NULL; @@ -10165,8 +9774,7 @@ test_Auth(void) _stopServer(serverPid); } -static void -test_AuthFailNoDisconnectCB(void) +void test_AuthFailNoDisconnectCB(void) { natsStatus s; natsOptions *opts = NULL; @@ -10206,8 +9814,7 @@ test_AuthFailNoDisconnectCB(void) _stopServer(serverPid); } -static void -test_AuthToken(void) +void test_AuthToken(void) { natsStatus s; natsConnection *nc = NULL; @@ -10257,8 +9864,7 @@ _tokenHandler(void* closure) return (char*) closure; } -static void -test_AuthTokenHandler(void) +void test_AuthTokenHandler(void) { natsStatus s; natsConnection *nc = NULL; @@ -10328,8 +9934,7 @@ _permsViolationHandler(natsConnection *nc, natsSubscription *sub, natsStatus err } } -static void -test_PermViolation(void) +void test_PermViolation(void) { natsStatus s; natsConnection *conn = NULL; @@ -10424,8 +10029,7 @@ _authViolationHandler(natsConnection *nc, natsSubscription *sub, natsStatus err, } } -static void -test_AuthViolation(void) +void test_AuthViolation(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -10628,8 +10232,7 @@ _authExpiredHandler(natsConnection *nc, natsSubscription *sub, natsStatus err, v } } -static void -test_AuthenticationExpired(void) +void test_AuthenticationExpired(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -10766,8 +10369,7 @@ _startServerSendErr2Thread(void *closure) natsSock_Close(sock); } -static void -test_AuthenticationExpiredReconnect(void) +void test_AuthenticationExpiredReconnect(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -10855,8 +10457,7 @@ test_AuthenticationExpiredReconnect(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ConnectedServer(void) +void test_ConnectedServer(void) { natsStatus s; natsConnection *nc = NULL; @@ -10899,8 +10500,7 @@ test_ConnectedServer(void) _stopServer(serverPid); } -static void -test_MultipleClose(void) +void test_MultipleClose(void) { natsStatus s; natsConnection *nc = NULL; @@ -10929,8 +10529,7 @@ test_MultipleClose(void) _stopServer(serverPid); } -static void -test_SimplePublish(void) +void test_SimplePublish(void) { natsStatus s; natsConnection *nc = NULL; @@ -10951,8 +10550,7 @@ test_SimplePublish(void) _stopServer(serverPid); } -static void -test_SimplePublishNoData(void) +void test_SimplePublishNoData(void) { natsStatus s; natsConnection *nc = NULL; @@ -10973,8 +10571,7 @@ test_SimplePublishNoData(void) _stopServer(serverPid); } -static void -test_PublishMsg(void) +void test_PublishMsg(void) { natsStatus s; natsConnection *nc = NULL; @@ -11027,8 +10624,7 @@ test_PublishMsg(void) _destroyDefaultThreadArgs(&arg); } -static void -test_InvalidSubsArgs(void) +void test_InvalidSubsArgs(void) { natsStatus s; natsConnection *nc = NULL; @@ -11191,8 +10787,7 @@ test_InvalidSubsArgs(void) _stopServer(serverPid); } -static void -test_AsyncSubscribe(void) +void test_AsyncSubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11314,8 +10909,7 @@ _asyncTimeoutCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *c natsMutex_Unlock(ai->arg->m); } -static void -test_AsyncSubscribeTimeout(void) +void test_AsyncSubscribeTimeout(void) { natsStatus s; natsConnection *nc = NULL; @@ -11421,8 +11015,7 @@ test_AsyncSubscribeTimeout(void) } } -static void -test_SyncSubscribe(void) +void test_SyncSubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11450,8 +11043,7 @@ test_SyncSubscribe(void) _stopServer(serverPid); } -static void -test_PubSubWithReply(void) +void test_PubSubWithReply(void) { natsStatus s; natsConnection *nc = NULL; @@ -11480,8 +11072,7 @@ test_PubSubWithReply(void) _stopServer(serverPid); } -static void -test_NoResponders(void) +void test_NoResponders(void) { natsStatus s; natsConnection *nc = NULL; @@ -11582,8 +11173,7 @@ _doFlush(void *arg) } } -static void -test_Flush(void) +void test_Flush(void) { natsStatus s; natsOptions *opts = NULL; @@ -11706,8 +11296,7 @@ test_Flush(void) _stopServer(serverPid); } -static void -test_ConnCloseDoesFlush(void) +void test_ConnCloseDoesFlush(void) { natsStatus s = NATS_OK; natsPid pid = NATS_INVALID_PID; @@ -11757,8 +11346,7 @@ test_ConnCloseDoesFlush(void) _stopServer(pid); } -static void -test_QueueSubscriber(void) +void test_QueueSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -11826,8 +11414,7 @@ test_QueueSubscriber(void) _stopServer(serverPid); } -static void -test_ReplyArg(void) +void test_ReplyArg(void) { natsStatus s; natsConnection *nc = NULL; @@ -11869,8 +11456,7 @@ test_ReplyArg(void) _stopServer(serverPid); } -static void -test_SyncReplyArg(void) +void test_SyncReplyArg(void) { natsStatus s; natsConnection *nc = NULL; @@ -11899,8 +11485,7 @@ test_SyncReplyArg(void) _stopServer(serverPid); } -static void -test_Unsubscribe(void) +void test_Unsubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11963,8 +11548,7 @@ test_Unsubscribe(void) _stopServer(serverPid); } -static void -test_DoubleUnsubscribe(void) +void test_DoubleUnsubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11993,8 +11577,7 @@ test_DoubleUnsubscribe(void) _stopServer(serverPid); } -static void -test_SubRemovedWhileProcessingMsg(void) +void test_SubRemovedWhileProcessingMsg(void) { natsStatus s; natsConnection *nc = NULL; @@ -12070,8 +11653,7 @@ test_SubRemovedWhileProcessingMsg(void) _stopServer(serverPid); } -static void -test_RequestTimeout(void) +void test_RequestTimeout(void) { natsStatus s; natsConnection *nc = NULL; @@ -12091,8 +11673,7 @@ test_RequestTimeout(void) _stopServer(serverPid); } -static void -test_Request(void) +void test_Request(void) { natsStatus s; natsConnection *nc = NULL; @@ -12205,8 +11786,7 @@ test_Request(void) _stopServer(serverPid); } -static void -test_RequestNoBody(void) +void test_RequestNoBody(void) { natsStatus s; natsConnection *nc = NULL; @@ -12321,8 +11901,7 @@ _serverForMuxWithMappedSubject(void *closure) natsSock_Close(sock); } -static void -test_RequestMuxWithMappedSubject(void) +void test_RequestMuxWithMappedSubject(void) { natsStatus s; natsConnection *nc = NULL; @@ -12371,8 +11950,7 @@ test_RequestMuxWithMappedSubject(void) _destroyDefaultThreadArgs(&arg); } -static void -test_OldRequest(void) +void test_OldRequest(void) { natsStatus s; natsConnection *nc = NULL; @@ -12456,8 +12034,7 @@ _sendRequest(void *closure) natsMsg_Destroy(msg); } -static void -test_SimultaneousRequest(void) +void test_SimultaneousRequest(void) { natsStatus s; natsConnection *nc = NULL; @@ -12520,8 +12097,7 @@ test_SimultaneousRequest(void) _stopServer(serverPid); } -static void -test_RequestClose(void) +void test_RequestClose(void) { natsStatus s; natsConnection *nc = NULL; @@ -12558,8 +12134,7 @@ test_RequestClose(void) } -static void -test_CustomInbox(void) +void test_CustomInbox(void) { natsStatus s; natsConnection *nc = NULL; @@ -12690,8 +12265,7 @@ test_CustomInbox(void) _stopServer(serverPid); } -static void -test_MessageBufferPadding(void) +void test_MessageBufferPadding(void) { natsStatus s; natsConnection *nc = NULL; @@ -12747,8 +12321,7 @@ test_MessageBufferPadding(void) _stopServer(serverPid); } -static void -test_FlushInCb(void) +void test_FlushInCb(void) { natsStatus s; natsConnection *nc = NULL; @@ -12791,8 +12364,7 @@ test_FlushInCb(void) _stopServer(serverPid); } -static void -test_ReleaseFlush(void) +void test_ReleaseFlush(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -12870,8 +12442,7 @@ test_ReleaseFlush(void) } -static void -test_FlushErrOnDisconnect(void) +void test_FlushErrOnDisconnect(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -12946,8 +12517,7 @@ test_FlushErrOnDisconnect(void) _destroyDefaultThreadArgs(&arg); } -static void -test_Inbox(void) +void test_Inbox(void) { natsStatus s; natsInbox *inbox = NULL; @@ -12961,8 +12531,7 @@ test_Inbox(void) natsInbox_Destroy(inbox); } -static void -test_Stats(void) +void test_Stats(void) { natsStatus s; natsConnection *nc = NULL; @@ -13025,8 +12594,7 @@ test_Stats(void) _stopServer(serverPid); } -static void -test_BadSubject(void) +void test_BadSubject(void) { natsStatus s; natsConnection *nc = NULL; @@ -13051,8 +12619,7 @@ test_BadSubject(void) _stopServer(serverPid); } -static void -test_SubBadSubjectAndQueueName(void) +void test_SubBadSubjectAndQueueName(void) { natsStatus s; natsConnection *nc = NULL; @@ -13141,8 +12708,7 @@ _subComplete(void *closure) natsMutex_Unlock(arg->m); } -static void -test_ClientAsyncAutoUnsub(void) +void test_ClientAsyncAutoUnsub(void) { natsStatus s; natsConnection *nc = NULL; @@ -13217,8 +12783,7 @@ test_ClientAsyncAutoUnsub(void) _stopServer(serverPid); } -static void -test_ClientSyncAutoUnsub(void) +void test_ClientSyncAutoUnsub(void) { natsStatus s; natsConnection *nc = NULL; @@ -13283,8 +12848,7 @@ test_ClientSyncAutoUnsub(void) _stopServer(serverPid); } -static void -test_ClientAutoUnsubAndReconnect(void) +void test_ClientAutoUnsubAndReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -13366,8 +12930,7 @@ _autoUnsub(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closur } -static void -test_AutoUnsubNoUnsubOnDestroy(void) +void test_AutoUnsubNoUnsubOnDestroy(void) { natsStatus s; natsConnection *nc = NULL; @@ -13413,8 +12976,7 @@ test_AutoUnsubNoUnsubOnDestroy(void) natsBuf_Destroy(buf); } -static void -test_NextMsgOnClosedSub(void) +void test_NextMsgOnClosedSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -13450,8 +13012,7 @@ _nextMsgKickedOut(void *closure) (void) natsSubscription_NextMsg(&msg, sub, 10000); } -static void -test_CloseSubRelease(void) +void test_CloseSubRelease(void) { natsStatus s; natsConnection *nc = NULL; @@ -13499,8 +13060,7 @@ test_CloseSubRelease(void) _stopServer(serverPid); } -static void -test_IsValidSubscriber(void) +void test_IsValidSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -13544,8 +13104,7 @@ test_IsValidSubscriber(void) _stopServer(serverPid); } -static void -test_SlowSubscriber(void) +void test_SlowSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -13598,8 +13157,7 @@ test_SlowSubscriber(void) _stopServer(serverPid); } -static void -test_SlowAsyncSubscriber(void) +void test_SlowAsyncSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -13707,8 +13265,7 @@ _slowConsErrCB(natsConnection *nc, natsSubscription *sub, natsStatus err, void * natsMutex_Unlock(arg->m); } -static void -test_SlowConsumerCB(void) +void test_SlowConsumerCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -13766,8 +13323,7 @@ test_SlowConsumerCB(void) _destroyDefaultThreadArgs(&arg); } -static void -test_PendingLimitsDeliveredAndDropped(void) +void test_PendingLimitsDeliveredAndDropped(void) { natsStatus s; natsConnection *nc = NULL; @@ -13996,8 +13552,7 @@ test_PendingLimitsDeliveredAndDropped(void) _stopServer(serverPid); } -static void -test_PendingLimitsWithSyncSub(void) +void test_PendingLimitsWithSyncSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -14064,8 +13619,7 @@ test_PendingLimitsWithSyncSub(void) _stopServer(serverPid); } -static void -test_AsyncSubscriptionPending(void) +void test_AsyncSubscriptionPending(void) { natsStatus s; natsConnection *nc = NULL; @@ -14177,8 +13731,7 @@ test_AsyncSubscriptionPending(void) _stopServer(serverPid); } -static void -test_AsyncSubscriptionPendingDrain(void) +void test_AsyncSubscriptionPendingDrain(void) { natsStatus s; natsConnection *nc = NULL; @@ -14242,8 +13795,7 @@ test_AsyncSubscriptionPendingDrain(void) _stopServer(serverPid); } -static void -test_SyncSubscriptionPending(void) +void test_SyncSubscriptionPending(void) { natsStatus s; natsConnection *nc = NULL; @@ -14328,8 +13880,7 @@ test_SyncSubscriptionPending(void) _stopServer(serverPid); } -static void -test_SyncSubscriptionPendingDrain(void) +void test_SyncSubscriptionPendingDrain(void) { natsStatus s; natsConnection *nc = NULL; @@ -14421,8 +13972,7 @@ _asyncErrCb(natsConnection *nc, natsSubscription *sub, natsStatus err, void* clo natsMutex_Unlock(arg->m); } -static void -test_AsyncErrHandler_MaxPendingMsgs(void) +void test_AsyncErrHandlerMaxPendingMsgs(void) { natsStatus s; natsConnection *nc = NULL; @@ -14484,8 +14034,7 @@ test_AsyncErrHandler_MaxPendingMsgs(void) _stopServer(serverPid); } -static void -test_AsyncErrHandler_MaxPendingBytes(void) +void test_AsyncErrHandlerMaxPendingBytes(void) { natsStatus s; natsConnection* nc = NULL; @@ -14580,8 +14129,7 @@ _asyncErrBlockingCb(natsConnection *nc, natsSubscription *sub, natsStatus err, v natsMutex_Unlock(arg->m); } -static void -test_AsyncErrHandlerSubDestroyed(void) +void test_AsyncErrHandlerSubDestroyed(void) { natsStatus s; natsConnection *nc = NULL; @@ -14716,8 +14264,7 @@ _startCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure) natsMsg_Destroy(msg); } -static void -test_AsyncSubscriberStarvation(void) +void test_AsyncSubscriberStarvation(void) { natsStatus s; natsConnection *nc = NULL; @@ -14766,8 +14313,7 @@ test_AsyncSubscriberStarvation(void) _stopServer(serverPid); } -static void -test_AsyncSubscriberOnClose(void) +void test_AsyncSubscriberOnClose(void) { natsStatus s; natsConnection *nc = NULL; @@ -14838,8 +14384,7 @@ test_AsyncSubscriberOnClose(void) _stopServer(serverPid); } -static void -test_NextMsgCallOnAsyncSub(void) +void test_NextMsgCallOnAsyncSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -14899,8 +14444,7 @@ testOnCompleteMsgHandler(natsConnection *nc, natsSubscription *sub, natsMsg *msg natsMsg_Destroy(msg); } -static void -test_SubOnComplete(void) +void test_SubOnComplete(void) { natsStatus s; natsConnection *nc = NULL; @@ -14983,8 +14527,7 @@ test_SubOnComplete(void) _stopServer(serverPid); } -static void -test_ServersOption(void) +void test_ServersOption(void) { natsStatus s; natsConnection *nc = NULL; @@ -15048,8 +14591,7 @@ test_ServersOption(void) _stopServer(serverPid); } -static void -test_AuthServers(void) +void test_AuthServers(void) { natsStatus s; natsConnection *nc = NULL; @@ -15103,8 +14645,7 @@ test_AuthServers(void) _stopServer(serverPid2); } -static void -test_AuthFailToReconnect(void) +void test_AuthFailToReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -15191,8 +14732,7 @@ test_AuthFailToReconnect(void) _stopServer(serverPid3); } -static void -test_BasicClusterReconnect(void) +void test_BasicClusterReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -15309,8 +14849,7 @@ _reconnectTokenHandler(void* closure) return token; } -static void -test_ReconnectWithTokenHandler(void) +void test_ReconnectWithTokenHandler(void) { natsStatus s; natsConnection *nc = NULL; @@ -15419,8 +14958,7 @@ struct hashCount }; -static void -test_HotSpotReconnect(void) +void test_HotSpotReconnect(void) { natsStatus s; natsConnection *nc[NUM_CLIENTS]; @@ -15573,8 +15111,7 @@ test_HotSpotReconnect(void) _stopServer(serverPid3); } -static void -test_ProperReconnectDelay(void) +void test_ProperReconnectDelay(void) { natsStatus s; natsConnection *nc = NULL; @@ -15643,8 +15180,7 @@ test_ProperReconnectDelay(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ProperFalloutAfterMaxAttempts(void) +void test_ProperFalloutAfterMaxAttempts(void) { natsStatus s; natsConnection *nc = NULL; @@ -15715,8 +15251,7 @@ test_ProperFalloutAfterMaxAttempts(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StopReconnectAfterTwoAuthErr(void) +void test_StopReconnectAfterTwoAuthErr(void) { natsStatus s; natsConnection *nc = NULL; @@ -15807,8 +15342,7 @@ test_StopReconnectAfterTwoAuthErr(void) _stopServer(serverPid2); } -static void -test_TimeoutOnNoServer(void) +void test_TimeoutOnNoServer(void) { natsStatus s; natsConnection *nc = NULL; @@ -15886,8 +15420,7 @@ test_TimeoutOnNoServer(void) _destroyDefaultThreadArgs(&arg); } -static void -test_PingReconnect(void) +void test_PingReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -15971,8 +15504,7 @@ test_PingReconnect(void) } -static void -test_GetServers(void) +void test_GetServers(void) { natsStatus s; natsConnection *conn = NULL; @@ -16056,8 +15588,7 @@ test_GetServers(void) _stopServer(s1Pid); } -static void -test_GetDiscoveredServers(void) +void test_GetDiscoveredServers(void) { natsStatus s; natsConnection *conn = NULL; @@ -16116,8 +15647,7 @@ _discoveredServersCb(natsConnection *nc, void *closure) natsMutex_Unlock(arg->m); } -static void -test_DiscoveredServersCb(void) +void test_DiscoveredServersCb(void) { natsStatus s; natsConnection *conn = NULL; @@ -16179,8 +15709,7 @@ test_DiscoveredServersCb(void) _destroyDefaultThreadArgs(&arg); } -static void -test_IgnoreDiscoveredServers(void) +void test_IgnoreDiscoveredServers(void) { natsStatus s; natsConnection *conn = NULL; @@ -16298,8 +15827,7 @@ _serverSendsINFOAfterPONG(void *closure) natsSock_Close(sock); } -static void -test_ReceiveINFORightAfterFirstPONG(void) +void test_ReceiveINFORightAfterFirstPONG(void) { natsStatus s = NATS_OK; natsThread *t = NULL; @@ -16375,8 +15903,7 @@ test_ReceiveINFORightAfterFirstPONG(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ServerPoolUpdatedOnClusterUpdate(void) +void test_ServerPoolUpdatedOnClusterUpdate(void) { natsStatus s; natsConnection *conn = NULL; @@ -16642,8 +16169,7 @@ test_ServerPoolUpdatedOnClusterUpdate(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ReconnectJitter(void) +void test_ReconnectJitter(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -16765,8 +16291,7 @@ _customReconnectDelayCB(natsConnection *nc, int attempts, void *closure) return delay; } -static void -test_CustomReconnectDelay(void) +void test_CustomReconnectDelay(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -16922,8 +16447,7 @@ _lameDuckMockupServerThread(void *closure) natsSock_Close(sock); } -static void -test_LameDuckMode(void) +void test_LameDuckMode(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -17004,8 +16528,7 @@ test_LameDuckMode(void) _destroyDefaultThreadArgs(&arg); } -static void -test_Version(void) +void test_Version(void) { const char *str = NULL; @@ -17021,8 +16544,7 @@ test_Version(void) testCond(nats_GetVersionNumber() == LIB_NATS_VERSION_NUMBER); } -static void -test_VersionMatchesTag(void) +void test_VersionMatchesTag(void) { natsStatus s = NATS_OK; const char *tag; @@ -17089,8 +16611,7 @@ _openCloseAndWaitThread(void *closure) natsLib_Release(); } -static void -test_OpenCloseAndWait(void) +void test_OpenCloseAndWait(void) { natsStatus s; natsConnection *nc = NULL; @@ -17216,8 +16737,7 @@ _testGetLastErrInThread(void *arg) natsOptions_Destroy(opts); } -static void -test_GetLastError(void) +void test_GetLastError(void) { natsStatus s, getLastErrSts; natsOptions *opts = NULL; @@ -17331,8 +16851,7 @@ test_GetLastError(void) nats_clearLastError(); } -static void -test_StaleConnection(void) +void test_StaleConnection(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -17453,8 +16972,7 @@ test_StaleConnection(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ServerErrorClosesConnection(void) +void test_ServerErrorClosesConnection(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -17558,8 +17076,7 @@ test_ServerErrorClosesConnection(void) _destroyDefaultThreadArgs(&arg); } -static void -test_NoEcho(void) +void test_NoEcho(void) { natsStatus s; natsOptions *opts = NULL; @@ -17666,8 +17183,7 @@ _startMockupServerThread(void *closure) natsSock_Close(sock); } -static void -test_NoEchoOldServer(void) +void test_NoEchoOldServer(void) { natsStatus s; natsConnection *conn = NULL; @@ -17723,8 +17239,7 @@ test_NoEchoOldServer(void) _destroyDefaultThreadArgs(&arg); } -static void -test_DrainSub(void) +void test_DrainSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -18069,8 +17584,7 @@ _drainSubCompleteCB(void *closure) natsMutex_Unlock(arg->m); } -static void -test_DrainSubStops(void) +void test_DrainSubStops(void) { natsStatus s; natsConnection *nc = NULL; @@ -18187,8 +17701,7 @@ test_DrainSubStops(void) _stopServer(pid); } -static void -test_DrainSubRaceOnAutoUnsub(void) +void test_DrainSubRaceOnAutoUnsub(void) { natsStatus s; natsConnection *nc = NULL; @@ -18239,8 +17752,7 @@ test_DrainSubRaceOnAutoUnsub(void) _stopServer(pid); } -static void -test_DrainSubNotResentOnReconnect(void) +void test_DrainSubNotResentOnReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -18386,8 +17898,7 @@ _drainConnErrHandler(natsConnection *nc, natsSubscription *sub, natsStatus err, natsMutex_Unlock(args->m); } -static void -test_DrainConn(void) +void test_DrainConn(void) { natsStatus s; natsConnection *nc = NULL; @@ -18672,8 +18183,7 @@ _noDoubleCbSubCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void * natsMsg_Destroy(msg); } -static void -test_NoDoubleConnClosedOnDrain(void) +void test_NoDoubleConnClosedOnDrain(void) { natsStatus s; natsConnection *nc = NULL; @@ -18736,8 +18246,7 @@ test_NoDoubleConnClosedOnDrain(void) _stopServer(pid); } -static void -test_GetClientID(void) +void test_GetClientID(void) { natsStatus s; natsPid pid1 = NATS_INVALID_PID; @@ -18863,8 +18372,7 @@ test_GetClientID(void) _destroyDefaultThreadArgs(&arg); } -static void -test_GetClientIP(void) +void test_GetClientIP(void) { natsStatus s; natsConnection *nc = NULL; @@ -18971,8 +18479,7 @@ test_GetClientIP(void) _destroyDefaultThreadArgs(&arg); } -static void -test_GetRTT(void) +void test_GetRTT(void) { natsStatus s; natsConnection *nc = NULL; @@ -19014,8 +18521,7 @@ test_GetRTT(void) natsOptions_Destroy(opts); } -static void -test_GetLocalIPAndPort(void) +void test_GetLocalIPAndPort(void) { natsStatus s; natsConnection *nc = NULL; @@ -19175,8 +18681,7 @@ _checkJWTAndSigCB(char *buffer) return NATS_OK; } -static void -test_UserCredsCallbacks(void) +void test_UserCredsCallbacks(void) { natsStatus s; natsConnection *nc = NULL; @@ -19360,8 +18865,7 @@ test_UserCredsCallbacks(void) _destroyDefaultThreadArgs(&arg); } -static void -test_UserCredsFromMemory(void) +void test_UserCredsFromMemory(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -19469,8 +18973,7 @@ test_UserCredsFromMemory(void) natsOptions_Destroy(opts); } -static void -test_UserCredsFromFiles(void) +void test_UserCredsFromFiles(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -19594,7 +19097,7 @@ test_UserCredsFromFiles(void) // Use a file that contains no userJWT.. test("UserOrChainedFile has no JWT: "); - s = natsOptions_SetUserCredentialsFromFiles(opts, "list.txt", NULL); + s = natsOptions_SetUserCredentialsFromFiles(opts, "list_test.txt", NULL); IFOK(s, natsConnection_Connect(&nc, opts)); // Since we return the whole content of the file when we don't find // the key for the user, but we don't for seed, the error we'll get @@ -19610,7 +19113,7 @@ test_UserCredsFromFiles(void) // Use a seed file that contains no seed.. test("SeedFile has no seed: "); - s = natsOptions_SetUserCredentialsFromFiles(opts, ucfn, "list.txt"); + s = natsOptions_SetUserCredentialsFromFiles(opts, ucfn, "list_test.txt"); IFOK(s, natsConnection_Connect(&nc, opts)); testCond((s == NATS_ERR) && (strstr(nats_GetLastError(NULL), "no nkey user seed found") != NULL)); @@ -19710,8 +19213,7 @@ _checkNKeyAndSig(char *buffer) return NATS_OK; } -static void -test_NKey(void) +void test_NKey(void) { natsStatus s; natsOptions *opts = NULL; @@ -19840,8 +19342,7 @@ _checkNKeyFromSeed(char *buffer) return NATS_OK; } -static void -test_NKeyFromSeed(void) +void test_NKeyFromSeed(void) { natsStatus s; natsOptions *opts = NULL; @@ -19976,8 +19477,7 @@ test_NKeyFromSeed(void) remove("seed.file"); } -static void -test_ConnSign(void) +void test_ConnSign(void) { natsStatus s; natsConnection *nc = NULL; @@ -20054,8 +19554,7 @@ test_ConnSign(void) remove(ucfn); } -static void -test_WriteDeadline(void) +void test_WriteDeadline(void) { natsStatus s; natsOptions *opts = NULL; @@ -20137,8 +19636,7 @@ _publish(void *arg) } -static void -test_NoPartialOnReconnect(void) +void test_NoPartialOnReconnect(void) { natsStatus s; natsOptions *opts = NULL; @@ -20237,8 +19735,7 @@ test_NoPartialOnReconnect(void) _stopServer(pid); } -static void -test_ForcedReconnect(void) +void test_ForcedReconnect(void) { natsStatus s; struct threadArg arg; @@ -20331,8 +19828,7 @@ _stopServerInThread(void *closure) _stopServer(pid); } -static void -test_ReconnectFailsPendingRequest(void) +void test_ReconnectFailsPendingRequest(void) { natsStatus s; natsOptions *opts = NULL; @@ -20386,8 +19882,7 @@ test_ReconnectFailsPendingRequest(void) } } -static void -test_HeadersNotSupported(void) +void test_HeadersNotSupported(void) { natsStatus s; natsConnection *conn = NULL; @@ -20459,8 +19954,7 @@ test_HeadersNotSupported(void) _destroyDefaultThreadArgs(&arg); } -static void -test_HeadersBasic(void) +void test_HeadersBasic(void) { natsStatus s; natsConnection *nc = NULL; @@ -20569,8 +20063,7 @@ _msgFilterDropMsg(natsConnection *nc, natsMsg **msg, void *closure) natsConn_setFilter(nc, NULL); } -static void -test_natsMsgsFilter(void) +void test_natsMsgsFilter(void) { natsStatus s; natsConnection *nc = NULL; @@ -20715,8 +20208,7 @@ _eventLoop(void *closure) } } -static void -test_EventLoop(void) +void test_EventLoop(void) { natsStatus s; natsConnection *nc = NULL; @@ -20818,8 +20310,7 @@ test_EventLoop(void) _stopServer(pid); } -static void -test_EventLoopRetryOnFailedConnect(void) +void test_EventLoopRetryOnFailedConnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -20906,8 +20397,7 @@ test_EventLoopRetryOnFailedConnect(void) _stopServer(pid); } -static void -test_EventLoopTLS(void) +void test_EventLoopTLS(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -20997,8 +20487,7 @@ test_EventLoopTLS(void) #endif } -static void -test_SSLBasic(void) +void test_SSLBasic(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21066,8 +20555,7 @@ test_SSLBasic(void) #endif } -static void -test_SSLVerify(void) +void test_SSLVerify(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21132,8 +20620,7 @@ test_SSLVerify(void) #endif } -static void -test_SSLLoadCAFromMemory(void) +void test_SSLLoadCAFromMemory(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21213,8 +20700,7 @@ test_SSLLoadCAFromMemory(void) #endif } -static void -test_SSLCertAndKeyFromMemory(void) +void test_SSLCertAndKeyFromMemory(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21307,8 +20793,7 @@ test_SSLCertAndKeyFromMemory(void) #endif } -static void -test_SSLVerifyHostname(void) +void test_SSLVerifyHostname(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21386,8 +20871,7 @@ test_SSLVerifyHostname(void) #endif } -static void -test_SSLSkipServerVerification(void) +void test_SSLSkipServerVerification(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21446,8 +20930,7 @@ test_SSLSkipServerVerification(void) #endif } -static void -test_SSLCiphers(void) +void test_SSLCiphers(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21580,8 +21063,7 @@ _sslMT(void *closure) #define SSL_THREADS (3) #endif -static void -test_SSLMultithreads(void) +void test_SSLMultithreads(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21648,8 +21130,7 @@ test_SSLMultithreads(void) #endif } -static void -test_SSLConnectVerboseOption(void) +void test_SSLConnectVerboseOption(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21730,8 +21211,7 @@ static natsStatus _elDummyDetach(void *userData) { return NATS_OK; } #endif -static void -test_SSLSocketLeakWithEventLoop(void) +void test_SSLSocketLeakWithEventLoop(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21774,8 +21254,7 @@ test_SSLSocketLeakWithEventLoop(void) #endif } -static void -test_SSLReconnectWithAuthError(void) +void test_SSLReconnectWithAuthError(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21830,8 +21309,7 @@ test_SSLReconnectWithAuthError(void) #endif } -static void -test_SSLAvailable(void) +void test_SSLAvailable(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21990,8 +21468,7 @@ _createConfFile(char *buf, int bufLen, const char *content) fclose(f); } -static void -test_ReconnectImplicitUserInfo(void) +void test_ReconnectImplicitUserInfo(void) { natsStatus s; natsPid pid1 = NATS_INVALID_PID; @@ -22082,8 +21559,7 @@ test_ReconnectImplicitUserInfo(void) remove(conf); } -static void -test_JetStreamUnmarshalAccountInfo(void) +void test_JetStreamUnmarshalAccountInfo(void) { natsStatus s; nats_JSON *json = NULL; @@ -22206,8 +21682,7 @@ test_JetStreamUnmarshalAccountInfo(void) jsAccountInfo_Destroy(ai); } -static void -test_JetStreamUnmarshalStreamState(void) +void test_JetStreamUnmarshalStreamState(void) { natsStatus s; nats_JSON *json = NULL; @@ -22284,8 +21759,7 @@ test_JetStreamUnmarshalStreamState(void) nats_JSONDestroy(json); } -static void -test_JetStreamUnmarshalStreamConfig(void) +void test_JetStreamUnmarshalStreamConfig(void) { natsStatus s; nats_JSON *json = NULL; @@ -22435,8 +21909,7 @@ test_JetStreamUnmarshalStreamConfig(void) json = NULL; } -static void -test_JetStreamUnmarshalStreamInfo(void) +void test_JetStreamUnmarshalStreamInfo(void) { natsStatus s; nats_JSON *json = NULL; @@ -22533,8 +22006,7 @@ test_JetStreamUnmarshalStreamInfo(void) } } -static void -test_JetStreamMarshalStreamConfig(void) +void test_JetStreamMarshalStreamConfig(void) { natsStatus s; jsStreamConfig sc; @@ -22786,8 +22258,7 @@ test_JetStreamMarshalStreamConfig(void) testCond((s == NATS_INVALID_ARG) && (buf == NULL)); } -static void -test_JetStreamUnmarshalConsumerInfo(void) +void test_JetStreamUnmarshalConsumerInfo(void) { natsStatus s; jsConsumerInfo *ci = NULL; @@ -23031,8 +22502,7 @@ natsConnection_Destroy(nc); \ _stopServer(pid); \ rmtree(datastore); -static void -test_JetStreamContext(void) +void test_JetStreamContext(void) { natsStatus s; natsConnection *nc = NULL; @@ -23208,8 +22678,7 @@ test_JetStreamContext(void) remove(confFile); } -static void -test_JetStreamContextDomain(void) +void test_JetStreamContextDomain(void) { natsStatus s; natsConnection *nc = NULL; @@ -23421,8 +22890,7 @@ _streamsNamesListReq(natsConnection *nc, natsMsg **msg, void *closure) } } -static void -test_JetStreamMgtStreams(void) +void test_JetStreamMgtStreams(void) { natsStatus s; jsCtx *js2= NULL; @@ -24196,8 +23664,7 @@ _consumerNamesListReq(natsConnection *nc, natsMsg **msg, void *closure) } } -static void -test_JetStreamMgtConsumers(void) +void test_JetStreamMgtConsumers(void) { natsStatus s; jsConsumerInfo *ci = NULL; @@ -25168,8 +24635,7 @@ test_JetStreamMgtConsumers(void) JS_TEARDOWN; } -static void -test_JetStreamPublish(void) +void test_JetStreamPublish(void) { natsStatus s; natsConnection *nc = NULL; @@ -25522,8 +24988,7 @@ _jsPubAckErrHandler(jsCtx *js, jsPubAckErr *pae, void *closure) natsMutex_Unlock(args->m); } -static void -test_JetStreamPublishAsync(void) +void test_JetStreamPublishAsync(void) { natsStatus s; natsSubscription *sub= NULL; @@ -26098,8 +25563,7 @@ _checkPubAckResult(natsStatus s, struct threadArg *args) return s; } -static void -test_JetStreamPublishAckHandler(void) +void test_JetStreamPublishAckHandler(void) { natsStatus s; jsOptions o; @@ -26298,8 +25762,7 @@ _jsDrainErrCb(natsConnection *nc, natsSubscription *sub, natsStatus err, void *c natsMutex_Unlock(args->m); } -static void -test_JetStreamSubscribe(void) +void test_JetStreamSubscribe(void) { natsStatus s; natsOptions *ncOpts = NULL; @@ -26973,8 +26436,7 @@ test_JetStreamSubscribe(void) _destroyDefaultThreadArgs(&args); } -static void -test_JetStreamSubscribeSync(void) +void test_JetStreamSubscribeSync(void) { natsStatus s; natsSubscription *sub= NULL; @@ -27475,8 +26937,7 @@ test_JetStreamSubscribeSync(void) JS_TEARDOWN; } -static void -test_JetStreamSubscribeConfigCheck(void) +void test_JetStreamSubscribeConfigCheck(void) { natsStatus s; natsSubscription *sub= NULL; @@ -27876,8 +27337,7 @@ _setMsgReply(natsConnection *nc, natsMsg **msg, void* closure) natsConn_setFilter(nc, NULL); } -static void -test_JetStreamSubscribeIdleHearbeat(void) +void test_JetStreamSubscribeIdleHearbeat(void) { natsStatus s; natsConnection *nc = NULL; @@ -28207,8 +27667,7 @@ test_JetStreamSubscribeIdleHearbeat(void) natsOptions_Destroy(opts); } -static void -test_JetStreamSubscribeFlowControl(void) +void test_JetStreamSubscribeFlowControl(void) { natsStatus s; natsSubscription *sub= NULL; @@ -28482,8 +27941,7 @@ _dropTimeoutProto(natsConnection *nc, natsMsg **msg, void* closure) *msg = NULL; } -static void -test_JetStreamSubscribePull(void) +void test_JetStreamSubscribePull(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29028,8 +28486,7 @@ test_JetStreamSubscribePull(void) _destroyDefaultThreadArgs(&args); } -static void -test_JetStreamSubscribeHeadersOnly(void) +void test_JetStreamSubscribeHeadersOnly(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29259,8 +28716,7 @@ _lastOnlyLoss(natsConnection *nc, natsMsg **msg, void* closure) } } -static void -test_JetStreamOrderedConsumer(void) +void test_JetStreamOrderedConsumer(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29501,8 +28957,7 @@ _jsOrderedErrHandler(natsConnection *nc, natsSubscription *subscription, natsSta natsMutex_Unlock(args->m); } -static void -test_JetStreamOrderedConsumerWithErrors(void) +void test_JetStreamOrderedConsumerWithErrors(void) { natsStatus s; natsConnection *nc = NULL; @@ -29636,8 +29091,7 @@ _dropMsgFive(natsConnection *nc, natsMsg **msg, void* closure) } } -static void -test_JetStreamOrderedConsumerWithAutoUnsub(void) +void test_JetStreamOrderedConsumerWithAutoUnsub(void) { natsStatus s; natsConnection *nc2= NULL; @@ -29755,8 +29209,7 @@ test_JetStreamOrderedConsumerWithAutoUnsub(void) JS_TEARDOWN; } -static void -test_JetStreamOrderedConsSrvRestart(void) +void test_JetStreamOrderedConsSrvRestart(void) { natsStatus s; natsSubscription *sub = NULL; @@ -29872,8 +29325,7 @@ test_JetStreamOrderedConsSrvRestart(void) JS_TEARDOWN; } -static void -test_JetStreamSubscribeWithFWC(void) +void test_JetStreamSubscribeWithFWC(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29910,8 +29362,7 @@ test_JetStreamSubscribeWithFWC(void) JS_TEARDOWN; } -static void -test_JetStreamStreamsSealAndRollup(void) +void test_JetStreamStreamsSealAndRollup(void) { natsStatus s; jsStreamInfo *si = NULL; @@ -30079,8 +29530,7 @@ test_JetStreamStreamsSealAndRollup(void) JS_TEARDOWN; } -static void -test_JetStreamGetMsgAndLastMsg(void) +void test_JetStreamGetMsgAndLastMsg(void) { natsStatus s; natsMsg *msg = NULL; @@ -30194,8 +29644,7 @@ test_JetStreamGetMsgAndLastMsg(void) JS_TEARDOWN; } -static void -test_JetStreamConvertDirectMsg(void) +void test_JetStreamConvertDirectMsg(void) { natsStatus s; natsMsg *msg = NULL; @@ -30306,8 +29755,7 @@ _checkDirectGet(jsCtx *js, uint64_t seq, const char *nextBySubj, const char *las return s; } -static void -test_JetStreamDirectGetMsg(void) +void test_JetStreamDirectGetMsg(void) { natsStatus s; natsMsg *msg = NULL; @@ -30402,8 +29850,7 @@ test_JetStreamDirectGetMsg(void) JS_TEARDOWN; } -static void -test_JetStreamNakWithDelay(void) +void test_JetStreamNakWithDelay(void) { natsStatus s; natsSubscription *sub= NULL; @@ -30480,8 +29927,7 @@ test_JetStreamNakWithDelay(void) JS_TEARDOWN; } -static void -test_JetStreamBackOffRedeliveries(void) +void test_JetStreamBackOffRedeliveries(void) { natsStatus s; natsSubscription *sub= NULL; @@ -30623,8 +30069,7 @@ _subjectsInfoReq(natsConnection *nc, natsMsg **msg, void *closure) } } -static void -test_JetStreamInfoWithSubjects(void) +void test_JetStreamInfoWithSubjects(void) { natsStatus s; jsStreamInfo *si = NULL; @@ -30791,8 +30236,7 @@ _checkJSClusterReady(const char *url) return s; } -static void -test_JetStreamInfoAlternates(void) +void test_JetStreamInfoAlternates(void) { char datastore1[256] = {'\0'}; char datastore2[256] = {'\0'}; @@ -30885,8 +30329,7 @@ test_JetStreamInfoAlternates(void) rmtree(datastore3); } -static void -test_KeyValueManager(void) +void test_KeyValueManager(void) { natsStatus s; kvStore *kv = NULL; @@ -31016,8 +30459,7 @@ test_KeyValueManager(void) JS_TEARDOWN; } -static void -test_KeyValueBasics(void) +void test_KeyValueBasics(void) { natsStatus s; kvStore *kv = NULL; @@ -31423,8 +30865,7 @@ _stopWatcher(void *closure) kvWatcher_Stop(w); } -static void -test_KeyValueWatch(void) +void test_KeyValueWatch(void) { natsStatus s; kvStore *kv = NULL; @@ -31570,8 +31011,7 @@ test_KeyValueWatch(void) JS_TEARDOWN; } -static void -test_KeyValueWatchMulti(void) +void test_KeyValueWatchMulti(void) { natsStatus s; kvStore *kv = NULL; @@ -31618,8 +31058,7 @@ test_KeyValueWatchMulti(void) JS_TEARDOWN; } -static void -test_KeyValueHistory(void) +void test_KeyValueHistory(void) { natsStatus s; kvStore *kv = NULL; @@ -31703,8 +31142,7 @@ test_KeyValueHistory(void) JS_TEARDOWN; } -static void -test_KeyValueKeys(void) +void test_KeyValueKeys(void) { natsStatus s; kvStore *kv = NULL; @@ -31806,8 +31244,7 @@ test_KeyValueKeys(void) JS_TEARDOWN; } -static void -test_KeyValueDeleteVsPurge(void) +void test_KeyValueDeleteVsPurge(void) { natsStatus s; kvStore *kv = NULL; @@ -31877,8 +31314,7 @@ test_KeyValueDeleteVsPurge(void) JS_TEARDOWN; } -static void -test_KeyValueDeleteTombstones(void) +void test_KeyValueDeleteTombstones(void) { natsStatus s; kvStore *kv = NULL; @@ -31947,8 +31383,7 @@ test_KeyValueDeleteTombstones(void) JS_TEARDOWN; } -static void -test_KeyValuePurgeDeletesMarkerThreshold(void) +void test_KeyValuePurgeDeletesMarkerThreshold(void) { natsStatus s; kvStore *kv = NULL; @@ -32008,8 +31443,7 @@ test_KeyValuePurgeDeletesMarkerThreshold(void) JS_TEARDOWN; } -static void -test_KeyValueCrossAccount(void) +void test_KeyValueCrossAccount(void) { natsStatus s; natsOptions *opts= NULL; @@ -32280,8 +31714,7 @@ _checkDiscard(jsCtx *js, jsDiscardPolicy expected, kvStore **newKV) return s; } -static void -test_KeyValueDiscardOldToNew(void) +void test_KeyValueDiscardOldToNew(void) { kvStore *kv = NULL; kvConfig kvc; @@ -32339,8 +31772,7 @@ test_KeyValueDiscardOldToNew(void) JS_TEARDOWN; } -static void -test_KeyValueRePublish(void) +void test_KeyValueRePublish(void) { kvStore *kv = NULL; jsStreamInfo *si = NULL; @@ -32411,8 +31843,7 @@ test_KeyValueRePublish(void) JS_TEARDOWN; } -static void -test_KeyValueMirrorDirectGet(void) +void test_KeyValueMirrorDirectGet(void) { kvStore *kv = NULL; kvConfig kvc; @@ -32501,8 +31932,7 @@ _connectToHubAndCheckLeaf(natsConnection **hub, natsConnection *lnc) return s; } -static void -test_KeyValueMirrorCrossDomains(void) +void test_KeyValueMirrorCrossDomains(void) { natsStatus s; natsConnection *nc = NULL; @@ -32826,8 +32256,7 @@ test_KeyValueMirrorCrossDomains(void) remove(lconfFile); } -static void -test_MicroMatchEndpointSubject(void) +void test_MicroMatchEndpointSubject(void) { // endpoint, actual, match const char *test_cases[] = { @@ -33006,8 +32435,7 @@ typedef struct int expected_num_subjects; } add_service_test_case_t; -static void -test_MicroAddService(void) +void test_MicroAddService(void) { natsStatus s = NATS_OK; microError *err = NULL; @@ -33242,8 +32670,7 @@ test_MicroAddService(void) _stopServer(serverPid); } -static void -test_MicroGroups(void) +void test_MicroGroups(void) { natsStatus s = NATS_OK; microError *err = NULL; @@ -33352,8 +32779,7 @@ test_MicroGroups(void) #define NUM_MICRO_SERVICES 5 -static void -test_MicroBasics(void) +void test_MicroBasics(void) { natsStatus s = NATS_OK; microError *err = NULL; @@ -33633,8 +33059,7 @@ test_MicroBasics(void) _stopServer(serverPid); } -static void -test_MicroStartStop(void) +void test_MicroStartStop(void) { natsStatus s = NATS_OK; struct threadArg arg; @@ -33702,8 +33127,7 @@ test_MicroStartStop(void) _stopServer(serverPid); } -static void -test_MicroServiceStopsOnClosedConn(void) +void test_MicroServiceStopsOnClosedConn(void) { natsStatus s; natsConnection *nc = NULL; @@ -33775,8 +33199,7 @@ test_MicroServiceStopsOnClosedConn(void) _stopServer(serverPid); } -static void -test_MicroServiceStopsWhenServerStops(void) +void test_MicroServiceStopsWhenServerStops(void) { natsStatus s; natsConnection *nc = NULL; @@ -33868,8 +33291,7 @@ _microAsyncErrorRequestHandler(microRequest *req) return NULL; } -static void -test_MicroAsyncErrorHandler_MaxPendingMsgs(void) +void test_MicroAsyncErrorHandlerMaxPendingMsgs(void) { natsStatus s; struct threadArg arg; @@ -33947,8 +33369,7 @@ test_MicroAsyncErrorHandler_MaxPendingMsgs(void) _stopServer(serverPid); } -static void -test_MicroAsyncErrorHandler_MaxPendingBytes(void) +void test_MicroAsyncErrorHandlerMaxPendingBytes(void) { natsStatus s; struct threadArg arg; @@ -34038,8 +33459,7 @@ _roundUp(int val) return ((val + (MEMALIGN-1))/MEMALIGN)*MEMALIGN; } -static void -test_StanPBufAllocator(void) +void test_StanPBufAllocator(void) { natsPBufAllocator *a = NULL; natsStatus s; @@ -34172,8 +33592,7 @@ _stanConnLostCB(stanConnection *sc, const char *errorTxt, void *closure) natsMutex_Unlock(arg->m); } -static void -test_StanConnOptions(void) +void test_StanConnOptions(void) { natsStatus s; stanConnOptions *opts = NULL; @@ -34340,8 +33759,7 @@ test_StanConnOptions(void) stanConnOptions_Destroy(clone); } -static void -test_StanSubOptions(void) +void test_StanSubOptions(void) { natsStatus s; stanSubOptions *opts = NULL; @@ -34463,8 +33881,7 @@ test_StanSubOptions(void) stanSubOptions_Destroy(clone); } -static void -test_StanMsg(void) +void test_StanMsg(void) { test("GetSequence with NULL msg: "); testCond(stanMsg_GetSequence(NULL) == 0); @@ -34484,8 +33901,7 @@ test_StanMsg(void) stanMsg_Destroy(NULL); } -static void -test_StanServerNotReachable(void) +void test_StanServerNotReachable(void) { natsStatus s; stanConnection *sc = NULL; @@ -34524,8 +33940,7 @@ test_StanServerNotReachable(void) _stopServer(serverPid); } -static void -test_StanBasicConnect(void) +void test_StanBasicConnect(void) { natsStatus s; stanConnection *sc = NULL; @@ -34592,8 +34007,7 @@ test_StanBasicConnect(void) _stopServer(pid); } -static void -test_StanConnectError(void) +void test_StanConnectError(void) { natsStatus s; stanConnection *sc = NULL; @@ -34629,8 +34043,7 @@ test_StanConnectError(void) } -static void -test_StanBasicPublish(void) +void test_StanBasicPublish(void) { natsStatus s; stanConnection *sc = NULL; @@ -34673,8 +34086,7 @@ _stanPubAckHandler(const char *guid, const char *errTxt, void* closure) natsMutex_Unlock(args->m); } -static void -test_StanBasicPublishAsync(void) +void test_StanBasicPublishAsync(void) { natsStatus s; stanConnection *sc = NULL; @@ -34712,8 +34124,7 @@ test_StanBasicPublishAsync(void) _stopServer(pid); } -static void -test_StanPublishTimeout(void) +void test_StanPublishTimeout(void) { natsStatus s; stanConnection *sc = NULL; @@ -34789,8 +34200,7 @@ _stanPublishSyncThread(void *closure) stanConnection_Publish(sc, "foo", (const void*)"hello", 5); } -static void -test_StanPublishMaxAcksInflight(void) +void test_StanPublishMaxAcksInflight(void) { natsStatus s; stanConnection *sc1 = NULL; @@ -34938,8 +34348,7 @@ _stanMsgHandlerBumpSum(stanConnection *sc, stanSubscription *sub, const char *ch stanMsg_Destroy(msg); } -static void -test_StanBasicSubscription(void) +void test_StanBasicSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -34982,8 +34391,7 @@ test_StanBasicSubscription(void) _stopServer(pid); } -static void -test_StanSubscriptionCloseAndUnsubscribe(void) +void test_StanSubscriptionCloseAndUnsubscribe(void) { natsStatus s; stanConnection *sc = NULL; @@ -35083,8 +34491,7 @@ test_StanSubscriptionCloseAndUnsubscribe(void) _stopServer(pid); } -static void -test_StanDurableSubscription(void) +void test_StanDurableSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -35171,8 +34578,7 @@ test_StanDurableSubscription(void) _stopServer(pid); } -static void -test_StanBasicQueueSubscription(void) +void test_StanBasicQueueSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -35235,8 +34641,7 @@ test_StanBasicQueueSubscription(void) _stopServer(pid); } -static void -test_StanDurableQueueSubscription(void) +void test_StanDurableQueueSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -35346,8 +34751,7 @@ _stanCheckRecvStanMsg(stanConnection *sc, stanSubscription *sub, const char *cha natsMutex_Unlock(args->m); } -static void -test_StanCheckReceivedvMsg(void) +void test_StanCheckReceivedMsg(void) { natsStatus s; stanConnection *sc = NULL; @@ -35435,8 +34839,7 @@ _stanGetMsg(stanConnection *sc, stanSubscription *sub, const char *channel, natsMutex_Unlock(args->m); } -static void -test_StanSubscriptionAckMsg(void) +void test_StanSubscriptionAckMsg(void) { natsStatus s; stanConnection *sc = NULL; @@ -35556,8 +34959,7 @@ test_StanSubscriptionAckMsg(void) _stopServer(pid); } -static void -test_StanPings(void) +void test_StanPings(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35646,8 +35048,7 @@ test_StanPings(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanPingsNoResponder(void) +void test_StanPingsNoResponder(void) { natsStatus s; natsPid nPid = NATS_INVALID_PID; @@ -35702,8 +35103,7 @@ test_StanPingsNoResponder(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanConnectionLostHandlerNotSet(void) +void test_StanConnectionLostHandlerNotSet(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35764,8 +35164,7 @@ test_StanConnectionLostHandlerNotSet(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanPingsUnblockPubCalls(void) +void test_StanPingsUnblockPubCalls(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35847,8 +35246,7 @@ test_StanPingsUnblockPubCalls(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanGetNATSConnection(void) +void test_StanGetNATSConnection(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35950,8 +35348,7 @@ test_StanGetNATSConnection(void) _stopServer(pid); } -static void -test_StanNoRetryOnFailedConnect(void) +void test_StanNoRetryOnFailedConnect(void) { natsStatus s; natsOptions *opts = NULL; @@ -35980,8 +35377,7 @@ _subDlvThreadPooled(natsSubscription *sub) return pooled; } -static void -test_StanInternalSubsNotPooled(void) +void test_StanInternalSubsNotPooled(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -36079,8 +35475,7 @@ _stanSubOnCompleteMsgCB(stanConnection *sc, stanSubscription *sub, const char *c stanMsg_Destroy(msg); } -static void -test_StanSubOnComplete(void) +void test_StanSubOnComplete(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -36159,8 +35554,7 @@ test_StanSubOnComplete(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanSubTimeout(void) +void test_StanSubTimeout(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -36243,363 +35637,6 @@ test_StanSubTimeout(void) #endif -typedef void (*testFunc)(void); - -typedef struct __testInfo -{ - const char *name; - testFunc func; - -} testInfo; - -static testInfo allTests[] = -{ - // Building blocks - {"Version", test_Version}, - {"VersionMatchesTag", test_VersionMatchesTag}, - {"OpenCloseAndWait", test_OpenCloseAndWait}, - {"natsNowAndSleep", test_natsNowAndSleep}, - {"natsAllocSprintf", test_natsAllocSprintf}, - {"natsStrCaseStr", test_natsStrCaseStr}, - {"natsSnprintf", test_natsSnprintf}, - {"natsBuffer", test_natsBuffer}, - {"natsParseInt64", test_natsParseInt64}, - {"natsParseControl", test_natsParseControl}, - {"natsNormalizeErr", test_natsNormalizeErr}, - {"natsMutex", test_natsMutex}, - {"natsThread", test_natsThread}, - {"natsCondition", test_natsCondition}, - {"natsTimer", test_natsTimer}, - {"natsUrl", test_natsUrl}, - {"natsCreateStringFromBuffer", test_natsCreateStringFromBuffer}, - {"natsHash", test_natsHash}, - {"natsHashing", test_natsHashing}, - {"natsStrHash", test_natsStrHash}, - {"natsInbox", test_natsInbox}, - {"natsOptions", test_natsOptions}, - {"natsSock_ConnectTcp", test_natsSock_ConnectTcp}, - {"natsSock_ShuffleIPs", test_natsSock_ShuffleIPs}, - {"natsSock_IPOrder", test_natsSock_IPOrder}, - {"natsSock_ReadLine", test_natsSock_ReadLine}, - {"natsJSON", test_natsJSON}, - {"natsEncodeTimeUTC", test_natsEncodeTimeUTC}, - {"natsErrWithLongText", test_natsErrWithLongText}, - {"natsErrStackMoreThanMaxFrames", test_natsErrStackMoreThanMaxFrames}, - {"natsMsg", test_natsMsg}, - {"natsBase32", test_natsBase32Decode}, - {"natsBase64", test_natsBase64Encode}, - {"natsCRC16", test_natsCRC16}, - {"natsKeys", test_natsKeys}, - {"natsReadFile", test_natsReadFile}, - {"natsGetJWTOrSeed", test_natsGetJWTOrSeed}, - {"natsHostIsIP", test_natsHostIsIP}, - {"natsWaitReady", test_natsWaitReady}, - {"natsSign", test_natsSign}, - {"HeadersLift", test_natsMsgHeadersLift}, - {"HeadersAPIs", test_natsMsgHeaderAPIs}, - {"MsgIsJSControl", test_natsMsgIsJSCtrl}, - {"SrvVersionAtLeast", test_natsSrvVersionAtLeast}, - {"FormatStringArray", test_natsFormatStringArray}, - - // Package Level Tests - - {"ReconnectServerStats", test_ReconnectServerStats}, - {"ParseStateReconnectFunctionality",test_ParseStateReconnectFunctionality}, - {"ServersRandomize", test_ServersRandomize}, - {"SelectNextServer", test_SelectNextServer}, - {"ParserPing", test_ParserPing}, - {"ParserErr", test_ParserErr}, - {"ParserOK", test_ParserOK}, - {"ParseINFO", test_ParseINFO}, - {"ParserShouldFail", test_ParserShouldFail}, - {"ParserSplitMsg", test_ParserSplitMsg}, - {"ProcessMsgArgs", test_ProcessMsgArgs}, - {"LibMsgDelivery", test_LibMsgDelivery}, - {"AsyncINFO", test_AsyncINFO}, - {"RequestPool", test_RequestPool}, - {"NoFlusherIfSendAsapOption", test_NoFlusherIfSendAsap}, - {"HeadersAndSubPendingBytes", test_HeadersAndSubPendingBytes}, - - // Public API Tests - - {"DefaultConnection", test_DefaultConnection}, - {"SimplifiedURLs", test_SimplifiedURLs}, - {"IPResolutionOrder", test_IPResolutionOrder}, - {"UseDefaultURLIfNoServerSpecified",test_UseDefaultURLIfNoServerSpecified}, - {"ConnectToWithMultipleURLs", test_ConnectToWithMultipleURLs}, - {"ConnectionWithNULLOptions", test_ConnectionWithNullOptions}, - {"ConnectionToWithNullURLs", test_ConnectionToWithNullURLs}, - {"ConnectionStatus", test_ConnectionStatus}, - {"ConnClosedCB", test_ConnClosedCB}, - {"CloseDisconnectedCB", test_CloseDisconnectedCB}, - {"ServerStopDisconnectedCB", test_ServerStopDisconnectedCB}, - {"ClosedConnections", test_ClosedConnections}, - {"ConnectVerboseOption", test_ConnectVerboseOption}, - {"ReconnectThreadLeak", test_ReconnectThreadLeak}, - {"ReconnectTotalTime", test_ReconnectTotalTime}, - {"ReconnectDisallowedFlags", test_ReconnectDisallowedFlags}, - {"ReconnectAllowedFlags", test_ReconnectAllowedFlags}, - {"ConnCloseBreaksReconnectLoop", test_ConnCloseBreaksReconnectLoop}, - {"BasicReconnectFunctionality", test_BasicReconnectFunctionality}, - {"ExtendedReconnectFunctionality", test_ExtendedReconnectFunctionality}, - {"QueueSubsOnReconnect", test_QueueSubsOnReconnect}, - {"IsClosed", test_IsClosed}, - {"IsReconnectingAndStatus", test_IsReconnectingAndStatus}, - {"ReconnectBufSize", test_ReconnectBufSize}, - {"RetryOnFailedConnect", test_RetryOnFailedConnect}, - {"NoPartialOnReconnect", test_NoPartialOnReconnect}, - {"ReconnectFailsPendingRequests", test_ReconnectFailsPendingRequest}, - {"ForcedReconnect", test_ForcedReconnect}, - - {"ErrOnConnectAndDeadlock", test_ErrOnConnectAndDeadlock}, - {"ErrOnMaxPayloadLimit", test_ErrOnMaxPayloadLimit}, - - {"Auth", test_Auth}, - {"AuthFailNoDisconnectCB", test_AuthFailNoDisconnectCB}, - {"AuthToken", test_AuthToken}, - {"AuthTokenHandler", test_AuthTokenHandler}, - {"PermViolation", test_PermViolation}, - {"AuthViolation", test_AuthViolation}, - {"AuthenticationExpired", test_AuthenticationExpired}, - {"AuthenticationExpiredReconnect", test_AuthenticationExpiredReconnect}, - {"ConnectedServer", test_ConnectedServer}, - {"MultipleClose", test_MultipleClose}, - {"SimplePublish", test_SimplePublish}, - {"SimplePublishNoData", test_SimplePublishNoData}, - {"PublishMsg", test_PublishMsg}, - {"InvalidSubsArgs", test_InvalidSubsArgs}, - {"AsyncSubscribe", test_AsyncSubscribe}, - {"AsyncSubscribeTimeout", test_AsyncSubscribeTimeout}, - {"SyncSubscribe", test_SyncSubscribe}, - {"PubSubWithReply", test_PubSubWithReply}, - {"NoResponders", test_NoResponders}, - {"Flush", test_Flush}, - {"ConnCloseDoesFlush", test_ConnCloseDoesFlush}, - {"QueueSubscriber", test_QueueSubscriber}, - {"ReplyArg", test_ReplyArg}, - {"SyncReplyArg", test_SyncReplyArg}, - {"Unsubscribe", test_Unsubscribe}, - {"DoubleUnsubscribe", test_DoubleUnsubscribe}, - {"SubRemovedWhileProcessingMsg", test_SubRemovedWhileProcessingMsg}, - {"RequestTimeout", test_RequestTimeout}, - {"Request", test_Request}, - {"RequestNoBody", test_RequestNoBody}, - {"RequestMuxWithMappedSubject", test_RequestMuxWithMappedSubject}, - {"OldRequest", test_OldRequest}, - {"SimultaneousRequests", test_SimultaneousRequest}, - {"RequestClose", test_RequestClose}, - {"CustomInbox", test_CustomInbox}, - {"MessagePadding", test_MessageBufferPadding}, - {"FlushInCb", test_FlushInCb}, - {"ReleaseFlush", test_ReleaseFlush}, - {"FlushErrOnDisconnect", test_FlushErrOnDisconnect}, - {"Inbox", test_Inbox}, - {"Stats", test_Stats}, - {"BadSubject", test_BadSubject}, - {"SubBadSubjectAndQueueNames", test_SubBadSubjectAndQueueName}, - {"ClientAsyncAutoUnsub", test_ClientAsyncAutoUnsub}, - {"ClientSyncAutoUnsub", test_ClientSyncAutoUnsub}, - {"ClientAutoUnsubAndReconnect", test_ClientAutoUnsubAndReconnect}, - {"AutoUnsubNoUnsubOnDestroy", test_AutoUnsubNoUnsubOnDestroy}, - {"NextMsgOnClosedSub", test_NextMsgOnClosedSub}, - {"CloseSubRelease", test_CloseSubRelease}, - {"IsValidSubscriber", test_IsValidSubscriber}, - {"SlowSubscriber", test_SlowSubscriber}, - {"SlowAsyncSubscriber", test_SlowAsyncSubscriber}, - {"SlowConsumerCb", test_SlowConsumerCB}, - {"PendingLimitsDeliveredAndDropped",test_PendingLimitsDeliveredAndDropped}, - {"PendingLimitsWithSyncSub", test_PendingLimitsWithSyncSub}, - {"AsyncSubscriptionPending", test_AsyncSubscriptionPending}, - {"AsyncSubscriptionPendingDrain", test_AsyncSubscriptionPendingDrain}, - {"SyncSubscriptionPending", test_SyncSubscriptionPending}, - {"SyncSubscriptionPendingDrain", test_SyncSubscriptionPendingDrain}, - {"AsyncErrHandlerMaxPendingMsgs", test_AsyncErrHandler_MaxPendingMsgs}, - {"AsyncErrHandlerMaxPendingBytes", test_AsyncErrHandler_MaxPendingBytes }, - {"AsyncErrHandlerSubDestroyed", test_AsyncErrHandlerSubDestroyed}, - {"AsyncSubscriberStarvation", test_AsyncSubscriberStarvation}, - {"AsyncSubscriberOnClose", test_AsyncSubscriberOnClose}, - {"NextMsgCallOnAsyncSub", test_NextMsgCallOnAsyncSub}, - {"SubOnComplete", test_SubOnComplete}, - {"GetLastError", test_GetLastError}, - {"StaleConnection", test_StaleConnection}, - {"ServerErrorClosesConnection", test_ServerErrorClosesConnection}, - {"NoEcho", test_NoEcho}, - {"NoEchoOldServer", test_NoEchoOldServer}, - {"DrainSub", test_DrainSub}, - {"DrainSubStops", test_DrainSubStops}, - {"DrainSubRaceOnAutoUnsub", test_DrainSubRaceOnAutoUnsub}, - {"DrainSubNotResentOnReconnect", test_DrainSubNotResentOnReconnect}, - {"DrainConn", test_DrainConn}, - {"NoDoubleCloseCbOnDrain", test_NoDoubleConnClosedOnDrain}, - {"GetClientID", test_GetClientID}, - {"GetClientIP", test_GetClientIP}, - {"GetRTT", test_GetRTT}, - {"GetLocalIPAndPort", test_GetLocalIPAndPort}, - {"UserCredsCallbacks", test_UserCredsCallbacks}, - {"UserCredsFromFiles", test_UserCredsFromFiles}, - {"UserCredsFromMemory", test_UserCredsFromMemory}, - {"NKey", test_NKey}, - {"NKeyFromSeed", test_NKeyFromSeed}, - {"ConnSign", test_ConnSign}, - {"WriteDeadline", test_WriteDeadline}, - {"HeadersNotSupported", test_HeadersNotSupported}, - {"HeadersBasic", test_HeadersBasic}, - {"MsgsFilter", test_natsMsgsFilter}, - {"EventLoop", test_EventLoop}, - {"EventLoopRetryOnFailedConnect", test_EventLoopRetryOnFailedConnect}, - {"EventLoopTLS", test_EventLoopTLS}, - {"SSLBasic", test_SSLBasic}, - {"SSLVerify", test_SSLVerify}, - {"SSLCAFromMemory", test_SSLLoadCAFromMemory}, - {"SSLCertAndKeyFromMemory", test_SSLCertAndKeyFromMemory}, - {"SSLVerifyHostname", test_SSLVerifyHostname}, - {"SSLSkipServerVerification", test_SSLSkipServerVerification}, - {"SSLCiphers", test_SSLCiphers}, - {"SSLMultithreads", test_SSLMultithreads}, - {"SSLConnectVerboseOption", test_SSLConnectVerboseOption}, - {"SSLSocketLeakEventLoop", test_SSLSocketLeakWithEventLoop}, - {"SSLReconnectWithAuthError", test_SSLReconnectWithAuthError}, - {"SSLAvailable", test_SSLAvailable}, - - // Clusters Tests - - {"ServersOption", test_ServersOption}, - {"AuthServers", test_AuthServers}, - {"AuthFailToReconnect", test_AuthFailToReconnect}, - {"ReconnectWithTokenHandler", test_ReconnectWithTokenHandler}, - {"BasicClusterReconnect", test_BasicClusterReconnect}, - {"HotSpotReconnect", test_HotSpotReconnect}, - {"ProperReconnectDelay", test_ProperReconnectDelay}, - {"ProperFalloutAfterMaxAttempts", test_ProperFalloutAfterMaxAttempts}, - {"StopReconnectAfterTwoAuthErr", test_StopReconnectAfterTwoAuthErr}, - {"TimeoutOnNoServer", test_TimeoutOnNoServer}, - {"PingReconnect", test_PingReconnect}, - {"GetServers", test_GetServers}, - {"GetDiscoveredServers", test_GetDiscoveredServers}, - {"DiscoveredServersCb", test_DiscoveredServersCb}, - {"IgnoreDiscoveredServers", test_IgnoreDiscoveredServers}, - {"INFOAfterFirstPONGisProcessedOK", test_ReceiveINFORightAfterFirstPONG}, - {"ServerPoolUpdatedOnClusterUpdate",test_ServerPoolUpdatedOnClusterUpdate}, - {"ReconnectJitter", test_ReconnectJitter}, - {"CustomReconnectDelay", test_CustomReconnectDelay}, - {"LameDuckMode", test_LameDuckMode}, - {"ReconnectImplicitUserInfo", test_ReconnectImplicitUserInfo}, - - {"JetStreamUnmarshalAccInfo", test_JetStreamUnmarshalAccountInfo}, - {"JetStreamUnmarshalStreamState", test_JetStreamUnmarshalStreamState}, - {"JetStreamUnmarshalStreamCfg", test_JetStreamUnmarshalStreamConfig}, - {"JetStreamUnmarshalStreamInfo", test_JetStreamUnmarshalStreamInfo}, - {"JetStreamMarshalStreamCfg", test_JetStreamMarshalStreamConfig}, - {"JetStreamUnmarshalConsumerInfo", test_JetStreamUnmarshalConsumerInfo}, - {"JetStreamContext", test_JetStreamContext}, - {"JetStreamDomain", test_JetStreamContextDomain}, - {"JetStreamMgtStreams", test_JetStreamMgtStreams}, - {"JetStreamMgtConsumers", test_JetStreamMgtConsumers}, - {"JetStreamPublish", test_JetStreamPublish}, - {"JetStreamPublishAsync", test_JetStreamPublishAsync}, - {"JetStreamPublishAckHandler", test_JetStreamPublishAckHandler}, - {"JetStreamSubscribe", test_JetStreamSubscribe}, - {"JetStreamSubscribeSync", test_JetStreamSubscribeSync}, - {"JetStreamSubscribeConfigCheck", test_JetStreamSubscribeConfigCheck}, - {"JetStreamSubscribeIdleHeartbeat", test_JetStreamSubscribeIdleHearbeat}, - {"JetStreamSubscribeFlowControl", test_JetStreamSubscribeFlowControl}, - {"JetStreamSubscribePull", test_JetStreamSubscribePull}, - {"JetStreamSubscribeHeadersOnly", test_JetStreamSubscribeHeadersOnly}, - {"JetStreamOrderedCons", test_JetStreamOrderedConsumer}, - {"JetStreamOrderedConsWithErrors", test_JetStreamOrderedConsumerWithErrors}, - {"JetStreamOrderedConsAutoUnsub", test_JetStreamOrderedConsumerWithAutoUnsub}, - {"JetStreamOrderedConsSrvRestart", test_JetStreamOrderedConsSrvRestart}, - {"JetStreamSubscribeWithFWC", test_JetStreamSubscribeWithFWC}, - {"JetStreamStreamsSealAndRollup", test_JetStreamStreamsSealAndRollup}, - {"JetStreamGetMsgAndLastMsg", test_JetStreamGetMsgAndLastMsg}, - {"JetStreamConvertDirectMsg", test_JetStreamConvertDirectMsg}, - {"JetStreamDirectGetMsg", test_JetStreamDirectGetMsg}, - {"JetStreamNakWithDelay", test_JetStreamNakWithDelay}, - {"JetStreamBackOffRedeliveries", test_JetStreamBackOffRedeliveries}, - {"JetStreamInfoWithSubjects", test_JetStreamInfoWithSubjects}, - {"JetStreamInfoAlternates", test_JetStreamInfoAlternates}, - - {"KeyValueManager", test_KeyValueManager}, - {"KeyValueBasics", test_KeyValueBasics}, - {"KeyValueWatch", test_KeyValueWatch}, - {"KeyValueWatchMulti", test_KeyValueWatchMulti}, - {"KeyValueHistory", test_KeyValueHistory}, - {"KeyValueKeys", test_KeyValueKeys}, - {"KeyValueDeleteVsPurge", test_KeyValueDeleteVsPurge}, - {"KeyValueDeleteTombstones", test_KeyValueDeleteTombstones}, - {"KeyValueDeleteMarkerThreshold", test_KeyValuePurgeDeletesMarkerThreshold}, - {"KeyValueCrossAccount", test_KeyValueCrossAccount}, - {"KeyValueDiscardOldToNew", test_KeyValueDiscardOldToNew}, - {"KeyValueRePublish", test_KeyValueRePublish}, - {"KeyValueMirrorDirectGet", test_KeyValueMirrorDirectGet}, - {"KeyValueMirrorCrossDomains", test_KeyValueMirrorCrossDomains}, - - {"MicroMatchEndpointSubject", test_MicroMatchEndpointSubject}, - {"MicroAddService", test_MicroAddService}, - {"MicroGroups", test_MicroGroups}, - {"MicroBasics", test_MicroBasics}, - {"MicroStartStop", test_MicroStartStop}, - {"MicroServiceStopsOnClosedConn", test_MicroServiceStopsOnClosedConn}, - {"MicroServiceStopsWhenServerStops", test_MicroServiceStopsWhenServerStops}, - {"MicroAsyncErrorHandlerMaxPendingMsgs", test_MicroAsyncErrorHandler_MaxPendingMsgs}, - {"MicroAsyncErrorHandlerMaxPendingBytes", test_MicroAsyncErrorHandler_MaxPendingBytes }, - -#if defined(NATS_HAS_STREAMING) - {"StanPBufAllocator", test_StanPBufAllocator}, - {"StanConnOptions", test_StanConnOptions}, - {"StanSubOptions", test_StanSubOptions}, - {"StanMsg", test_StanMsg}, - {"StanServerNotReachable", test_StanServerNotReachable}, - {"StanBasicConnect", test_StanBasicConnect}, - {"StanConnectError", test_StanConnectError}, - {"StanBasicPublish", test_StanBasicPublish}, - {"StanBasicPublishAsync", test_StanBasicPublishAsync}, - {"StanPublishTimeout", test_StanPublishTimeout}, - {"StanPublishMaxAcksInflight", test_StanPublishMaxAcksInflight}, - {"StanBasicSubscription", test_StanBasicSubscription}, - {"StanSubscriptionCloseAndUnsub", test_StanSubscriptionCloseAndUnsubscribe}, - {"StanDurableSubscription", test_StanDurableSubscription}, - {"StanBasicQueueSubscription", test_StanBasicQueueSubscription}, - {"StanDurableQueueSubscription", test_StanDurableQueueSubscription}, - {"StanCheckReceivedMsg", test_StanCheckReceivedvMsg}, - {"StanSubscriptionAckMsg", test_StanSubscriptionAckMsg}, - {"StanPings", test_StanPings}, - {"StanPingsNoResponder", test_StanPingsNoResponder}, - {"StanConnectionLostHandlerNotSet", test_StanConnectionLostHandlerNotSet}, - {"StanPingsUnblockPublishCalls", test_StanPingsUnblockPubCalls}, - {"StanGetNATSConnection", test_StanGetNATSConnection}, - {"StanNoRetryOnFailedConnect", test_StanNoRetryOnFailedConnect}, - {"StanInternalSubsNotPooled", test_StanInternalSubsNotPooled}, - {"StanSubOnComplete", test_StanSubOnComplete}, - {"StanSubTimeout", test_StanSubTimeout}, - -#endif - -}; - -static int maxTests = (int) (sizeof(allTests)/sizeof(testInfo)); - -static void -generateList(void) -{ - FILE *list = fopen("list.txt", "w"); - int i; - - if (list == NULL) - { - printf("@@ Unable to create file 'list.txt': %d\n", errno); - return; - } - - printf("Number of tests: %d\n", maxTests); - - for (i=0; i= maxTests) - || (testEnd < 0) || (testEnd >= maxTests) - || (testStart > testEnd)) + if (argc != 2) { - printf("@@ Usage: %s [start] [end] (0 .. %d)\n", argv[0], (maxTests - 1)); + printf("@@ Usage: %s [testname]\n", argv[0]); return 1; } + testName = argv[1]; #ifndef _WIN32 signal(SIGSEGV, _sigsegv_handler); @@ -36702,14 +35724,25 @@ int main(int argc, char **argv) } // Execute tests - for (i=testStart; (i<=testEnd) && !failed; i++) + testFunc f = NULL; + for (i = 0; i < (int)(sizeof(allTests) / sizeof(*allTests)); i++) { + if (strcmp(testName, allTests[i].name) != 0) + continue; + #ifdef _WIN32 printf("\n== %s ==\n", allTests[i].name); #else printf("\033[0;34m\n== %s ==\n\033[0;0m", allTests[i].name); #endif - (*(allTests[i].func))(); + f = allTests[i].func; + f(); + break; + } + if (f == NULL) + { + printf("@@ Test '%s' not found!\n", testName); + return 1; } #ifdef _WIN32 diff --git a/test/test.h b/test/test.h new file mode 100644 index 000000000..6a9bc8edf --- /dev/null +++ b/test/test.h @@ -0,0 +1,360 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "natsp.h" +#include "comsock.h" + +#if defined(NATS_HAS_STREAMING) +static const char *clusterName = "test-cluster"; +#endif + +#ifdef _WIN32 +#define NATS_INVALID_PID (NULL) +#define LOGFILE_NAME "wserver.log" +#else +#define NATS_INVALID_PID (-1) +#define LOGFILE_NAME "server.log" +#endif + +#define FAIL(m) \ + { \ + printf("@@ %s @@\n", (m)); \ + failed = true; \ + return; \ + } + +#define CHECK_SERVER_STARTED(p) \ + if ((p) == NATS_INVALID_PID) \ + FAIL("Unable to start or verify that the server was started!") + +extern natsMutex *slMu; +extern natsHash *slMap; +extern bool keepServerOutput; +extern bool failed; + +static const char *natsServerExe = "nats-server"; + +static natsStatus +_checkStreamingStart(const char *url, int maxAttempts) +{ + natsStatus s = NATS_NOT_PERMITTED; + +#if defined(NATS_HAS_STREAMING) + + stanConnOptions *opts = NULL; + stanConnection *sc = NULL; + int attempts = 0; + + s = stanConnOptions_Create(&opts); + IFOK(s, stanConnOptions_SetURL(opts, url)); + IFOK(s, stanConnOptions_SetConnectionWait(opts, 250)); + if (s == NATS_OK) + { + while (((s = stanConnection_Connect(&sc, clusterName, "checkStart", opts)) != NATS_OK) && (attempts++ < maxAttempts)) + { + nats_Sleep(200); + } + } + + stanConnection_Destroy(sc); + stanConnOptions_Destroy(opts); + + if (s != NATS_OK) + nats_clearLastError(); +#else +#endif + return s; +} + +static natsStatus +_checkStart(const char *url, int orderIP, int maxAttempts) +{ + natsStatus s = NATS_OK; + natsUrl *nUrl = NULL; + int attempts = 0; + natsSockCtx ctx; + + natsSock_Init(&ctx); + ctx.orderIP = orderIP; + + natsDeadline_Init(&(ctx.writeDeadline), 2000); + + s = natsUrl_Create(&nUrl, url); + if (s == NATS_OK) + { + while (((s = natsSock_ConnectTcp(&ctx, + nUrl->host, nUrl->port)) != NATS_OK) && + (attempts++ < maxAttempts)) + { + nats_Sleep(200); + } + + natsUrl_Destroy(nUrl); + + if (s == NATS_OK) + natsSock_Close(ctx.fd); + else + s = NATS_NO_SERVER; + } + + nats_clearLastError(); + + return s; +} + +#ifdef _WIN32 + +typedef PROCESS_INFORMATION *natsPid; + +static HANDLE logHandle = NULL; + +static void +_stopServer(natsPid pid) +{ + if (pid == NATS_INVALID_PID) + return; + + TerminateProcess(pid->hProcess, 0); + WaitForSingleObject(pid->hProcess, INFINITE); + + CloseHandle(pid->hProcess); + CloseHandle(pid->hThread); + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Remove(slMap, (int64_t)pid); + natsMutex_Unlock(slMu); + + free(pid); +} + +static natsPid +_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) +{ + SECURITY_ATTRIBUTES sa; + STARTUPINFO si; + HANDLE h; + PROCESS_INFORMATION *pid; + DWORD flags = 0; + BOOL hInheritance = FALSE; + char *exeAndCmdLine = NULL; + int ret; + + pid = calloc(1, sizeof(PROCESS_INFORMATION)); + if (pid == NULL) + return NATS_INVALID_PID; + + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + + ret = nats_asprintf(&exeAndCmdLine, "%s%s%s", serverExe, + (cmdLineOpts != NULL ? " " : ""), + (cmdLineOpts != NULL ? cmdLineOpts : "")); + if (ret < 0) + { + printf("No memory allocating command line string!\n"); + free(pid); + return NATS_INVALID_PID; + } + + if (!keepServerOutput) + { + ZeroMemory(&sa, sizeof(sa)); + sa.nLength = sizeof(sa); + sa.lpSecurityDescriptor = NULL; + sa.bInheritHandle = TRUE; + + h = logHandle; + if (h == NULL) + { + h = CreateFile(LOGFILE_NAME, + GENERIC_WRITE, + FILE_SHARE_WRITE | FILE_SHARE_READ, + &sa, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + } + + si.dwFlags |= STARTF_USESTDHANDLES; + si.hStdInput = NULL; + si.hStdError = h; + si.hStdOutput = h; + + hInheritance = TRUE; + flags = CREATE_NO_WINDOW; + + if (logHandle == NULL) + logHandle = h; + } + + // Start the child process. + if (!CreateProcess(NULL, + (LPSTR)exeAndCmdLine, + NULL, // Process handle not inheritable + NULL, // Thread handle not inheritable + hInheritance, // Set handle inheritance + flags, // Creation flags + NULL, // Use parent's environment block + NULL, // Use parent's starting directory + &si, // Pointer to STARTUPINFO structure + pid)) // Pointer to PROCESS_INFORMATION structure + { + + printf("Unable to start '%s': error (%d).\n", + exeAndCmdLine, GetLastError()); + free(exeAndCmdLine); + return NATS_INVALID_PID; + } + + free(exeAndCmdLine); + + if (checkStart) + { + natsStatus s; + + if (strcmp(serverExe, natsServerExe) == 0) + s = _checkStart(url, 46, 10); + else + s = _checkStreamingStart(url, 10); + + if (s != NATS_OK) + { + _stopServer(pid); + return NATS_INVALID_PID; + } + } + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Set(slMap, (int64_t)pid, NULL, NULL); + natsMutex_Unlock(slMu); + + return (natsPid)pid; +} + +#else + +typedef pid_t natsPid; + +static void +_stopServer(natsPid pid) +{ + int status = 0; + + if (pid == NATS_INVALID_PID) + return; + + if (kill(pid, SIGINT) < 0) + { + perror("kill with SIGINT"); + if (kill(pid, SIGKILL) < 0) + { + perror("kill with SIGKILL"); + } + } + + waitpid(pid, &status, 0); + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Remove(slMap, (int64_t)pid); + natsMutex_Unlock(slMu); +} + +static natsPid +_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) +{ + natsPid pid = fork(); + if (pid == -1) + { + perror("fork"); + return NATS_INVALID_PID; + } + + if (pid == 0) + { + char *exeAndCmdLine = NULL; + char *argvPtrs[64]; + char *line = NULL; + int index = 0; + int ret = 0; + bool overrideAddr = false; + + if ((cmdLineOpts == NULL) || (strstr(cmdLineOpts, "-a ") == NULL)) + overrideAddr = true; + + ret = nats_asprintf(&exeAndCmdLine, "%s%s%s%s%s", serverExe, + (cmdLineOpts != NULL ? " " : ""), + (cmdLineOpts != NULL ? cmdLineOpts : ""), + (overrideAddr ? " -a 127.0.0.1" : ""), + (keepServerOutput ? "" : " -l " LOGFILE_NAME)); + if (ret < 0) + { + perror("No memory allocating command line string!\n"); + exit(1); + } + + memset(argvPtrs, 0, sizeof(argvPtrs)); + line = exeAndCmdLine; + + while (*line != '\0') + { + while ((*line == ' ') || (*line == '\t') || (*line == '\n')) + *line++ = '\0'; + + argvPtrs[index++] = line; + while ((*line != '\0') && (*line != ' ') && (*line != '\t') && (*line != '\n')) + { + line++; + } + } + argvPtrs[index++] = NULL; + + // Child process. Replace with NATS server + execvp(argvPtrs[0], argvPtrs); + perror("Exec failed: "); + exit(1); + } + else if (checkStart) + { + natsStatus s; + + if (strcmp(serverExe, natsServerExe) == 0) + s = _checkStart(url, 46, 10); + else + s = _checkStreamingStart(url, 10); + + if (s != NATS_OK) + { + _stopServer(pid); + return NATS_INVALID_PID; + } + } + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Set(slMap, (int64_t)pid, NULL, NULL); + natsMutex_Unlock(slMu); + + // parent, return the child's PID back. + return pid; +} +#endif + +static natsPid +_startServer(const char *url, const char *cmdLineOpts, bool checkStart) +{ + return _startServerImpl(natsServerExe, url, cmdLineOpts, checkStart); +} + From 613ec74b6504ea2dcf753d0cad9c424bf92e5963 Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Sat, 27 Jul 2024 17:48:26 -0600 Subject: [PATCH 05/13] Fixed test_JetStreamInfoAlternates flapper (#775) Signed-off-by: Ivan Kozlovic --- test/test.c | 55 +++++++++++++---------------------------------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/test/test.c b/test/test.c index 614e8b4da..cf0d3716a 100644 --- a/test/test.c +++ b/test/test.c @@ -30199,43 +30199,6 @@ void test_JetStreamInfoWithSubjects(void) JS_TEARDOWN; } -static natsStatus -_checkJSClusterReady(const char *url) -{ - natsStatus s = NATS_OK; - natsConnection *nc = NULL; - jsCtx *js = NULL; - jsErrCode jerr= 0; - int i; - jsOptions jo; - - jsOptions_Init(&jo); - jo.Wait = 1000; - - s = natsConnection_ConnectTo(&nc, url); - IFOK(s, natsConnection_JetStream(&js, nc, &jo)); - for (i=0; (s == NATS_OK) && (i<10); i++) - { - jsStreamInfo *si = NULL; - - s = js_GetStreamInfo(&si, js, "CHECK_CLUSTER", &jo, &jerr); - if (jerr == JSStreamNotFoundErr) - { - nats_clearLastError(); - s = NATS_OK; - break; - } - if ((s != NATS_OK) && (i < 9)) - { - s = NATS_OK; - nats_Sleep(500); - } - } - jsCtx_Destroy(js); - natsConnection_Destroy(nc); - return s; -} - void test_JetStreamInfoAlternates(void) { char datastore1[256] = {'\0'}; @@ -30251,6 +30214,7 @@ void test_JetStreamInfoAlternates(void) jsStreamConfig sc; jsStreamSource ss; natsStatus s; + int i; ENSURE_JS_VERSION(2, 9, 0); @@ -30271,10 +30235,6 @@ void test_JetStreamInfoAlternates(void) CHECK_SERVER_STARTED(pid1); testCond(true); - test("Check cluster: "); - s = _checkJSClusterReady("nats://127.0.0.1:4224"); - testCond(s == NATS_OK); - test("Connect: "); s = natsConnection_ConnectTo(&nc, NATS_DEFAULT_URL); testCond(s == NATS_OK); @@ -30288,7 +30248,18 @@ void test_JetStreamInfoAlternates(void) sc.Name = "TEST"; sc.Subjects = (const char*[1]){"foo"}; sc.SubjectsLen = 1; - s = js_AddStream(NULL, js, &sc, NULL, NULL); + // We will try up to 10 times to create the stream. It may fail if the cluster + // is not ready to accept the request. + for (i=0; (s == NATS_OK) && (i < 10); i++) + { + s = js_AddStream(NULL, js, &sc, NULL, NULL); + if ((s != NATS_OK) && (i < 9)) + { + nats_clearLastError(); + s = NATS_OK; + nats_Sleep(100); + } + } testCond(s == NATS_OK); test("Create mirror: "); From fbcddd113d800ff4041b16b129e60f559319ee0a Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Wed, 7 Aug 2024 07:39:35 -0600 Subject: [PATCH 06/13] [ADDED] TLS: natsOptions_TLSHandshakeFirst() (#780) * [ADDED] TLS: natsOptions_TLSHandshakeFirst() This is to force a client to perform the TLS handshake first, that is, not wait for the INFO protocol from the server. This is needed if the server is configured to require clients to perform the TLS handshake first (before sending the INFO protocol). Resolves #779 Signed-off-by: Ivan Kozlovic * Fixed flapper If the cluster was not fully established when the rest of the test was running, the library may get additional client URLs to reconnect to (gossip protocol). Looks like this was happening more on Windows CI than Linux. Using some options to clamp all that. Signed-off-by: Ivan Kozlovic --------- Signed-off-by: Ivan Kozlovic --- src/conn.c | 20 +++++++++++-- src/nats.h | 21 ++++++++++++-- src/natsp.h | 1 + src/opts.c | 21 ++++++++++++++ test/list_test.txt | 1 + test/test.c | 70 +++++++++++++++++++++++++++++++++++++++++++++- test/tlsfirst.conf | 16 +++++++++++ 7 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 test/tlsfirst.conf diff --git a/src/conn.c b/src/conn.c index 7cfe9beb1..710c0dd34 100644 --- a/src/conn.c +++ b/src/conn.c @@ -786,7 +786,12 @@ _checkForSecure(natsConnection *nc) } if ((s == NATS_OK) && nc->opts->secure) - s = _makeTLSConn(nc); + { + // If TLS handshake first is true, we have already done + // the handshake, so do it only if false. + if (!nc->opts->tlsHandshakeFirst) + s = _makeTLSConn(nc); + } return NATS_UPDATE_ERR_STACK(s); } @@ -1968,8 +1973,14 @@ _processConnInit(natsConnection *nc) nc->status = NATS_CONN_STATUS_CONNECTING; + // If we need to have a TLS connection and want the TLS handshake to occur + // first, do it now. + if (nc->opts->secure && nc->opts->tlsHandshakeFirst) + s = _makeTLSConn(nc); + // Process the INFO protocol that we should be receiving - s = _processExpectedInfo(nc); + if (s == NATS_OK) + s = _processExpectedInfo(nc); // Send the CONNECT and PING protocol, and wait for the PONG. if (s == NATS_OK) @@ -3280,6 +3291,11 @@ natsConn_create(natsConnection **newConn, natsOptions *options) nc->sockCtx.fd = NATS_SOCK_INVALID; nc->opts = options; + // If the TLSHandshakeFirst option is specified, make sure that + // the Secure boolean is true. + if (nc->opts->tlsHandshakeFirst) + nc->opts->secure = true; + nc->errStr[0] = '\0'; s = natsMutex_Create(&(nc->mu)); diff --git a/src/nats.h b/src/nats.h index dc331e08f..295af3a30 100644 --- a/src/nats.h +++ b/src/nats.h @@ -1043,7 +1043,7 @@ typedef struct jsConsumerNamesList */ typedef struct jsConsumerPauseResponse { - bool Paused; + bool Paused; int64_t PauseUntil; ///< UTC time expressed as number of nanoseconds since epoch. int64_t PauseRemaining; ///< Remaining time in nanoseconds. } jsConsumerPauseResponse; @@ -2327,6 +2327,23 @@ natsOptions_SetName(natsOptions *opts, const char *name); NATS_EXTERN natsStatus natsOptions_SetSecure(natsOptions *opts, bool secure); +/** \brief Performs TLS handshake first. + * + * If the server is not configured to require the client to perform + * the TLS handshake first, the server sends an INFO protocol first. + * When receiving it, the client and server are then initiate the + * TLS handshake. + * + * If the server is configured to require the client to perform + * the TLS handshake first, the client will fail to connect if + * not setting this option. Conversely, if the client is configured + * with this option but the server is not, the connection will fail. + * + * @param opts the pointer to the #natsOptions object. + */ +NATS_EXTERN natsStatus +natsOptions_TLSHandshakeFirst(natsOptions *opts); + /** \brief Loads the trusted CA certificates from a file. * * Loads the trusted CA certificates from a file. @@ -4039,7 +4056,7 @@ natsConnection_Connect(natsConnection **nc, natsOptions *options); * This means that all subscriptions and consumers should be resubscribed and * their work resumed after successful reconnect where all reconnect options are * respected. - * + * * @param nc the pointer to the #natsConnection object. */ natsStatus diff --git a/src/natsp.h b/src/natsp.h index b9887b5d3..1f7894ecd 100644 --- a/src/natsp.h +++ b/src/natsp.h @@ -224,6 +224,7 @@ struct __natsOptions bool pedantic; bool allowReconnect; bool secure; + bool tlsHandshakeFirst; int ioBufSize; int maxReconnect; int64_t reconnectWait; diff --git a/src/opts.c b/src/opts.c index 1c7864635..078691a38 100644 --- a/src/opts.c +++ b/src/opts.c @@ -363,6 +363,21 @@ natsOptions_SetSecure(natsOptions *opts, bool secure) return NATS_UPDATE_ERR_STACK(s); } +natsStatus +natsOptions_TLSHandshakeFirst(natsOptions *opts) +{ + natsStatus s = NATS_OK; + + LOCK_AND_CHECK_OPTIONS(opts, 0); + + opts->tlsHandshakeFirst = true; + opts->secure = true; + + UNLOCK_OPTS(opts); + + return NATS_UPDATE_ERR_STACK(s); +} + natsStatus natsOptions_LoadCATrustedCertificates(natsOptions *opts, const char *fileName) { @@ -689,6 +704,12 @@ natsOptions_SetSecure(natsOptions *opts, bool secure) return nats_setError(NATS_ILLEGAL_STATE, "%s", NO_SSL_ERR); } +natsStatus +natsOptions_TLSHandshakeFirst(natsOptions *opts) +{ + return nats_setError(NATS_ILLEGAL_STATE, "%s", NO_SSL_ERR); +} + natsStatus natsOptions_LoadCATrustedCertificates(natsOptions *opts, const char *fileName) { diff --git a/test/list_test.txt b/test/list_test.txt index 0804b84fb..4b2bd6cf0 100644 --- a/test/list_test.txt +++ b/test/list_test.txt @@ -246,6 +246,7 @@ _test(SSLBasic) _test(SSLCertAndKeyFromMemory) _test(SSLCiphers) _test(SSLConnectVerboseOption) +_test(SSLHandshakeFirst) _test(SSLLoadCAFromMemory) _test(SSLMultithreads) _test(SSLReconnectWithAuthError) diff --git a/test/test.c b/test/test.c index cf0d3716a..48cf966f8 100644 --- a/test/test.c +++ b/test/test.c @@ -2614,7 +2614,8 @@ void test_natsOptions(void) && (opts->writeDeadline == natsLib_defaultWriteDeadline()) && !opts->noEcho && !opts->retryOnFailedConnect - && !opts->ignoreDiscoveredServers) + && !opts->ignoreDiscoveredServers + && !opts->tlsHandshakeFirst); test("Add URL: "); s = natsOptions_SetURL(opts, "test"); @@ -2764,6 +2765,14 @@ void test_natsOptions(void) testCond((s == NATS_ILLEGAL_STATE) && (opts->secure == false)); #endif + test("Set TLSHandshakeFirst: "); + s = natsOptions_TLSHandshakeFirst(opts); +#if defined(NATS_HAS_TLS) + testCond((s == NATS_OK) && (opts->tlsHandshakeFirst == true) && (opts->secure == true)); +#else + testCond((s == NATS_ILLEGAL_STATE) && (opts->secure == false) && (opts->tlsHandshakeFirst == false)); +#endif + test("Set Pedantic: "); s = natsOptions_SetPedantic(opts, true); testCond((s == NATS_OK) && (opts->pedantic == true)); @@ -21197,6 +21206,62 @@ void test_SSLConnectVerboseOption(void) #endif } +void test_SSLHandshakeFirst(void) +{ +#if defined(NATS_HAS_TLS) + natsStatus s; + natsConnection *nc = NULL; + natsOptions *opts = NULL; + natsPid serverPid = NATS_INVALID_PID; + + serverPid = _startServer("nats://127.0.0.1:4443", "-config tlsfirst.conf", true); + CHECK_SERVER_STARTED(serverPid); + + test("Set options: "); + s = natsOptions_Create(&opts); + IFOK(s, natsOptions_SetURL(opts, "nats://127.0.0.1:4443")); + IFOK(s, natsOptions_SetSecure(opts, true)); + IFOK(s, natsOptions_SkipServerVerification(opts, true)); + IFOK(s, natsOptions_SetTimeout(opts, 500)); + testCond(s == NATS_OK); + + test("Check that connect fails if option not set: "); + s = natsConnection_Connect(&nc, opts); + testCond(s != NATS_OK); + nats_clearLastError(); + + test("Set TLSHandshakeFirst option error: "); + s = natsOptions_TLSHandshakeFirst(NULL); + testCond(s == NATS_INVALID_ARG); + nats_clearLastError(); + + test("Set TLSHandshakeFirst option: "); + s = natsOptions_TLSHandshakeFirst(opts); + testCond(s == NATS_OK); + + test("Check that connect succeeds: "); + s = natsConnection_Connect(&nc, opts); + testCond(s == NATS_OK); + natsConnection_Destroy(nc); + nc = NULL; + + _stopServer(serverPid); + serverPid = _startServer("nats://127.0.0.1:4443", "-config tls.conf", true); + CHECK_SERVER_STARTED(serverPid); + + test("Check that connect fails if option is set but not in the server: "); + s = natsConnection_Connect(&nc, opts); + testCond(s != NATS_OK); + nats_clearLastError(); + + natsOptions_Destroy(opts); + +#else + test("Skipped when built with no SSL support: "); + testCond(true); +#endif +} + #if defined(NATS_HAS_TLS) static natsStatus _elDummyAttach(void **userData, void *loop, natsConnection *nc, natsSock socket) { return NATS_OK; } @@ -21271,8 +21336,11 @@ void test_SSLReconnectWithAuthError(void) IFOK(s, natsOptions_SetTimeout(opts, 250)); IFOK(s, natsOptions_SetMaxReconnect(opts, 1000)); IFOK(s, natsOptions_SetReconnectWait(opts, 100)); + IFOK(s, natsOptions_SetReconnectJitter(opts, 0, 0)); IFOK(s, natsOptions_SetClosedCB(opts, _closedCb, (void*) &args)); IFOK(s, natsOptions_SetServers(opts, (const char*[2]){"tls://user:pwd@127.0.0.1:4443", "tls://bad:pwd@127.0.0.1:4444"}, 2)); + IFOK(s, natsOptions_SetNoRandomize(opts, true)); + IFOK(s, natsOptions_SetIgnoreDiscoveredServers(opts, true)); if (opts == NULL) FAIL("Unable to create reconnect options!"); diff --git a/test/tlsfirst.conf b/test/tlsfirst.conf new file mode 100644 index 000000000..5ed521c17 --- /dev/null +++ b/test/tlsfirst.conf @@ -0,0 +1,16 @@ + +# Simple TLS config file + +port: 4443 +net: "0.0.0.0" + +tls { + # Server cert + cert_file: "certs/server-cert.pem" + # Server private key + key_file: "certs/server-key.pem" + # Increase timeout for valgrind tests + timeout: 2 + # Force client to do the handshake first + handshake_first: true +} From 50ffc0ffd403d19aa5d189f2ff8eba94917c50a2 Mon Sep 17 00:00:00 2001 From: Lev <1187448+levb@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:55:38 -0700 Subject: [PATCH 07/13] squashed (#777) restore bench, CI --- .github/workflows/build-test.yml | 53 +++++---- test/bench_sub_async.c | 43 ++++--- test/diffstat_sub_async.go | 193 +++++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 36 deletions(-) create mode 100644 test/diffstat_sub_async.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 14ed168c8..e4eda5c70 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -181,13 +181,13 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} # verbose: true - - name: "Download benchmark results from ${{ github.event.pull_request.base.ref }}" + - name: "Cache for base benchmark for ${{ github.event.pull_request.base.ref }}" + id: cache-base-bench if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref - uses: actions/download-artifact@v2 + uses: actions/cache@v4 with: - name: benchmark_results_${{ github.event.pull_request.base.ref }} - path: ./build/prev_bench.log - continue-on-error: true + key: bench-${{ github.event.pull_request.base.ref }} + path: ./build/bench-${{ github.event.pull_request.base.ref }}.log - name: "Benchmark" if: inputs.benchmark == 'ON' @@ -197,24 +197,35 @@ jobs: export NATS_TEST_SERVER_VERSION="$(nats-server -v)" flags="" ctest -L 'bench' --timeout 600 -VV | tee bench.log - # - # ...coming: compare to base branch - - name: "Upload benchmark result for PR ${{ github.event.pull_request.head.ref }}" - if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref - uses: actions/upload-artifact@v4 + - name: "Checkout nats.c for ${{ github.event.pull_request.base.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref && steps.cache-base-bench.outputs.cache-hit != 'true' + uses: actions/checkout@v4 with: - name: benchmark_results_${{ github.event.pull_request.head.ref }} - path: ./build/bench.log + ref: ${{ github.event.pull_request.base.ref }} + clean: false - - name: Extract branch name - if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref - id: extract_branch - run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: "Benchmark ${{ github.event.pull_request.base.ref }} for comparison" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref && steps.cache-base-bench.outputs.cache-hit != 'true' + run: | + mkdir -p build + cd build + rm -rf CMakeFiles CMakeCache.txt + cmake .. ${{ steps.cmake-flags.outputs.flags }} + make rebuild_cache && make + export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH + export NATS_TEST_SERVER_VERSION="$(nats-server -v)" + flags="" + ctest -L 'bench' --timeout 600 -VV | tee bench-${{ github.event.pull_request.base.ref }}.log - - name: "Upload benchmark result for branch ${{ env.BRANCH_NAME }}" - if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref - uses: actions/upload-artifact@v4 + - name: "Checkout HEAD ${{ github.event.pull_request.head.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref && steps.cache-base-bench.outputs.cache-hit != 'true' + uses: actions/checkout@v4 with: - name: benchmark_results_${{ env.BRANCH_NAME }} - path: ./build/bench.log + clean: false + + - name: "Compare benchmark to ${{ github.event.pull_request.base.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref + run: | + cd build + go run ../test/diffstat_sub_async.go bench-${{ github.event.pull_request.base.ref }}.log bench.log diff --git a/test/bench_sub_async.c b/test/bench_sub_async.c index bda14a1d1..374f0178b 100644 --- a/test/bench_sub_async.c +++ b/test/bench_sub_async.c @@ -39,6 +39,7 @@ typedef natsStatus (*publishFunc)(natsConnection *nc, const char *subject, ENV * struct __env { + natsMutex *mu; int numSubs; threadConfig threads; int numPubMessages; @@ -72,18 +73,19 @@ void test_BenchSubscribeAsync_Small(void) threadConfig threads[] = { {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize {true, 1}, + {true, 2}, + {true, 3}, {true, 5}, - // These should show no material difference since no extra threads will be spun up {true, 7}, }; - int subs[] = {1, 2, 3, 5}; + int subs[] = {1, 2, 3, 7, 8, 13}; ENV env = { .pubf = _publish, .progressiveFlush = false, }; - RUN_MATRIX(threads, subs, 500 * 1000, &env); + RUN_MATRIX(threads, subs, 200 * 1000, &env); } // This benchmark publishes messages, flushing the connection every now and then @@ -93,21 +95,22 @@ void test_BenchSubscribeAsync_Small(void) void test_BenchSubscribeAsync_Large(void) { threadConfig threads[] = { - {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize - {true, 1}, - {true, 2}, + {false, 1}, + {true, 5}, {true, 11}, - {true, 163}, // to compare to non-pooled + {true, 23}, + {true, 47}, + {true, 91}, }; - int subs[] = {23, 83, 163}; + int subs[] = {1, 2, 23, 47, 81, 120}; ENV env = { .pubf = _publish, .progressiveFlush = true, }; - RUN_MATRIX(threads, subs, 500 * 1000, &env); + RUN_MATRIX(threads, subs, 100 * 1000, &env); } // This benchmark injects the messages directly into the relevant queue for @@ -125,13 +128,13 @@ void test_BenchSubscribeAsync_Inject(void) {true, 163}, }; - int subs[] = {1, 8, 23, 83, 163, 499}; + int subs[] = {1, 2, 3, 5, 10, 23, 83, 163, 499}; ENV env = { .pubf = _inject, }; - RUN_MATRIX(threads, subs, 1000 * 1000, &env); + RUN_MATRIX(threads, subs, 100 * 1000, &env); } // This benchmark injects the messages directly into the relevant queue for @@ -154,22 +157,27 @@ void test_BenchSubscribeAsync_InjectSlow(void) {true, 7}, {true, 11}, {true, 79}, - {true, 499}, + {true, 163}, }; - int subs[] = {1, 8, 12, 83, 163, 499}; + int subs[] = {1, 3, 7, 23, 83, 163, 499}; ENV env = { .pubf = _inject, .delayNano = 10 * 1000, // 10µs }; - RUN_MATRIX(threads, subs, 20 * 1000, &env); + RUN_MATRIX(threads, subs, 10000, &env); #endif // _WIN32 } static void _benchMatrix(threadConfig *threadsVector, int lent, int *subsVector, int lens, int NMessages, ENV *env) { + if (natsMutex_Create(&env->mu) != NATS_OK) + { + fprintf(stderr, "Error creating mutex\n"); + exit(1); + } printf("[\n"); for (int *sv = subsVector; sv < subsVector + lens; sv++) { @@ -222,6 +230,7 @@ static void _benchMatrix(threadConfig *threadsVector, int lent, int *subsVector, } } printf("]\n"); + natsMutex_Destroy(env->mu); } static natsStatus _bench(ENV *env, int *best, int *avg, int *worst) @@ -284,6 +293,7 @@ static natsStatus _bench(ENV *env, int *best, int *avg, int *worst) } b = w = a = 0; + natsMutex_Lock(env->mu); if (s == NATS_OK) { for (int i = 0; i < env->numSubs; i++) @@ -315,6 +325,7 @@ static natsStatus _bench(ENV *env, int *best, int *avg, int *worst) a += dur; } } + natsMutex_Unlock(env->mu); // cleanup for (int i = 0; i < env->numSubs; i++) @@ -362,7 +373,9 @@ static void _onMessage(natsConnection *nc, natsSubscription *sub, natsMsg *msg, static void _onComplete(void *closure) { subState *ss = (subState *)closure; + natsMutex_Lock(ss->env->mu); ss->closedTimestamp = nats_Now(); + natsMutex_Unlock(ss->env->mu); } static natsStatus _publish(natsConnection *nc, const char *subject, ENV *env) @@ -370,7 +383,7 @@ static natsStatus _publish(natsConnection *nc, const char *subject, ENV *env) natsStatus s = NATS_OK; char buf[16]; - int flushAfter = env->progressiveFlush ? env->numPubMessages / (env->numSubs * 2) : // trigger + int flushAfter = env->progressiveFlush ? env->numPubMessages / (env->numSubs * 3) : // trigger env->numPubMessages + 1; // do not trigger for (int i = 0; i < env->numPubMessages; i++) { diff --git a/test/diffstat_sub_async.go b/test/diffstat_sub_async.go new file mode 100644 index 000000000..67e039668 --- /dev/null +++ b/test/diffstat_sub_async.go @@ -0,0 +1,193 @@ +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "regexp" + "sort" + "strings" +) + +const NOISE_THRESHOLD = 0.03 + +type Key struct { + Subs int `json:"subs"` + Threads int `json:"threads"` + Messages int `json:"messages"` +} + +type TestData struct { + Key + Best int `json:"best"` + Average int `json:"average"` + Worst int `json:"worst"` +} + +type Diff struct { + Key + BaseAverage int `json:"base"` + BranchAverage int `json:"branch"` + Diff float64 `json:"diff"` +} + +type DiffData struct { + Records []Diff `json:"records"` + + Total struct { + BestDiff float64 `json:"best"` + AverageDiff float64 `json:"average"` + WorstDiff float64 `json:"worst"` + } `json:"total"` +} + +func main() { + flag.Parse() + + if len(os.Args) != 3 { + log.Fatalf("usage: %s
", os.Args[0]) + } + + m, err := readFile(os.Args[1]) + if err != nil { + log.Fatal(err) + } + b, err := readFile(os.Args[2]) + if err != nil { + log.Fatal(err) + } + + diff := map[string]*DiffData{} + for benchName := range b { + if _, ok := m[benchName]; !ok { + log.Printf("missing bench %s in main data", benchName) + continue + } + diff[benchName], err = calculateDiff(m[benchName], b[benchName]) + if err != nil { + log.Fatal(err) + } + } + + // bb, err := json.MarshalIndent(diff, "", " ") + // if err != nil { + // log.Fatal(err) + // } + // fmt.Println(string(bb)) + + for key, d := range diff { + fmt.Printf("== %s ==\n", key) + fmt.Printf("Best: %.2f%%\n", d.Total.BestDiff*100) + fmt.Printf("Average: %.2f%%\n", d.Total.AverageDiff*100) + fmt.Printf("Worst: %.2f%%\n", d.Total.WorstDiff*100) + fmt.Println() + for _, r := range d.Records { + fmt.Printf("subs=%d threads=%d messages=%d base=%d branch=%d diff=%.2f%%\n", + r.Subs, r.Threads, r.Messages, r.BaseAverage, r.BranchAverage, r.Diff*100) + } + } +} + +func calculateDiff(main, bench map[Key]TestData) (*DiffData, error) { + diff := DiffData{} + mBestSum, mAverageSum, mWorstSum := 0, 0, 0 + bBestSum, bAverageSum, bWorstSum := 0, 0, 0 + + for key, b := range bench { + m, ok := main[key] + if !ok { + log.Printf("warning: missing key %+v in main data", key) + continue + } + + // Exclude records with less than .5% difference from the output + d := float64(b.Average-m.Average) / float64(m.Average) + if d >= NOISE_THRESHOLD || d <= -NOISE_THRESHOLD { + diff.Records = append(diff.Records, Diff{ + Key: Key{ + Subs: m.Subs, + Threads: m.Threads, + Messages: m.Messages, + }, + BaseAverage: m.Average, + BranchAverage: b.Average, + Diff: d, + }) + } + + mBestSum += m.Best + mAverageSum += m.Average + mWorstSum += m.Worst + bBestSum += b.Best + bAverageSum += b.Average + bWorstSum += b.Worst + } + + sort.Slice(diff.Records, func(i, j int) bool { + return diff.Records[i].Diff > diff.Records[j].Diff + }) + + diff.Total.WorstDiff = float64(bWorstSum-mWorstSum) / float64(mWorstSum) + diff.Total.AverageDiff = float64(bAverageSum-mAverageSum) / float64(mAverageSum) + diff.Total.BestDiff = float64(bBestSum-mBestSum) / float64(mBestSum) + return &diff, nil +} + +func readFile(path string) (map[string]map[Key]TestData, error) { + r, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + scanner := bufio.NewScanner(r) + result := make(map[string]map[Key]TestData) + var benchName string + re := regexp.MustCompile(`^\d+: (.*)`) + + for scanner.Scan() { + line := scanner.Text() + + // ignore irrelevant lines, strip the test # prefix + matches := re.FindStringSubmatch(line) + if matches == nil { + continue + } + line = matches[1] + + if strings.HasPrefix(line, "== ") && strings.HasSuffix(line, " ==") { + benchName = strings.TrimSpace(strings.TrimPrefix(strings.TrimSuffix(line, " =="), "== ")) + continue + } + + line = strings.TrimPrefix(line, "\x1b[0;0m") + if strings.HasPrefix(line, "[") { + var data []TestData + jsonData := strings.Join([]string{line}, "") + for scanner.Scan() { + line := scanner.Text() + if matches := re.FindStringSubmatch(line); matches != nil { + line = matches[1] + jsonData += line + if strings.HasSuffix(line, "]") { + break + } + } + } + if err := json.Unmarshal([]byte(jsonData), &data); err != nil { + return nil, fmt.Errorf("%s: failed to parse JSON data: %w", path, err) + } + + hash := make(map[Key]TestData) + for _, d := range data { + hash[d.Key] = d + } + if benchName != "" { + result[benchName] = hash + } + } + } + + return result, nil +} From e56a792943bee153c0fc94e00e113b7039415762 Mon Sep 17 00:00:00 2001 From: Lev <1187448+levb@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:57:34 -0700 Subject: [PATCH 08/13] [CHANGED] refactored nats.c, prep for js_PullSubscribeAsync (#778) * squashed restore bench, CI * Refactor of nats.c, squashed * nit: moved into enqueueUserMessage * PR feedback --- .github/workflows/on-push-release.yml | 23 - src/CMakeLists.txt | 6 +- src/asynccb.c | 3 +- src/conn.c | 120 +- src/dispatch.c | 84 ++ src/dispatch.h | 55 + src/glib/glib.c | 491 ++++++ src/glib/glib.h | 44 + src/glib/glib_async_cb.c | 148 ++ src/glib/glib_dispatch_pool.c | 180 +++ src/glib/glib_gc.c | 125 ++ src/glib/glib_last_error.c | 349 +++++ src/glib/glib_ssl.c | 69 + src/glib/glib_timer.c | 332 +++++ src/glib/glibp.h | 145 ++ src/js.c | 107 +- src/msg.h | 3 +- src/nats.c | 1976 +++---------------------- src/nats.h | 41 +- src/natsp.h | 165 +-- src/natstime.c | 9 + src/natstime.h | 2 + src/opts.c | 15 +- src/stan/conn.c | 1 + src/sub.c | 660 ++++----- src/sub.h | 105 +- src/timer.c | 8 +- src/unix/sock.c | 2 +- src/unix/thread.c | 3 +- src/win/sock.c | 2 +- src/win/thread.c | 3 +- test/bench_sub_async.c | 75 +- test/list_test.txt | 2 +- test/test.c | 391 +++-- 34 files changed, 3028 insertions(+), 2716 deletions(-) create mode 100644 src/dispatch.c create mode 100644 src/dispatch.h create mode 100644 src/glib/glib.c create mode 100644 src/glib/glib.h create mode 100644 src/glib/glib_async_cb.c create mode 100644 src/glib/glib_dispatch_pool.c create mode 100644 src/glib/glib_gc.c create mode 100644 src/glib/glib_last_error.c create mode 100644 src/glib/glib_ssl.c create mode 100644 src/glib/glib_timer.c create mode 100644 src/glib/glibp.h diff --git a/.github/workflows/on-push-release.yml b/.github/workflows/on-push-release.yml index 50513575c..d47c20308 100644 --- a/.github/workflows/on-push-release.yml +++ b/.github/workflows/on-push-release.yml @@ -22,26 +22,3 @@ jobs: server_version: main ubuntu_version: ${{ matrix.ubuntu_version }} compiler: ${{ matrix.compiler }} - - dev-mode: - name: "DEV_MODE" - uses: ./.github/workflows/build-test.yml - with: - dev_mode: ON - server_version: main - verbose_test_output: ON - verbose_make_output: ON - - sanitize-addr: - name: "Sanitize address" - uses: ./.github/workflows/build-test.yml - with: - sanitize: address - server_version: main - - san-thread: - name: "Sanitize thread" - uses: ./.github/workflows/build-test.yml - with: - sanitize: thread - server_version: main diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a1351d82f..b18f72406 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,6 +3,7 @@ #------------------------ include_directories(include) include_directories(${NATS_PLATFORM_INCLUDE}) +include_directories(glib) if(NATS_BUILD_WITH_TLS) include_directories(${OPENSSL_INCLUDE_DIR}) @@ -24,6 +25,7 @@ endif(NATS_BUILD_USE_SODIUM) #--------------------------------------- file(GLOB SOURCES "*.c") file(GLOB PS_SOURCES "${NATS_PLATFORM_INCLUDE}/*.c") +file(GLOB GLIB_SOURCES "glib/*.c") # Add stan directory if building with Streaming support if(NATS_BUILD_STREAMING) @@ -34,7 +36,7 @@ endif(NATS_BUILD_STREAMING) # Create the shared and static libraries # -------------------------------------- if(NATS_BUILD_LIB_SHARED) - add_library(nats SHARED ${SOURCES} ${PS_SOURCES} ${S_SOURCES}) + add_library(nats SHARED ${SOURCES} ${GLIB_SOURCES} ${PS_SOURCES} ${S_SOURCES}) target_link_libraries(nats ${NATS_OPENSSL_LIBS} ${NATS_EXTRA_LIB} ${NATS_PROTOBUF_LIBRARIES} ${NATS_SODIUM_LIBRARIES}) set_target_properties(nats PROPERTIES VERSION ${NATS_VERSION_MAJOR}.${NATS_VERSION_MINOR}.${NATS_VERSION_PATCH} @@ -42,7 +44,7 @@ if(NATS_BUILD_LIB_SHARED) endif(NATS_BUILD_LIB_SHARED) if(NATS_BUILD_LIB_STATIC) - add_library(nats_static STATIC ${SOURCES} ${PS_SOURCES} ${S_SOURCES}) + add_library(nats_static STATIC ${SOURCES} ${GLIB_SOURCES} ${PS_SOURCES} ${S_SOURCES}) target_link_libraries(nats_static ${NATS_OPENSSL_LIBS} ${NATS_PROTOBUF_LIBRARIES} ${NATS_SODIUM_LIBRARIES}) set_target_properties(nats_static PROPERTIES VERSION ${NATS_VERSION_MAJOR}.${NATS_VERSION_MINOR}.${NATS_VERSION_PATCH} diff --git a/src/asynccb.c b/src/asynccb.c index 57d20ceff..11c589ad8 100644 --- a/src/asynccb.c +++ b/src/asynccb.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -18,6 +18,7 @@ #if defined(NATS_HAS_STREAMING) #include "stan/conn.h" #endif +#include "glib/glib.h" static void _freeAsyncCbInfo(natsAsyncCbInfo *info) diff --git a/src/conn.c b/src/conn.c index 710c0dd34..54bb697dd 100644 --- a/src/conn.c +++ b/src/conn.c @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -35,6 +35,7 @@ #include "nkeys.h" #include "crypto.h" #include "js.h" +#include "glib/glib.h" #define DEFAULT_SCRATCH_SIZE (512) #define MAX_INFO_MESSAGE_SIZE (32768) @@ -1141,31 +1142,19 @@ _resendSubscriptions(natsConnection *nc) sub = subs[i]; adjustedMax = 0; - natsSub_Lock(sub); - if (sub->libDlvWorker != NULL) - { - natsMutex_Lock(sub->libDlvWorker->lock); - } + nats_lockSubAndDispatcher(sub); // If JS ordered consumer, trigger a reset. Don't check the error // condition here. If there is a failure, it will be retried // at the next HB interval. if ((sub->jsi != NULL) && (sub->jsi->ordered)) { jsSub_resetOrderedConsumer(sub, sub->jsi->sseq+1); - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); continue; } if (natsSub_drainStarted(sub)) { - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); continue; } if (sub->max > 0) @@ -1177,11 +1166,7 @@ _resendSubscriptions(natsConnection *nc) // messages have reached the max, if so, unsubscribe. if (adjustedMax == 0) { - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); s = natsConn_sendUnsubProto(nc, sub->sid, 0); continue; } @@ -1193,11 +1178,7 @@ _resendSubscriptions(natsConnection *nc) // Hold the lock up to that point so we are sure not to resend // any SUB/UNSUB for a subscription that is in draining mode. - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } NATS_FREE(subs); @@ -2678,11 +2659,8 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) natsStatus s = NATS_OK; natsSubscription *sub = NULL; natsMsg *msg = NULL; - natsMsgDlvWorker *ldw = NULL; bool sc = false; bool sm = false; - nats_MsgList *list = NULL; - natsCondition *cond = NULL; // For JetStream cases jsSub *jsi = NULL; bool ctrlMsg = false; @@ -2728,34 +2706,17 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) // We need to retain the subscription since as soon as we release the // nc->subsMu lock, the subscription could be destroyed and we would // reference freed memory. - natsSubAndLdw_LockAndRetain(sub); + nats_lockRetainSubAndDispatcher(sub); natsMutex_Unlock(nc->subsMu); if (sub->closed || sub->drainSkip) { - natsSubAndLdw_UnlockAndRelease(sub); + nats_unlockReleaseSubAndDispatcher(sub); natsMsg_Destroy(msg); return NATS_OK; } - // Pick condition variable and list based on if the sub is - // part of a global delivery thread pool or not. - // Note about `list`: this is used only to link messages, but - // sub->msgList needs to be used to update/check number of pending - // messages, since in case of delivery thread pool, `list` will have - // messages from many different subscriptions. - if ((ldw = sub->libDlvWorker) != NULL) - { - cond = ldw->cond; - list = &(ldw->msgList); - } - else - { - cond = sub->cond; - list = &(sub->msgList); - } - jsi = sub->jsi; // For JS subscriptions (but not pull ones), handle hearbeat and flow control here. if (jsi && !jsi->pull) @@ -2775,7 +2736,7 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) s = jsSub_checkOrderedMsg(sub, msg, &replaced); if ((s != NATS_OK) || replaced) { - natsSubAndLdw_UnlockAndRelease(sub); + nats_unlockReleaseSubAndDispatcher(sub); natsMsg_Destroy(msg); return s; } @@ -2784,58 +2745,27 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) if (!ctrlMsg) { - sub->msgList.msgs++; - sub->msgList.bytes += bufLen; - - if (((sub->msgsLimit > 0) && (sub->msgList.msgs > sub->msgsLimit)) - || ((sub->bytesLimit > 0) && (sub->msgList.bytes > sub->bytesLimit))) - { - natsMsg_Destroy(msg); - - sub->dropped++; - - sc = !sub->slowConsumer; - sub->slowConsumer = true; - - // Undo stats from above. - sub->msgList.msgs--; - sub->msgList.bytes -= bufLen; - } - else + s = natsSub_enqueueUserMessage(sub, msg); + if (s == NATS_OK) { - bool signal= false; - - if ((jsi != NULL) && jsi->ackNone) - natsMsg_setAcked(msg); - - if (sub->msgList.msgs > sub->msgsMax) - sub->msgsMax = sub->msgList.msgs; - - if (sub->msgList.bytes > sub->bytesMax) - sub->bytesMax = sub->msgList.bytes; - sub->slowConsumer = false; - msg->sub = sub; - - if (list->head == NULL) - { - list->head = msg; - signal = true; - } - else - list->tail->next = msg; - - list->tail = msg; - - if (signal) - natsCondition_Signal(cond); - // Store the ACK metadata from the message to // compare later on with the received heartbeat. if (jsi != NULL) s = jsSub_trackSequences(jsi, msg->reply); } + else + { + // Slow consumer is the only reason to fail here. Handle it and + // reset the status, so we continue. + natsMsg_Destroy(msg); + sub->dropped++; + sc = !sub->slowConsumer; + sub->slowConsumer = true; + + s = NATS_OK; + } } else if ((jct == jsCtrlHeartbeat) && (msg->reply == NULL)) { @@ -2859,9 +2789,9 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) // If we are going to post to the error handler, do not release yet. if (sc || sm) - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); else - natsSubAndLdw_UnlockAndRelease(sub); + nats_unlockReleaseSubAndDispatcher(sub); if ((s == NATS_OK) && fcReply) s = natsConnection_Publish(nc, fcReply, NULL, 0); diff --git a/src/dispatch.c b/src/dispatch.c new file mode 100644 index 000000000..7284ae5ef --- /dev/null +++ b/src/dispatch.c @@ -0,0 +1,84 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "natsp.h" + +#include +#include + +#include "mem.h" +#include "conn.h" +#include "sub.h" +#include "glib/glib.h" + +// sub and dispatcher locks must be held. +void natsSub_enqueueMessage(natsSubscription *sub, natsMsg *msg) +{ + bool signal = false; + natsDispatchQueue *q = &sub->dispatcher->queue; + + if (q->head == NULL) + { + signal = true; + msg->next = NULL; + q->head = msg; + } + else + { + q->tail->next = msg; + } + q->tail = msg; + q->msgs++; + q->bytes += natsMsg_dataAndHdrLen(msg); + + if (signal) + natsCondition_Signal(sub->dispatcher->cond); +} + +// sub and dispatcher locks must be held. +natsStatus natsSub_enqueueUserMessage(natsSubscription *sub, natsMsg *msg) +{ + natsDispatchQueue *toQ = &sub->dispatcher->queue; + natsDispatchQueue *statsQ = &sub->ownDispatcher.queue; + int newMsgs = statsQ->msgs + 1; + int newBytes = statsQ->bytes + natsMsg_dataAndHdrLen(msg); + + msg->sub = sub; + + if (((sub->msgsLimit > 0) && (newMsgs > sub->msgsLimit)) || + ((sub->bytesLimit > 0) && (newBytes > sub->bytesLimit))) + { + return NATS_SLOW_CONSUMER; + } + sub->slowConsumer = false; + + if (newMsgs > sub->msgsMax) + sub->msgsMax = newMsgs; + if (newBytes > sub->bytesMax) + sub->bytesMax = newBytes; + + if ((sub->jsi != NULL) && sub->jsi->ackNone) + natsMsg_setAcked(msg); + + // Update the subscription stats if separate, the queue stats will be + // updated below. + if (toQ != statsQ) + { + statsQ->msgs++; + statsQ->bytes += natsMsg_dataAndHdrLen(msg); + } + + natsSub_enqueueMessage(sub, msg); + return NATS_OK; +} + diff --git a/src/dispatch.h b/src/dispatch.h new file mode 100644 index 000000000..9662a5533 --- /dev/null +++ b/src/dispatch.h @@ -0,0 +1,55 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DISPATCH_H_ +#define DISPATCH_H_ + +typedef struct __natsDispatchQueue_s +{ + natsMsg *head; + natsMsg *tail; + int msgs; + int bytes; +} natsDispatchQueue; + +typedef struct __natsDispatcher_s +{ + // When created as a dedicated dispatcher for a subscription, we use the + // sub's mutex (for performance? TODO: benchmack), so there is special + // handling for mu in the code. + natsSubscription *dedicatedTo; + natsMutex *mu; + + natsThread *thread; + natsCondition *cond; + natsDispatchQueue queue; + + // Flags. + bool running; + bool shutdown; +} natsDispatcher; + +static inline void nats_destroyQueuedMessages(natsDispatchQueue *queue) +{ + natsMsg *next = NULL; + for (natsMsg *msg = queue->head; msg != NULL; msg = next) + { + next = msg->next; + natsMsg_Destroy(msg); + } +} + +void natsSub_enqueueMessage(natsSubscription *sub, natsMsg *msg); +natsStatus natsSub_enqueueUserMessage(natsSubscription *sub, natsMsg *msg); + +#endif /* DISPATCH_H_ */ diff --git a/src/glib/glib.c b/src/glib/glib.c new file mode 100644 index 000000000..4b8e4bc31 --- /dev/null +++ b/src/glib/glib.c @@ -0,0 +1,491 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +#include "../util.h" +#include "../crypto.h" + +int64_t gLockSpinCount = 2000; + +static natsInitOnceType gInitOnce = NATS_ONCE_STATIC_INIT; +static natsLib gLib; + +natsLib* nats_lib(void) +{ + return &gLib; +} + +static void _freeLib(void); +static void _finalCleanup(void); + +void natsLib_Retain(void) +{ + natsMutex_Lock(gLib.lock); + + gLib.refs++; + + natsMutex_Unlock(gLib.lock); +} + +void natsLib_Release(void) +{ + int refs = 0; + + natsMutex_Lock(gLib.lock); + + refs = --(gLib.refs); + + natsMutex_Unlock(gLib.lock); + + if (refs == 0) + _freeLib(); +} + +static void +_finalCleanup(void) +{ + if (gLib.sslInitialized) + { +#if defined(NATS_HAS_TLS) +#if !defined(NATS_USE_OPENSSL_1_1) + ERR_free_strings(); + EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); + ERR_remove_thread_state(0); +#endif + sk_SSL_COMP_free(SSL_COMP_get_compression_methods()); +#endif + natsThreadLocal_DestroyKey(gLib.sslTLKey); + } + + natsThreadLocal_DestroyKey(gLib.errTLKey); + natsThreadLocal_DestroyKey(gLib.natsThreadKey); + natsMutex_Destroy(gLib.lock); + gLib.lock = NULL; +} + +static void +natsLib_Destructor(void) +{ + int refs = 0; + + if (!(gLib.wasOpenedOnce)) + return; + + // Destroy thread locals for the current thread. + nats_ReleaseThreadMemory(); + + // Do the final cleanup if possible + natsMutex_Lock(gLib.lock); + refs = gLib.refs; + if (refs > 0) + { + // If some thread is still around when the process exits and has a + // reference to the library, then don't do the final cleanup now. + // If the process has not fully exited when the lib's last reference + // is decremented, the final cleanup will be executed from that thread. + gLib.finalCleanup = true; + } + natsMutex_Unlock(gLib.lock); + + if (refs != 0) + return; + + _finalCleanup(); +} + +static void +_freeLib(void) +{ + const unsigned int offset = (unsigned int)offsetof(natsLib, refs); + bool callFinalCleanup = false; + + nats_freeTimers(&gLib); + nats_freeAsyncCbs(&gLib); + nats_freeGC(&gLib); + + nats_freeDispatcherPool(&gLib.messageDispatchers); + nats_freeDispatcherPool(&gLib.replyDispatchers); + + natsNUID_free(); + natsMutex_Destroy(gLib.service_callback_mu); + natsHash_Destroy(gLib.all_services_to_callback); + + natsCondition_Destroy(gLib.cond); + + memset((void *)(offset + (char *)&gLib), 0, sizeof(natsLib) - offset); + + natsMutex_Lock(gLib.lock); + callFinalCleanup = gLib.finalCleanup; + if (gLib.closeCompleteCond != NULL) + { + if (gLib.closeCompleteSignal) + { + *gLib.closeCompleteBool = true; + natsCondition_Signal(gLib.closeCompleteCond); + } + gLib.closeCompleteCond = NULL; + gLib.closeCompleteBool = NULL; + gLib.closeCompleteSignal = false; + } + gLib.closed = false; + gLib.initialized = false; + gLib.finalCleanup = false; + natsMutex_Unlock(gLib.lock); + + if (callFinalCleanup) + _finalCleanup(); +} + +static void +_destroyErrTL(void *localStorage) +{ + natsTLError *err = (natsTLError *)localStorage; + NATS_FREE(err); +} + +static void +_doInitOnce(void) +{ + natsStatus s; + + memset(&gLib, 0, sizeof(natsLib)); + + s = natsMutex_Create(&(gLib.lock)); + if (s == NATS_OK) + s = natsThreadLocal_CreateKey(&(gLib.errTLKey), _destroyErrTL); + if (s == NATS_OK) + s = natsThreadLocal_CreateKey(&(gLib.natsThreadKey), NULL); + if (s != NATS_OK) + { + fprintf(stderr, "FATAL ERROR: Unable to initialize library!\n"); + fflush(stderr); + abort(); + } + + nats_initForOS(); + + // Setup a hook for when the process exits. + atexit(natsLib_Destructor); +} + + +static void +_libTearDown(void) +{ + nats_waitForDispatcherPoolShutdown(&gLib.messageDispatchers); + nats_waitForDispatcherPoolShutdown(&gLib.replyDispatchers); + + if (gLib.timers.thread != NULL) + natsThread_Join(gLib.timers.thread); + + if (gLib.asyncCbs.thread != NULL) + natsThread_Join(gLib.asyncCbs.thread); + + if (gLib.gc.thread != NULL) + natsThread_Join(gLib.gc.thread); + + natsLib_Release(); +} + +// environment variables will override the default options. +natsStatus +nats_openLib(natsClientConfig *config) +{ + natsStatus s = NATS_OK; + + natsClientConfig defaultConfig = { + .LockSpinCount = -1, + .ThreadPoolMax = 1, + }; + if (config == NULL) + config = &defaultConfig; + + if (!nats_InitOnce(&gInitOnce, _doInitOnce)) + return NATS_FAILED_TO_INITIALIZE; + + natsMutex_Lock(gLib.lock); + + if (gLib.closed || gLib.initialized || gLib.initializing) + { + if (gLib.closed) + s = NATS_FAILED_TO_INITIALIZE; + else if (gLib.initializing) + s = NATS_ILLEGAL_STATE; + + natsMutex_Unlock(gLib.lock); + return s; + } + + gLib.initializing = true; + gLib.initAborted = false; + +#if !defined(_WIN32) + signal(SIGPIPE, SIG_IGN); +#endif + + srand((unsigned int)nats_NowInNanoSeconds()); + + gLib.refs = 1; + + // If the caller specifies negative value, then we use the default + if (config->LockSpinCount >= 0) + gLockSpinCount = config->LockSpinCount; + + gLib.config = *config; + nats_Base32_Init(); + + s = natsCondition_Create(&(gLib.cond)); + + if (s == NATS_OK) + s = natsCrypto_Init(); + + if (s == NATS_OK) + s = natsMutex_Create(&(gLib.timers.lock)); + if (s == NATS_OK) + s = natsCondition_Create(&(gLib.timers.cond)); + if (s == NATS_OK) + { + s = natsThread_Create(&(gLib.timers.thread), nats_timerThreadf, &gLib); + if (s == NATS_OK) + gLib.refs++; + } + + if (s == NATS_OK) + s = natsMutex_Create(&(gLib.asyncCbs.lock)); + if (s == NATS_OK) + s = natsCondition_Create(&(gLib.asyncCbs.cond)); + if (s == NATS_OK) + { + s = natsThread_Create(&(gLib.asyncCbs.thread), nats_asyncCbsThreadf, &gLib); + if (s == NATS_OK) + gLib.refs++; + } + if (s == NATS_OK) + s = natsMutex_Create(&(gLib.gc.lock)); + if (s == NATS_OK) + s = natsCondition_Create(&(gLib.gc.cond)); + if (s == NATS_OK) + { + s = natsThread_Create(&(gLib.gc.thread), nats_garbageCollectorThreadf, &gLib); + if (s == NATS_OK) + gLib.refs++; + } + if (s == NATS_OK) + s = natsNUID_init(); + + if (s == NATS_OK) + s = nats_initDispatcherPool(&(gLib.messageDispatchers), config->ThreadPoolMax); + if (s == NATS_OK) + s = nats_initDispatcherPool(&(gLib.replyDispatchers), config->ReplyThreadPoolMax); + + if (s == NATS_OK) + s = natsMutex_Create(&gLib.service_callback_mu); + if (s == NATS_OK) + s = natsHash_Create(&gLib.all_services_to_callback, 8); + + if (s == NATS_OK) + gLib.initialized = true; + + // In case of success or error, broadcast so that lib's threads + // can proceed. + if (gLib.cond != NULL) + { + if (s != NATS_OK) + { + gLib.initAborted = true; + gLib.timers.shutdown = true; + gLib.asyncCbs.shutdown = true; + gLib.gc.shutdown = true; + } + natsCondition_Broadcast(gLib.cond); + } + + gLib.initializing = false; + gLib.wasOpenedOnce = true; + + natsMutex_Unlock(gLib.lock); + + if (s != NATS_OK) + _libTearDown(); + + return s; +} + +natsStatus +nats_closeLib(bool wait, int64_t timeout) +{ + natsStatus s = NATS_OK; + natsCondition *cond = NULL; + bool complete = false; + // int i; + + // This is to protect against a call to nats_Close() while there + // was no prior call to nats_Open(), either directly or indirectly. + if (!nats_InitOnce(&gInitOnce, _doInitOnce)) + return NATS_ERR; + + natsMutex_Lock(gLib.lock); + + if (gLib.closed || !gLib.initialized) + { + bool closed = gLib.closed; + + natsMutex_Unlock(gLib.lock); + + if (closed) + return NATS_ILLEGAL_STATE; + return NATS_NOT_INITIALIZED; + } + if (wait) + { + if (natsThreadLocal_Get(gLib.natsThreadKey) != NULL) + s = NATS_ILLEGAL_STATE; + if (s == NATS_OK) + s = natsCondition_Create(&cond); + if (s != NATS_OK) + { + natsMutex_Unlock(gLib.lock); + return s; + } + gLib.closeCompleteCond = cond; + gLib.closeCompleteBool = &complete; + gLib.closeCompleteSignal = true; + } + + gLib.closed = true; + + natsMutex_Lock(gLib.timers.lock); + gLib.timers.shutdown = true; + natsCondition_Signal(gLib.timers.cond); + natsMutex_Unlock(gLib.timers.lock); + + natsMutex_Lock(gLib.asyncCbs.lock); + gLib.asyncCbs.shutdown = true; + natsCondition_Signal(gLib.asyncCbs.cond); + natsMutex_Unlock(gLib.asyncCbs.lock); + + natsMutex_Lock(gLib.gc.lock); + gLib.gc.shutdown = true; + natsCondition_Signal(gLib.gc.cond); + natsMutex_Unlock(gLib.gc.lock); + + natsMutex_Unlock(gLib.lock); + + nats_signalDispatcherPoolToShutdown(&gLib.messageDispatchers); + nats_signalDispatcherPoolToShutdown(&gLib.replyDispatchers); + + nats_ReleaseThreadMemory(); + _libTearDown(); + + if (wait) + { + natsMutex_Lock(gLib.lock); + while ((s != NATS_TIMEOUT) && !complete) + { + if (timeout <= 0) + natsCondition_Wait(cond, gLib.lock); + else + s = natsCondition_TimedWait(cond, gLib.lock, timeout); + } + if (s != NATS_OK) + gLib.closeCompleteSignal = false; + natsMutex_Unlock(gLib.lock); + + natsCondition_Destroy(cond); + } + + return s; +} + +void nats_setNATSThreadKey(void) +{ + natsThreadLocal_Set(gLib.natsThreadKey, (const void *)1); +} + +void nats_ReleaseThreadMemory(void) +{ + void *tl = NULL; + + if (!(gLib.wasOpenedOnce)) + return; + + tl = natsThreadLocal_Get(gLib.errTLKey); + if (tl != NULL) + { + _destroyErrTL(tl); + natsThreadLocal_SetEx(gLib.errTLKey, NULL, false); + } + + tl = NULL; + + natsMutex_Lock(gLib.lock); + if (gLib.sslInitialized) + { + tl = natsThreadLocal_Get(gLib.sslTLKey); + if (tl != NULL) + { + nats_cleanupThreadSSL(tl); + natsThreadLocal_SetEx(gLib.sslTLKey, NULL, false); + } + } + natsMutex_Unlock(gLib.lock); +} + +natsStatus +natsLib_startServiceCallbacks(microService *m) +{ + natsStatus s; + + natsMutex_Lock(gLib.service_callback_mu); + s = natsHash_Set(gLib.all_services_to_callback, (int64_t)m, (void *)m, NULL); + natsMutex_Unlock(gLib.service_callback_mu); + + return NATS_UPDATE_ERR_STACK(s); +} + +void natsLib_stopServiceCallbacks(microService *m) +{ + if (m == NULL) + return; + + natsMutex_Lock(gLib.service_callback_mu); + natsHash_Remove(gLib.all_services_to_callback, (int64_t)m); + natsMutex_Unlock(gLib.service_callback_mu); +} + +natsMutex * +natsLib_getServiceCallbackMutex(void) +{ + return gLib.service_callback_mu; +} + +natsHash * +natsLib_getAllServicesToCallback(void) +{ + return gLib.all_services_to_callback; +} + +natsClientConfig *nats_testInspectClientConfig(void) +{ + // Immutable after startup + return &gLib.config; +} + +void nats_overrideDefaultOptionsWithConfig(natsOptions *opts) +{ + opts->writeDeadline = gLib.config.DefaultWriteDeadline; + opts->useSharedDispatcher = gLib.config.DefaultToThreadPool; + opts->useSharedReplyDispatcher = gLib.config.DefaultRepliesToThreadPool; +} diff --git a/src/glib/glib.h b/src/glib/glib.h new file mode 100644 index 000000000..96afbe78b --- /dev/null +++ b/src/glib/glib.h @@ -0,0 +1,44 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GLIB_H_ +#define GLIB_H_ + +typedef struct __natsDispatcherPool natsDispatcherPool; + +// Functions that are used internally by the library. + +void natsLib_Retain(void); +void natsLib_Release(void); + +natsStatus nats_assignSubToDispatch(natsSubscription *sub); +natsStatus nats_closeLib(bool wait, int64_t timeout); +natsStatus nats_initSSL(void); +natsStatus nats_openLib(natsClientConfig *config); +natsStatus nats_postAsyncCbInfo(natsAsyncCbInfo *info); +natsStatus nats_setMessageDispatcherPoolCap(int max); + +void nats_initForOS(void); +void nats_overrideDefaultOptionsWithConfig(natsOptions *opts); +void nats_resetTimer(natsTimer *t, int64_t newInterval); +void nats_stopTimer(natsTimer *t); + +// Returns the number of timers that have been created and not stopped. +int nats_getTimersCount(void); + +// Returns the number of timers actually in the list. This should be +// equal to nats_getTimersCount() or nats_getTimersCount() - 1 when a +// timer thread is invoking a timer's callback. +int nats_getTimersCountInList(void); + +#endif // GLIB_H_ diff --git a/src/glib/glib_async_cb.c b/src/glib/glib_async_cb.c new file mode 100644 index 000000000..8b2646539 --- /dev/null +++ b/src/glib/glib_async_cb.c @@ -0,0 +1,148 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +void +nats_freeAsyncCbs(natsLib *lib) +{ + natsLibAsyncCbs *cbs = &(lib->asyncCbs); + + natsThread_Destroy(cbs->thread); + natsCondition_Destroy(cbs->cond); + natsMutex_Destroy(cbs->lock); +} + + +void +nats_asyncCbsThreadf(void *arg) +{ + natsLib *lib = (natsLib*) arg; + natsLibAsyncCbs *asyncCbs = &(lib->asyncCbs); + natsAsyncCbInfo *cb = NULL; + natsConnection *nc = NULL; +#if defined(NATS_HAS_STREAMING) + stanConnection *sc = NULL; +#endif + + WAIT_LIB_INITIALIZED(lib); + + natsMutex_Lock(asyncCbs->lock); + + // We want all callbacks to be invoked even on shutdown + while (true) + { + while (((cb = asyncCbs->head) == NULL) && !asyncCbs->shutdown) + natsCondition_Wait(asyncCbs->cond, asyncCbs->lock); + + if ((cb == NULL) && asyncCbs->shutdown) + break; + + asyncCbs->head = cb->next; + + if (asyncCbs->tail == cb) + asyncCbs->tail = NULL; + + cb->next = NULL; + + natsMutex_Unlock(asyncCbs->lock); + + nc = cb->nc; +#if defined(NATS_HAS_STREAMING) + sc = cb->sc; +#endif + + switch (cb->type) + { + case ASYNC_CLOSED: + { + (*(nc->opts->closedCb))(nc, nc->opts->closedCbClosure); + if (nc->opts->microClosedCb != NULL) + (*(nc->opts->microClosedCb))(nc, NULL); + break; + } + + case ASYNC_DISCONNECTED: + (*(nc->opts->disconnectedCb))(nc, nc->opts->disconnectedCbClosure); + break; + case ASYNC_RECONNECTED: + (*(nc->opts->reconnectedCb))(nc, nc->opts->reconnectedCbClosure); + break; + case ASYNC_CONNECTED: + (*(nc->opts->connectedCb))(nc, nc->opts->connectedCbClosure); + break; + case ASYNC_DISCOVERED_SERVERS: + (*(nc->opts->discoveredServersCb))(nc, nc->opts->discoveredServersClosure); + break; + case ASYNC_LAME_DUCK_MODE: + (*(nc->opts->lameDuckCb))(nc, nc->opts->lameDuckClosure); + break; + case ASYNC_ERROR: + { + if (cb->errTxt != NULL) + nats_setErrStatusAndTxt(cb->err, cb->errTxt); + + (*(nc->opts->asyncErrCb))(nc, cb->sub, cb->err, nc->opts->asyncErrCbClosure); + if (nc->opts->microAsyncErrCb != NULL) + (*(nc->opts->microAsyncErrCb))(nc, cb->sub, cb->err, NULL); + break; + } +#if defined(NATS_HAS_STREAMING) + case ASYNC_STAN_CONN_LOST: + (*(sc->opts->connectionLostCB))(sc, sc->connLostErrTxt, sc->opts->connectionLostCBClosure); + break; +#endif + default: + break; + } + + natsAsyncCb_Destroy(cb); + nats_clearLastError(); + + natsMutex_Lock(asyncCbs->lock); + } + + natsMutex_Unlock(asyncCbs->lock); + + natsLib_Release(); +} + +natsStatus +nats_postAsyncCbInfo(natsAsyncCbInfo *info) +{ + natsLib *lib = nats_lib(); + natsMutex_Lock(lib->asyncCbs.lock); + + if (lib->asyncCbs.shutdown) + { + natsMutex_Unlock(lib->asyncCbs.lock); + return NATS_NOT_INITIALIZED; + } + + info->next = NULL; + + if (lib->asyncCbs.head == NULL) + lib->asyncCbs.head = info; + + if (lib->asyncCbs.tail != NULL) + lib->asyncCbs.tail->next = info; + + lib->asyncCbs.tail = info; + + natsCondition_Signal(lib->asyncCbs.cond); + + natsMutex_Unlock(lib->asyncCbs.lock); + + return NATS_OK; +} + diff --git a/src/glib/glib_dispatch_pool.c b/src/glib/glib_dispatch_pool.c new file mode 100644 index 000000000..5f4be28e4 --- /dev/null +++ b/src/glib/glib_dispatch_pool.c @@ -0,0 +1,180 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" +#include "../sub.h" + +static inline void +_destroyDispatcher(natsDispatcher *d) +{ + if ((d == NULL) || !d->running) + return; + + natsThread_Destroy(d->thread); + nats_destroyQueuedMessages(&d->queue); // there's NEVER anything there, remove? + natsCondition_Destroy(d->cond); + natsMutex_Destroy(d->mu); + memset(d, 0, sizeof(*d)); +} + +static inline natsStatus +_startDispatcher(natsDispatcher *d, void (*threadf)(void *)) +{ + natsStatus s = NATS_OK; + + if (d->running) + return NATS_OK; + + s = natsMutex_Create(&d->mu); + if (s != NATS_OK) + return s; + + natsCondition_Create(&d->cond); + + natsMutex_Lock(d->mu); + natsLib_Retain(); + s = natsThread_Create(&d->thread, threadf, (void *)d); + if (s == NATS_OK) + d->running = true; + natsMutex_Unlock(d->mu); + + if (s != NATS_OK) + { + _destroyDispatcher(d); + natsLib_Release(); + } + return NATS_UPDATE_ERR_STACK(s); +} + +static natsStatus +_growPool(natsDispatcherPool *pool, int cap) +{ + natsStatus s = NATS_OK; + + if (cap <= 0) + return nats_setError(NATS_ERR, "%s", "Pool size cannot be negative or zero"); + + // Do not error on max < workers->maxSize in case we allow shrinking + // the pool in the future. Make it a no-op for now. + if (cap > pool->cap) + { + natsDispatcher *newDispatchers = NATS_CALLOC(cap, sizeof(natsDispatcher)); + if (newDispatchers == NULL) + s = nats_setDefaultError(NATS_NO_MEMORY); + if (s == NATS_OK) + { + memcpy( + newDispatchers, + pool->dispatchers, + pool->cap * sizeof(*newDispatchers)); + NATS_FREE(pool->dispatchers); + pool->dispatchers = newDispatchers; + pool->cap = cap; + } + } + return NATS_UPDATE_ERR_STACK(s); +} + +void nats_freeDispatcherPool(natsDispatcherPool *pool) +{ + for (int i = 0; i < pool->cap; i++) + _destroyDispatcher(&pool->dispatchers[i]); + natsMutex_Destroy(pool->lock); + NATS_FREE(pool->dispatchers); + memset(pool, 0, sizeof(*pool)); +} + +natsStatus +nats_initDispatcherPool(natsDispatcherPool *pool, int cap) +{ + natsStatus s = NATS_OK; + + memset(pool, 0, sizeof(*pool)); + + s = natsMutex_Create(&pool->lock); + if (cap > 0) + IFOK(s, _growPool(pool, cap)); + + if (s != NATS_OK) + nats_freeDispatcherPool(pool); + return NATS_UPDATE_ERR_STACK(s); +} + +void nats_signalDispatcherPoolToShutdown(natsDispatcherPool *pool) +{ + natsCondition *cond = NULL; + + for (int i = 0; i < pool->cap; i++) + { + // These are no-ops for empty slots + nats_lockDispatcher(&pool->dispatchers[i]); + pool->dispatchers[i].shutdown = true; + cond = pool->dispatchers[i].cond; + if (cond != NULL) + natsCondition_Signal(cond); + + nats_unlockDispatcher(&pool->dispatchers[i]); + } +} + +void nats_waitForDispatcherPoolShutdown(natsDispatcherPool *pool) +{ + for (int i = 0; i < pool->cap; i++) + { + if (pool->dispatchers[i].thread != NULL) + natsThread_Join(pool->dispatchers[i].thread); + } +} + +natsStatus nats_setMessageDispatcherPoolCap(int max) +{ + natsLib *lib = nats_lib(); + + natsMutex_Lock(lib->messageDispatchers.lock); + natsStatus s = _growPool(&lib->messageDispatchers, max); + natsMutex_Unlock(lib->messageDispatchers.lock); + + return NATS_UPDATE_ERR_STACK(s); +} + +// no lock on sub->mu needed because we are called during subscription creation. +natsStatus +nats_assignSubToDispatch(natsSubscription *sub) +{ + natsLib *lib = nats_lib(); + natsStatus s = NATS_OK; + natsDispatcher *d = NULL; + natsDispatcherPool *pool = &lib->messageDispatchers; + + natsMutex_Lock(pool->lock); + + if (pool->cap == 0) + s = nats_setError(NATS_FAILED_TO_INITIALIZE, "%s", "No message dispatchers available, the pool is empty."); + + if (s == NATS_OK) + { + // Get the next dispatcher + d = &pool->dispatchers[pool->useNext]; + pool->useNext = (pool->useNext + 1) % pool->cap; + } + if ((s == NATS_OK) && (d->thread == NULL)) + s = _startDispatcher(d, nats_deliverMsgsPoolf); + + // Assign it to the sub. + if (s == NATS_OK) + sub->dispatcher = d; + + natsMutex_Unlock(pool->lock); + + return NATS_UPDATE_ERR_STACK(s); +} diff --git a/src/glib/glib_gc.c b/src/glib/glib_gc.c new file mode 100644 index 000000000..aa82a13cd --- /dev/null +++ b/src/glib/glib_gc.c @@ -0,0 +1,125 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + + +void +nats_freeGC(natsLib *lib) +{ + natsGCList *gc = &(lib->gc); + + natsThread_Destroy(gc->thread); + natsCondition_Destroy(gc->cond); + natsMutex_Destroy(gc->lock); +} + + +void +nats_garbageCollectorThreadf(void *closure) +{ + natsLib *lib = (natsLib*) closure; + natsGCList *gc = &(lib->gc); + natsGCItem *item; + natsGCItem *list; + + WAIT_LIB_INITIALIZED(lib); + + natsMutex_Lock(gc->lock); + + // Process all elements in the list, even on shutdown + while (true) + { + // Go into wait until we are notified to shutdown + // or there is something to garbage collect + gc->inWait = true; + + while (!(gc->shutdown) && (gc->head == NULL)) + { + natsCondition_Wait(gc->cond, gc->lock); + } + + // Out of the wait. Setting this boolean avoids unnecessary + // signaling when an item is added to the collector. + gc->inWait = false; + + // Do not break out on shutdown here, we want to clear the list, + // even on exit so that valgrind and the like are happy. + + // Under the lock, we will switch to a local list and reset the + // GC's list (so that others can add to the list without contention + // (at least from the GC itself). + do + { + list = gc->head; + gc->head = NULL; + + natsMutex_Unlock(gc->lock); + + // Now that we are outside of the lock, we can empty the list. + while ((item = list) != NULL) + { + // Pops item from the beginning of the list. + list = item->next; + item->next = NULL; + + // Invoke the freeCb associated with this object + (*(item->freeCb))((void*) item); + } + + natsMutex_Lock(gc->lock); + } + while (gc->head != NULL); + + // If we were ask to shutdown and since the list is now empty, exit + if (gc->shutdown) + break; + } + + natsMutex_Unlock(gc->lock); + + natsLib_Release(); +} + +bool +natsGC_collect(natsGCItem *item) +{ + natsGCList *gc; + bool signal; + + // If the object was not setup for garbage collection, return false + // so the caller frees the object. + if (item->freeCb == NULL) + return false; + + gc = &(nats_lib()->gc); + + natsMutex_Lock(gc->lock); + + // We will signal only if the GC is in the condition wait. + signal = gc->inWait; + + // Add to the front of the list. + item->next = gc->head; + + // Update head. + gc->head = item; + + if (signal) + natsCondition_Signal(gc->cond); + + natsMutex_Unlock(gc->lock); + + return true; +} + diff --git a/src/glib/glib_last_error.c b/src/glib/glib_last_error.c new file mode 100644 index 000000000..6ff9bddb7 --- /dev/null +++ b/src/glib/glib_last_error.c @@ -0,0 +1,349 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +static natsTLError* +_getThreadError(void) +{ + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + bool needFree = false; + + // The library should already be initialized, but let's protect against + // situations where foo() invokes bar(), which invokes baz(), which + // invokes nats_Open(). If that last call fails, when we un-wind down + // to foo(), it may be difficult to know that nats_Open() failed and + // that we should not try to invoke natsLib_setError. So we check again + // here that the library has been initialized properly, and if not, we + // simply don't set the error. + if (nats_Open(-1) != NATS_OK) + return NULL; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if (errTL == NULL) + { + errTL = (natsTLError*) NATS_CALLOC(1, sizeof(natsTLError)); + if (errTL != NULL) + errTL->framesCount = -1; + needFree = (errTL != NULL); + + } + + if ((errTL != NULL) + && (natsThreadLocal_SetEx(lib->errTLKey, + (const void*) errTL, false) != NATS_OK)) + { + if (needFree) + NATS_FREE(errTL); + + errTL = NULL; + } + + return errTL; +} + +static char* +_getErrorShortFileName(const char* fileName) +{ + char *file = strstr(fileName, "src"); + + if (file != NULL) + file = (file + 4); + else + file = (char*) fileName; + + return file; +} + +static void +_updateStack(natsTLError *errTL, const char *funcName, natsStatus errSts, + bool calledFromSetError) +{ + int idx; + + idx = errTL->framesCount; + if ((idx >= 0) + && (idx < MAX_FRAMES) + && (strcmp(errTL->func[idx], funcName) == 0)) + { + return; + } + + // In case no error was already set... + if ((errTL->framesCount == -1) && !calledFromSetError) + errTL->sts = errSts; + + idx = ++(errTL->framesCount); + + if (idx >= MAX_FRAMES) + return; + + errTL->func[idx] = funcName; +} + +#if !defined(_WIN32) +__attribute__ ((format (printf, 5, 6))) +#endif +natsStatus +nats_setErrorReal(const char *fileName, const char *funcName, int line, natsStatus errSts, const char *errTxtFmt, ...) +{ + natsTLError *errTL = _getThreadError(); + char tmp[256]; + va_list ap; + int n; + + if ((errTL == NULL) || errTL->skipUpdate) + return errSts; + + errTL->sts = errSts; + errTL->framesCount = -1; + + tmp[0] = '\0'; + + va_start(ap, errTxtFmt); + nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); + va_end(ap); + + if (strlen(tmp) > 0) + { + n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", + _getErrorShortFileName(fileName), line, tmp); + if ((n < 0) || (n >= (int) sizeof(errTL->text))) + { + int pos = ((int) strlen(errTL->text)) - 1; + int i; + + for (i=0; i<3; i++) + errTL->text[pos--] = '.'; + } + } + + _updateStack(errTL, funcName, errSts, true); + + return errSts; +} + +#if !defined(_WIN32) +__attribute__ ((format (printf, 4, 5))) +#endif +void +nats_updateErrTxt(const char *fileName, const char *funcName, int line, const char *errTxtFmt, ...) +{ + natsTLError *errTL = _getThreadError(); + char tmp[256]; + va_list ap; + int n; + + if ((errTL == NULL) || errTL->skipUpdate) + return; + + tmp[0] = '\0'; + + va_start(ap, errTxtFmt); + nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); + va_end(ap); + + if (strlen(tmp) > 0) + { + n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", + _getErrorShortFileName(fileName), line, tmp); + if ((n < 0) || (n >= (int) sizeof(errTL->text))) + { + int pos = ((int) strlen(errTL->text)) - 1; + int i; + + for (i=0; i<3; i++) + errTL->text[pos--] = '.'; + } + } +} + +void +nats_setErrStatusAndTxt(natsStatus err, const char *errTxt) +{ + natsTLError *errTL = _getThreadError(); + + if ((errTL == NULL) || errTL->skipUpdate) + return; + + errTL->sts = err; + snprintf(errTL->text, sizeof(errTL->text), "%s", errTxt); + errTL->framesCount = -1; +} + +natsStatus +nats_updateErrStack(natsStatus err, const char *func) +{ + natsTLError *errTL = _getThreadError(); + + if ((errTL == NULL) || errTL->skipUpdate) + return err; + + _updateStack(errTL, func, err, false); + + return err; +} + +void +nats_clearLastError(void) +{ + natsTLError *errTL = _getThreadError(); + + if ((errTL == NULL) || errTL->skipUpdate) + return; + + errTL->sts = NATS_OK; + errTL->text[0] = '\0'; + errTL->framesCount = -1; +} + +void +nats_doNotUpdateErrStack(bool skipStackUpdate) +{ + natsTLError *errTL = _getThreadError(); + + if (errTL == NULL) + return; + + if (skipStackUpdate) + { + errTL->skipUpdate++; + } + else + { + errTL->skipUpdate--; + assert(errTL->skipUpdate >= 0); + } +} + +const char* +nats_GetLastError(natsStatus *status) +{ + natsStatus s; + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + + if (status != NULL) + *status = NATS_OK; + + // Ensure the library is loaded + s = nats_Open(-1); + if (s != NATS_OK) + return NULL; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if ((errTL == NULL) || (errTL->sts == NATS_OK)) + return NULL; + + if (status != NULL) + *status = errTL->sts; + + return errTL->text; +} + +natsStatus +nats_GetLastErrorStack(char *buffer, size_t bufLen) +{ + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + int offset = 0; + int i, max, n, len; + + if ((buffer == NULL) || (bufLen == 0)) + return NATS_INVALID_ARG; + + buffer[0] = '\0'; + len = (int) bufLen; + + // Ensure the library is loaded + if (nats_Open(-1) != NATS_OK) + return NATS_FAILED_TO_INITIALIZE; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) + return NATS_OK; + + max = errTL->framesCount; + if (max >= MAX_FRAMES) + max = MAX_FRAMES - 1; + + for (i=0; (i<=max) && (len > 0); i++) + { + n = snprintf(buffer + offset, len, "%s%s", + errTL->func[i], + (i < max ? "\n" : "")); + // On Windows, n will be < 0 if len is not big enough. + if (n < 0) + { + len = 0; + } + else + { + offset += n; + len -= n; + } + } + + if ((max != errTL->framesCount) && (len > 0)) + { + n = snprintf(buffer + offset, len, "\n%d more...", + errTL->framesCount - max); + // On Windows, n will be < 0 if len is not big enough. + if (n < 0) + len = 0; + else + len -= n; + } + + if (len <= 0) + return NATS_INSUFFICIENT_BUFFER; + + return NATS_OK; +} + +void +nats_PrintLastErrorStack(FILE *file) +{ + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + int i, max; + + // Ensure the library is loaded + if (nats_Open(-1) != NATS_OK) + return; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) + return; + + fprintf(file, "Error: %u - %s", + errTL->sts, natsStatus_GetText(errTL->sts)); + if (errTL->text[0] != '\0') + fprintf(file, " - %s", errTL->text); + fprintf(file, "\n"); + fprintf(file, "Stack: (library version: %s)\n", nats_GetVersion()); + + max = errTL->framesCount; + if (max >= MAX_FRAMES) + max = MAX_FRAMES - 1; + + for (i=0; i<=max; i++) + fprintf(file, " %02d - %s\n", (i+1), errTL->func[i]); + + if (max != errTL->framesCount) + fprintf(file, " %d more...\n", errTL->framesCount - max); + + fflush(file); +} + diff --git a/src/glib/glib_ssl.c b/src/glib/glib_ssl.c new file mode 100644 index 000000000..e7470aae8 --- /dev/null +++ b/src/glib/glib_ssl.c @@ -0,0 +1,69 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + + +void +nats_cleanupThreadSSL(void *localStorage) +{ +#if defined(NATS_HAS_TLS) && !defined(NATS_USE_OPENSSL_1_1) + ERR_remove_thread_state(0); +#endif +} + +void nats_sslRegisterThreadForCleanup(void) +{ +#if defined(NATS_HAS_TLS) + natsLib *lib = nats_lib(); + // Set anything. The goal is that at thread exit, the thread local key + // will have something non NULL associated, which will trigger the + // destructor that we have registered. + (void)natsThreadLocal_Set(lib->sslTLKey, (void *)1); +#endif +} + +natsStatus +nats_initSSL(void) +{ + natsLib *lib = nats_lib(); + natsStatus s = NATS_OK; + + // Ensure the library is loaded + s = nats_openLib(NULL); + if (s != NATS_OK) + return s; + + natsMutex_Lock(lib->lock); + + if (!lib->sslInitialized) + { + // Regardless of success, mark as initialized so that we + // can do cleanup on exit. + lib->sslInitialized = true; + +#if defined(NATS_HAS_TLS) +#if !defined(NATS_USE_OPENSSL_1_1) + // Initialize SSL. + SSL_library_init(); + SSL_load_error_strings(); +#endif +#endif + s = natsThreadLocal_CreateKey(&(lib->sslTLKey), nats_cleanupThreadSSL); + } + + natsMutex_Unlock(lib->lock); + + return NATS_UPDATE_ERR_STACK(s); +} + diff --git a/src/glib/glib_timer.c b/src/glib/glib_timer.c new file mode 100644 index 000000000..ba99480a6 --- /dev/null +++ b/src/glib/glib_timer.c @@ -0,0 +1,332 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +void nats_freeTimers(natsLib *lib) +{ + natsLibTimers *timers = &(lib->timers); + + natsThread_Destroy(timers->thread); + natsCondition_Destroy(timers->cond); + natsMutex_Destroy(timers->lock); +} + +static void +_insertTimer(natsTimer *t) +{ + natsTimer *cur = nats_lib()->timers.timers; + natsTimer *prev = NULL; + + while ((cur != NULL) && (cur->absoluteTime <= t->absoluteTime)) + { + prev = cur; + cur = cur->next; + } + + if (cur != NULL) + { + t->prev = prev; + t->next = cur; + cur->prev = t; + + if (prev != NULL) + prev->next = t; + } + else if (prev != NULL) + { + prev->next = t; + t->prev = prev; + t->next = NULL; + } + + if (prev == NULL) + nats_lib()->timers.timers = t; +} + +// Locks must be held before entering this function +static inline void +_removeTimer(natsLib *lib, natsTimer *t) +{ + // Switch flag + t->stopped = true; + + // It the timer was in the callback, it has already been removed from the + // list, so skip that. + if (!(t->inCallback)) + { + if (t->prev != NULL) + t->prev->next = t->next; + if (t->next != NULL) + t->next->prev = t->prev; + + if (t == lib->timers.timers) + lib->timers.timers = t->next; + + t->prev = NULL; + t->next = NULL; + } + + // Decrease the global count of timers + lib->timers.count--; +} + +void +nats_resetTimer(natsTimer *t, int64_t newInterval) +{ + natsLib *lib = nats_lib(); + natsLibTimers *timers = &(lib->timers); + + natsMutex_Lock(timers->lock); + natsMutex_Lock(t->mu); + + // If timer is active, we need first to remove it. This call does the + // right thing if the timer is in the callback. + if (!(t->stopped)) + _removeTimer(lib, t); + + // Bump the timer's global count (it as decreased in the _removeTimers call + timers->count++; + + // Switch stopped flag + t->stopped = false; + + // Set the new interval (may be same than it was before, but that's ok) + t->interval = newInterval; + + // If the timer is in the callback, the insertion and setting of the + // absolute time will be done by the timer thread when returning from + // the timer's callback. + if (!(t->inCallback)) + { + t->absoluteTime = nats_setTargetTime(t->interval); + _insertTimer(t); + } + + natsMutex_Unlock(t->mu); + + if (!(timers->changed)) + natsCondition_Signal(timers->cond); + + timers->changed = true; + + natsMutex_Unlock(timers->lock); +} + +void +nats_stopTimer(natsTimer *t) +{ + natsLib *lib = nats_lib(); + natsLibTimers *timers = &(lib->timers); + bool doCb = false; + + natsMutex_Lock(timers->lock); + natsMutex_Lock(t->mu); + + // If the timer was already stopped, nothing to do. + if (t->stopped) + { + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + return; + } + + _removeTimer(lib, t); + + doCb = (!(t->inCallback) && (t->stopCb != NULL)); + + natsMutex_Unlock(t->mu); + + if (!(timers->changed)) + natsCondition_Signal(timers->cond); + + timers->changed = true; + + natsMutex_Unlock(timers->lock); + + if (doCb) + (*(t->stopCb))(t, t->closure); +} + +int +nats_getTimersCount(void) +{ + natsLib *lib = nats_lib(); + int count = 0; + + natsMutex_Lock(lib->timers.lock); + + count = lib->timers.count; + + natsMutex_Unlock(lib->timers.lock); + + return count; +} + +int +nats_getTimersCountInList(void) +{ + natsLib *lib = nats_lib(); + int count = 0; + natsTimer *t; + + natsMutex_Lock(lib->timers.lock); + + t = lib->timers.timers; + while (t != NULL) + { + count++; + t = t->next; + } + + natsMutex_Unlock(lib->timers.lock); + + return count; +} + +void nats_timerThreadf(void *arg) +{ + natsLib *lib = (natsLib *)arg; + natsLibTimers *timers = &lib->timers; + natsTimer *t = NULL; + natsStatus s = NATS_OK; + bool doStopCb; + int64_t target; + + WAIT_LIB_INITIALIZED(lib); + + natsMutex_Lock(timers->lock); + + while (!(timers->shutdown)) + { + // Take the first timer that needs to fire. + t = timers->timers; + + if (t == NULL) + { + // No timer, fire in an hour... + target = nats_setTargetTime(3600 * 1000); + } + else + { + target = t->absoluteTime; + } + + timers->changed = false; + + s = NATS_OK; + + while (!(timers->shutdown) + && (s != NATS_TIMEOUT) + && !(timers->changed)) + { + s = natsCondition_AbsoluteTimedWait(timers->cond, timers->lock, + target); + } + + if (timers->shutdown) + break; + + if ((t == NULL) || timers->changed) + continue; + + natsMutex_Lock(t->mu); + + // Remove timer from the list: + timers->timers = t->next; + if (t->next != NULL) + t->next->prev = NULL; + + t->prev = NULL; + t->next = NULL; + + t->inCallback = true; + + // Retain the timer, since we are going to release the locks for the + // callback. The user may "destroy" the timer from there, so we need + // to be protected with reference counting. + t->refs++; + + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + (*(t->cb))(t, t->closure); + + natsMutex_Lock(timers->lock); + natsMutex_Lock(t->mu); + + t->inCallback = false; + + // Timer may have been stopped from within the callback, or during + // the window the locks were released. + doStopCb = (t->stopped && (t->stopCb != NULL)); + + // If not stopped, we need to put it back in our list + if (!doStopCb) + { + // Reset our view of what is the time this timer should fire + // because: + // 1- the callback may have taken longer than it should + // 2- the user may have called Reset() with a new interval + t->absoluteTime = nats_setTargetTime(t->interval); + _insertTimer(t); + } + + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + if (doStopCb) + (*(t->stopCb))(t, t->closure); + + // Compensate for the retain that we made before invoking the timer's + // callback + natsTimer_Release(t); + + natsMutex_Lock(timers->lock); + } + + // Process the timers that were left in the list (not stopped) when the + // library is shutdown. + while ((t = timers->timers) != NULL) + { + natsMutex_Lock(t->mu); + + // Check if we should invoke the callback. Note that although we are + // releasing the locks below, a timer present in the list here is + // guaranteed not to have been stopped (because it would not be in + // the list otherwise, since there is no chance that it is in the + // timer's callback). So just check if there is a stopCb to invoke. + doStopCb = (t->stopCb != NULL); + + // Remove the timer from the list. + _removeTimer(lib, t); + + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + // Invoke the callback now. + if (doStopCb) + (*(t->stopCb))(t, t->closure); + + // No release of the timer here. The user is still responsible to call + // natsTimer_Destroy(). + + natsMutex_Lock(timers->lock); + } + + natsMutex_Unlock(timers->lock); + + natsLib_Release(); +} + diff --git a/src/glib/glibp.h b/src/glib/glibp.h new file mode 100644 index 000000000..c6c4a8bb2 --- /dev/null +++ b/src/glib/glibp.h @@ -0,0 +1,145 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GLIBP_H_ +#define GLIBP_H_ + +#include "../natsp.h" +#include "../mem.h" +#if defined(NATS_HAS_STREAMING) +#include "../stan/stanp.h" +#endif + +#include +#include + +#include "glib.h" + +#define WAIT_LIB_INITIALIZED(_l) \ + natsMutex_Lock((_l)->lock); \ + while (!((_l)->initialized) && !((_l)->initAborted)) \ + natsCondition_Wait((_l)->cond, (_l)->lock); \ + natsMutex_Unlock((_l)->lock) + +typedef struct natsTLError +{ + natsStatus sts; + char text[256]; + const char *func[MAX_FRAMES]; + int framesCount; + int skipUpdate; + +} natsTLError; + +typedef struct __natsLibTimers +{ + natsMutex *lock; + natsCondition *cond; + natsThread *thread; + natsTimer *timers; + int count; + bool changed; + bool shutdown; + +} natsLibTimers; + +typedef struct __natsLibAsyncCbs +{ + natsMutex *lock; + natsCondition *cond; + natsThread *thread; + natsAsyncCbInfo *head; + natsAsyncCbInfo *tail; + bool shutdown; + +} natsLibAsyncCbs; + +typedef struct __natsGCList +{ + natsMutex *lock; + natsCondition *cond; + natsThread *thread; + natsGCItem *head; + bool shutdown; + bool inWait; + +} natsGCList; + +struct __natsDispatcherPool +{ + natsMutex *lock; + int useNext; // index of next dispatcher to use + int cap; // maximum number of concurrent dispatchers allowed + + natsDispatcher *dispatchers; + +}; + +typedef struct __natsLib +{ + // Leave these fields before 'refs' + natsMutex *lock; + volatile bool wasOpenedOnce; + bool sslInitialized; + natsThreadLocal errTLKey; + natsThreadLocal sslTLKey; + natsThreadLocal natsThreadKey; + bool initialized; + bool closed; + natsCondition *closeCompleteCond; + bool *closeCompleteBool; + bool closeCompleteSignal; + bool finalCleanup; + // Do not move 'refs' without checking _freeLib() + int refs; + + bool initializing; + bool initAborted; + + natsClientConfig config; + natsDispatcherPool messageDispatchers; + natsDispatcherPool replyDispatchers; + + natsLibTimers timers; + natsLibAsyncCbs asyncCbs; + + natsCondition *cond; + + natsGCList gc; + + // For micro services code + natsMutex *service_callback_mu; + // uses `microService*` as the key and the value. + natsHash *all_services_to_callback; + +} natsLib; + +natsLib *nats_lib(void); + +void nats_freeTimers(natsLib *lib); +void nats_timerThreadf(void *arg); // arg is &gLib + +void nats_freeAsyncCbs(natsLib *lib); +void nats_asyncCbsThreadf(void *arg); // arg is &gLib + +void nats_freeGC(natsLib *lib); +void nats_garbageCollectorThreadf(void *arg); // arg is &gLib + +natsStatus nats_initDispatcherPool(natsDispatcherPool *pool, int cap); +void nats_freeDispatcherPool(natsDispatcherPool *pool); +void nats_signalDispatcherPoolToShutdown(natsDispatcherPool *pool); +void nats_waitForDispatcherPoolShutdown(natsDispatcherPool *pool); + +void nats_cleanupThreadSSL(void *localStorage); + +#endif // GLIBP_H_ diff --git a/src/js.c b/src/js.c index 221640738..42a404275 100644 --- a/src/js.c +++ b/src/js.c @@ -1,4 +1,4 @@ -// Copyright 2021-2022 The NATS Authors +// Copyright 2021-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,6 +19,8 @@ #include "util.h" #include "opts.h" #include "sub.h" +#include "dispatch.h" +#include "glib/glib.h" #ifdef DEV_MODE // For type safety @@ -837,23 +839,13 @@ _timeoutPubAsync(natsTimer *t, void *closure) if (natsMsg_Create(&m, pm->subject, NULL, NULL, 0) != NATS_OK) break; - m->sub = js->rsub; natsMsg_setTimeout(m); - natsSub_Lock(js->rsub); - if (js->rsub->msgList.tail != NULL) - { - js->rsub->msgList.tail->next = m; - } - else - { - js->rsub->msgList.head = m; - natsCondition_Signal(js->rsub->cond); - } - js->rsub->msgList.tail = m; - js->rsub->msgList.msgs++; - js->rsub->msgList.bytes += natsMsg_dataAndHdrLen(m); - natsSub_Unlock(js->rsub); + // Best attempt, ignore NATS_SLOW_CONSUMER errors which may be returned + // here. + nats_lockSubAndDispatcher(js->rsub); + natsSub_enqueueUserMessage(js->rsub, m); + nats_unlockSubAndDispatcher(js->rsub); js->pmHead = pm->next; _destroyPMInfo(pm); @@ -1236,11 +1228,7 @@ jsSub_free(jsSub *jsi) js = jsi->js; natsTimer_Destroy(jsi->hbTimer); - if (jsi->mhMsg != NULL) - { - natsMsg_clearNoDestroy(jsi->mhMsg); - natsMsg_Destroy(jsi->mhMsg); - } + NATS_FREE(jsi->stream); NATS_FREE(jsi->consumer); NATS_FREE(jsi->nxtMsgSubj); @@ -1644,24 +1632,24 @@ natsSubscription_GetSequenceMismatch(jsConsumerSequenceMismatch *csm, natsSubscr if ((csm == NULL) || (sub == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->jsi == NULL) { - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setError(NATS_INVALID_SUBSCRIPTION, "%s", jsErrNotAJetStreamSubscription); } jsi = sub->jsi; m = &jsi->mismatch; if (m->dseq == m->ldseq) { - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_NOT_FOUND; } memset(csm, 0, sizeof(jsConsumerSequenceMismatch)); csm->Stream = m->sseq; csm->ConsumerClient = m->dseq; csm->ConsumerServer = m->ldseq; - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1841,7 +1829,7 @@ _fetch(natsMsgList *list, natsSubscription *sub, jsFetchRequest *req, bool simpl natsBuf_InitWithBackend(&buf, buffer, 0, sizeof(buffer)); nc = sub->conn; subj = jsi->nxtMsgSubj; - pmc = (sub->msgList.msgs > 0); + pmc = (sub->ownDispatcher.queue.msgs > 0); jsi->inFetch = true; jsi->fetchID++; if (nats_asprintf(&reqSubj, "%.*s%" PRIu64, (int) strlen(sub->subject)-1, sub->subject, jsi->fetchID) < 0) @@ -1850,23 +1838,22 @@ _fetch(natsMsgList *list, natsSubscription *sub, jsFetchRequest *req, bool simpl rply = (const char*) reqSubj; if ((s == NATS_OK) && req->Heartbeat) { - int64_t hbi = req->Heartbeat / 1000000; - sub->refs++; - if (jsi->hbTimer == NULL) + s = nats_createControlMessages(sub); + if (s == NATS_OK) { - s = natsMsg_create(&jsi->mhMsg, NULL, 0, NULL, 0, NULL, 0, -1); - if (s == NATS_OK) + int64_t hbi = req->Heartbeat / 1000000; + sub->refs++; + if (jsi->hbTimer == NULL) { - natsMsg_setNoDestroy(jsi->mhMsg); - s = natsTimer_Create(&jsi->hbTimer, _hbTimerFired, _hbTimerStopped, hbi*2, (void*) sub); + s = natsTimer_Create(&jsi->hbTimer, _hbTimerFired, _hbTimerStopped, hbi * 2, (void *)sub); + if (s != NATS_OK) + sub->refs--; } - if (s != NATS_OK) - sub->refs--; - } - else - natsTimer_Reset(jsi->hbTimer, hbi); + else + natsTimer_Reset(jsi->hbTimer, hbi); - mhMsg = jsi->mhMsg; + mhMsg = sub->control->batch.missedHeartbeat; + } } natsSub_Unlock(sub); @@ -2044,11 +2031,7 @@ _hbTimerFired(natsTimer *timer, void* closure) bool oc = false; natsStatus s = NATS_OK; - natsSub_Lock(sub); - if (sub->libDlvWorker != NULL) - { - natsMutex_Lock(sub->libDlvWorker->lock); - } + nats_lockSubAndDispatcher(sub); alert = !jsi->active; oc = jsi->ordered; jsi->active = false; @@ -2057,28 +2040,16 @@ _hbTimerFired(natsTimer *timer, void* closure) // If there are messages pending then we can't really consider // that we missed hearbeats. Wait for those to be processed and // we will check missed HBs again. - if (sub->msgList.msgs == 0) + if (sub->ownDispatcher.queue.msgs == 0) { - sub->msgList.msgs++; - sub->msgList.head = jsi->mhMsg; - sub->msgList.tail = jsi->mhMsg; - sub->msgList.bytes = natsMsg_dataAndHdrLen(jsi->mhMsg); - natsCondition_Signal(sub->cond); + natsSub_enqueueMessage(sub, sub->control->batch.missedHeartbeat); natsTimer_Stop(timer); } - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return; } nc = sub->conn; - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); if (!alert) return; @@ -2086,22 +2057,14 @@ _hbTimerFired(natsTimer *timer, void* closure) // For ordered consumers, we will need to reset if (oc) { - natsSub_Lock(sub); - if (sub->libDlvWorker != NULL) - { - natsMutex_Lock(sub->libDlvWorker->lock); - } + nats_lockSubAndDispatcher(sub); if (!sub->closed) { // If we fail in that call, we will report to async err callback // (if one is specified). s = jsSub_resetOrderedConsumer(sub, sub->jsi->sseq+1); } - if (sub->libDlvWorker != NULL) - { - natsMutex_Unlock(sub->libDlvWorker->lock); - } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } natsConn_Lock(nc); @@ -2708,7 +2671,7 @@ _subscribeMulti(natsSubscription **new_sub, jsCtx *js, const char **subjects, in else { maxap = info->Config->MaxAckPending; - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); jsi->dc = true; jsi->pending = info->NumPending + info->Delivered.Consumer; // There may be a race in the case of an ordered consumer where by this @@ -2725,7 +2688,7 @@ _subscribeMulti(natsSubscription **new_sub, jsCtx *js, const char **subjects, in s = nats_setDefaultError(NATS_NO_MEMORY); } } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } } diff --git a/src/msg.h b/src/msg.h index 2cbc9433c..4f56611d5 100644 --- a/src/msg.h +++ b/src/msg.h @@ -1,4 +1,4 @@ -// Copyright 2015-2022 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -40,6 +40,7 @@ #define natsMsg_setNoDestroy(m) ((m)->flags |= (1 << 2)) #define natsMsg_isNoDestroy(m) (((m)->flags & (1 << 2)) != 0) #define natsMsg_clearNoDestroy(m) ((m)->flags &= ~(1 << 2)) +#define natsMsg_noDestroyFlag (1 << 2) #define natsMsg_setTimeout(m) ((m)->flags |= (1 << 3)) #define natsMsg_isTimeout(m) (((m)->flags & (1 << 3)) != 0) diff --git a/src/nats.c b/src/nats.c index 1c2973d63..06e46c5b9 100644 --- a/src/nats.c +++ b/src/nats.c @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,1680 +16,222 @@ #include "stan/stanp.h" #endif -#include -#include -#include -#include -#include -#include - #include "mem.h" -#include "timer.h" -#include "util.h" -#include "asynccb.h" -#include "conn.h" +#include "glib/glib.h" #include "sub.h" -#include "nkeys.h" -#include "crypto.h" - -#define WAIT_LIB_INITIALIZED \ - natsMutex_Lock(gLib.lock); \ - while (!(gLib.initialized) && !(gLib.initAborted)) \ - natsCondition_Wait(gLib.cond, gLib.lock); \ - natsMutex_Unlock(gLib.lock) - -typedef struct natsTLError -{ - natsStatus sts; - char text[256]; - const char *func[MAX_FRAMES]; - int framesCount; - int skipUpdate; - -} natsTLError; - -typedef struct __natsLibTimers -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - natsTimer *timers; - int count; - bool changed; - bool shutdown; - -} natsLibTimers; - -typedef struct __natsLibAsyncCbs -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - natsAsyncCbInfo *head; - natsAsyncCbInfo *tail; - bool shutdown; - -} natsLibAsyncCbs; - -typedef struct __natsGCList -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - natsGCItem *head; - bool shutdown; - bool inWait; - -} natsGCList; - -typedef struct __natsLibDlvWorkers -{ - natsMutex *lock; - int idx; - int size; - int maxSize; - natsMsgDlvWorker **workers; - -} natsLibDlvWorkers; - -typedef struct __natsLib -{ - // Leave these fields before 'refs' - natsMutex *lock; - volatile bool wasOpenedOnce; - bool sslInitialized; - natsThreadLocal errTLKey; - natsThreadLocal sslTLKey; - natsThreadLocal natsThreadKey; - bool initialized; - bool closed; - natsCondition *closeCompleteCond; - bool *closeCompleteBool; - bool closeCompleteSignal; - bool finalCleanup; - // Do not move 'refs' without checking _freeLib() - int refs; - - bool initializing; - bool initAborted; - bool libHandlingMsgDeliveryByDefault; - int64_t libDefaultWriteDeadline; - - natsLibTimers timers; - natsLibAsyncCbs asyncCbs; - natsLibDlvWorkers dlvWorkers; - - natsCondition *cond; - - natsGCList gc; - - // For micro services code - natsMutex *service_callback_mu; - // uses `microService*` as the key and the value. - natsHash *all_services_to_callback; - -} natsLib; - -int64_t gLockSpinCount = 2000; - -static natsInitOnceType gInitOnce = NATS_ONCE_STATIC_INIT; -static natsLib gLib; - -static void -_destroyErrTL(void *localStorage) -{ - natsTLError *err = (natsTLError*) localStorage; - - NATS_FREE(err); -} - -static void -_cleanupThreadSSL(void *localStorage) -{ -#if defined(NATS_HAS_TLS) && !defined(NATS_USE_OPENSSL_1_1) - ERR_remove_thread_state(0); -#endif -} - -static void -_finalCleanup(void) -{ - if (gLib.sslInitialized) - { -#if defined(NATS_HAS_TLS) -#if !defined(NATS_USE_OPENSSL_1_1) - ERR_free_strings(); - EVP_cleanup(); - CRYPTO_cleanup_all_ex_data(); - ERR_remove_thread_state(0); -#endif - sk_SSL_COMP_free(SSL_COMP_get_compression_methods()); -#endif - natsThreadLocal_DestroyKey(gLib.sslTLKey); - } - - natsThreadLocal_DestroyKey(gLib.errTLKey); - natsThreadLocal_DestroyKey(gLib.natsThreadKey); - natsMutex_Destroy(gLib.lock); - gLib.lock = NULL; -} - -void -nats_setNATSThreadKey(void) -{ - natsThreadLocal_Set(gLib.natsThreadKey, (const void*)1); -} - -void -nats_ReleaseThreadMemory(void) -{ - void *tl = NULL; - - if (!(gLib.wasOpenedOnce)) - return; - - tl = natsThreadLocal_Get(gLib.errTLKey); - if (tl != NULL) - { - _destroyErrTL(tl); - natsThreadLocal_SetEx(gLib.errTLKey, NULL, false); - } - - tl = NULL; - - natsMutex_Lock(gLib.lock); - if (gLib.sslInitialized) - { - tl = natsThreadLocal_Get(gLib.sslTLKey); - if (tl != NULL) - { - _cleanupThreadSSL(tl); - natsThreadLocal_SetEx(gLib.sslTLKey, NULL, false); - } - } - natsMutex_Unlock(gLib.lock); -} - -#if defined(_WIN32) && _WIN32 -#ifndef NATS_STATIC -BOOL WINAPI DllMain(HINSTANCE hinstDLL, // DLL module handle - DWORD fdwReason, // reason called - LPVOID lpvReserved) // reserved -{ - switch (fdwReason) - { - // For applications linking dynamically NATS library, - // release thread-local memory for user-created threads. - // For portable applications, the user should manually call - // nats_ReleaseThreadMemory() before the thread returns so - // that no memory is leaked regardless if they link statically - // or dynamically. It is safe to call nats_ReleaseThreadMemory() - // twice for the same threads. - case DLL_THREAD_DETACH: - { - nats_ReleaseThreadMemory(); - break; - } - default: - break; - } - - return TRUE; - UNREFERENCED_PARAMETER(hinstDLL); - UNREFERENCED_PARAMETER(lpvReserved); -} -#endif -#endif - -static void -natsLib_Destructor(void) -{ - int refs = 0; - - if (!(gLib.wasOpenedOnce)) - return; - - // Destroy thread locals for the current thread. - nats_ReleaseThreadMemory(); - - // Do the final cleanup if possible - natsMutex_Lock(gLib.lock); - refs = gLib.refs; - if (refs > 0) - { - // If some thread is still around when the process exits and has a - // reference to the library, then don't do the final cleanup now. - // If the process has not fully exited when the lib's last reference - // is decremented, the final cleanup will be executed from that thread. - gLib.finalCleanup = true; - } - natsMutex_Unlock(gLib.lock); - - if (refs != 0) - return; - - _finalCleanup(); -} - -static void -_freeTimers(void) -{ - natsLibTimers *timers = &(gLib.timers); - - natsThread_Destroy(timers->thread); - natsCondition_Destroy(timers->cond); - natsMutex_Destroy(timers->lock); -} - -static void -_freeAsyncCbs(void) -{ - natsLibAsyncCbs *cbs = &(gLib.asyncCbs); - - natsThread_Destroy(cbs->thread); - natsCondition_Destroy(cbs->cond); - natsMutex_Destroy(cbs->lock); -} - -static void -_freeGC(void) -{ - natsGCList *gc = &(gLib.gc); - - natsThread_Destroy(gc->thread); - natsCondition_Destroy(gc->cond); - natsMutex_Destroy(gc->lock); -} - -static void -_freeDlvWorker(natsMsgDlvWorker *worker) -{ - natsThread_Destroy(worker->thread); - natsCondition_Destroy(worker->cond); - natsMutex_Destroy(worker->lock); - NATS_FREE(worker); -} - -static void -_freeDlvWorkers(void) -{ - int i; - natsLibDlvWorkers *workers = &(gLib.dlvWorkers); - - for (i=0; isize; i++) - _freeDlvWorker(workers->workers[i]); - - NATS_FREE(workers->workers); - natsMutex_Destroy(workers->lock); - workers->idx = 0; - workers->size = 0; - workers->workers = NULL; -} - -static void -_freeLib(void) -{ - const unsigned int offset = (unsigned int) offsetof(natsLib, refs); - bool callFinalCleanup = false; - - _freeTimers(); - _freeAsyncCbs(); - _freeGC(); - _freeDlvWorkers(); - natsNUID_free(); - natsMutex_Destroy(gLib.service_callback_mu); - natsHash_Destroy(gLib.all_services_to_callback); - - natsCondition_Destroy(gLib.cond); - - memset((void*) (offset + (char*)&gLib), 0, sizeof(natsLib) - offset); - - natsMutex_Lock(gLib.lock); - callFinalCleanup = gLib.finalCleanup; - if (gLib.closeCompleteCond != NULL) - { - if (gLib.closeCompleteSignal) - { - *gLib.closeCompleteBool = true; - natsCondition_Signal(gLib.closeCompleteCond); - } - gLib.closeCompleteCond = NULL; - gLib.closeCompleteBool = NULL; - gLib.closeCompleteSignal = false; - } - gLib.closed = false; - gLib.initialized = false; - gLib.finalCleanup= false; - natsMutex_Unlock(gLib.lock); - - if (callFinalCleanup) - _finalCleanup(); -} - -void -natsLib_Retain(void) -{ - natsMutex_Lock(gLib.lock); - - gLib.refs++; - - natsMutex_Unlock(gLib.lock); -} - -void -natsLib_Release(void) -{ - int refs = 0; - - natsMutex_Lock(gLib.lock); - - refs = --(gLib.refs); - - natsMutex_Unlock(gLib.lock); - - if (refs == 0) - _freeLib(); -} - -static void -_doInitOnce(void) -{ - natsStatus s; - - memset(&gLib, 0, sizeof(natsLib)); - - s = natsMutex_Create(&(gLib.lock)); - if (s == NATS_OK) - s = natsThreadLocal_CreateKey(&(gLib.errTLKey), _destroyErrTL); - if (s == NATS_OK) - s = natsThreadLocal_CreateKey(&(gLib.natsThreadKey), NULL); - if (s != NATS_OK) - { - fprintf(stderr, "FATAL ERROR: Unable to initialize library!\n"); - fflush(stderr); - abort(); - } - - natsSys_Init(); - - // Setup a hook for when the process exits. - atexit(natsLib_Destructor); -} - -static void -_insertTimer(natsTimer *t) -{ - natsTimer *cur = gLib.timers.timers; - natsTimer *prev = NULL; - - while ((cur != NULL) && (cur->absoluteTime <= t->absoluteTime)) - { - prev = cur; - cur = cur->next; - } - - if (cur != NULL) - { - t->prev = prev; - t->next = cur; - cur->prev = t; - - if (prev != NULL) - prev->next = t; - } - else if (prev != NULL) - { - prev->next = t; - t->prev = prev; - t->next = NULL; - } - - if (prev == NULL) - gLib.timers.timers = t; -} - -// Locks must be held before entering this function -static void -_removeTimer(natsLibTimers *timers, natsTimer *t) -{ - // Switch flag - t->stopped = true; - - // It the timer was in the callback, it has already been removed from the - // list, so skip that. - if (!(t->inCallback)) - { - if (t->prev != NULL) - t->prev->next = t->next; - if (t->next != NULL) - t->next->prev = t->prev; - - if (t == gLib.timers.timers) - gLib.timers.timers = t->next; - - t->prev = NULL; - t->next = NULL; - } - - // Decrease the global count of timers - timers->count--; -} - -int64_t -nats_setTargetTime(int64_t timeout) -{ - int64_t target = nats_Now() + timeout; - if (target < 0) - target = 0x7FFFFFFFFFFFFFFF; - return target; -} - -void -nats_resetTimer(natsTimer *t, int64_t newInterval) -{ - natsLibTimers *timers = &(gLib.timers); - - natsMutex_Lock(timers->lock); - natsMutex_Lock(t->mu); - - // If timer is active, we need first to remove it. This call does the - // right thing if the timer is in the callback. - if (!(t->stopped)) - _removeTimer(timers, t); - - // Bump the timer's global count (it as decreased in the _removeTimers call - timers->count++; - - // Switch stopped flag - t->stopped = false; - - // Set the new interval (may be same than it was before, but that's ok) - t->interval = newInterval; - - // If the timer is in the callback, the insertion and setting of the - // absolute time will be done by the timer thread when returning from - // the timer's callback. - if (!(t->inCallback)) - { - t->absoluteTime = nats_setTargetTime(t->interval); - _insertTimer(t); - } - - natsMutex_Unlock(t->mu); - - if (!(timers->changed)) - natsCondition_Signal(timers->cond); - - timers->changed = true; - - natsMutex_Unlock(timers->lock); -} - -void -nats_stopTimer(natsTimer *t) -{ - natsLibTimers *timers = &(gLib.timers); - bool doCb = false; - - natsMutex_Lock(timers->lock); - natsMutex_Lock(t->mu); - - // If the timer was already stopped, nothing to do. - if (t->stopped) - { - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - return; - } - - _removeTimer(timers, t); - - doCb = (!(t->inCallback) && (t->stopCb != NULL)); - - natsMutex_Unlock(t->mu); - - if (!(timers->changed)) - natsCondition_Signal(timers->cond); - - timers->changed = true; - - natsMutex_Unlock(timers->lock); - - if (doCb) - (*(t->stopCb))(t, t->closure); -} - -int -nats_getTimersCount(void) -{ - int count = 0; - - natsMutex_Lock(gLib.timers.lock); - - count = gLib.timers.count; - - natsMutex_Unlock(gLib.timers.lock); - - return count; -} - -int -nats_getTimersCountInList(void) -{ - int count = 0; - natsTimer *t; - - natsMutex_Lock(gLib.timers.lock); - - t = gLib.timers.timers; - while (t != NULL) - { - count++; - t = t->next; - } - - natsMutex_Unlock(gLib.timers.lock); - - return count; -} - - -static void -_timerThread(void *arg) -{ - natsLibTimers *timers = &(gLib.timers); - natsTimer *t = NULL; - natsStatus s = NATS_OK; - bool doStopCb; - int64_t target; - - WAIT_LIB_INITIALIZED; - - natsMutex_Lock(timers->lock); - - while (!(timers->shutdown)) - { - // Take the first timer that needs to fire. - t = timers->timers; - - if (t == NULL) - { - // No timer, fire in an hour... - target = nats_setTargetTime(3600 * 1000); - } - else - { - target = t->absoluteTime; - } - - timers->changed = false; - - s = NATS_OK; - - while (!(timers->shutdown) - && (s != NATS_TIMEOUT) - && !(timers->changed)) - { - s = natsCondition_AbsoluteTimedWait(timers->cond, timers->lock, - target); - } - - if (timers->shutdown) - break; - - if ((t == NULL) || timers->changed) - continue; - - natsMutex_Lock(t->mu); - - // Remove timer from the list: - timers->timers = t->next; - if (t->next != NULL) - t->next->prev = NULL; - - t->prev = NULL; - t->next = NULL; - - t->inCallback = true; - - // Retain the timer, since we are going to release the locks for the - // callback. The user may "destroy" the timer from there, so we need - // to be protected with reference counting. - t->refs++; - - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - (*(t->cb))(t, t->closure); - - natsMutex_Lock(timers->lock); - natsMutex_Lock(t->mu); - - t->inCallback = false; - - // Timer may have been stopped from within the callback, or during - // the window the locks were released. - doStopCb = (t->stopped && (t->stopCb != NULL)); - - // If not stopped, we need to put it back in our list - if (!doStopCb) - { - // Reset our view of what is the time this timer should fire - // because: - // 1- the callback may have taken longer than it should - // 2- the user may have called Reset() with a new interval - t->absoluteTime = nats_setTargetTime(t->interval); - _insertTimer(t); - } - - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - if (doStopCb) - (*(t->stopCb))(t, t->closure); - - // Compensate for the retain that we made before invoking the timer's - // callback - natsTimer_Release(t); - - natsMutex_Lock(timers->lock); - } - - // Process the timers that were left in the list (not stopped) when the - // library is shutdown. - while ((t = timers->timers) != NULL) - { - natsMutex_Lock(t->mu); - - // Check if we should invoke the callback. Note that although we are - // releasing the locks below, a timer present in the list here is - // guaranteed not to have been stopped (because it would not be in - // the list otherwise, since there is no chance that it is in the - // timer's callback). So just check if there is a stopCb to invoke. - doStopCb = (t->stopCb != NULL); - - // Remove the timer from the list. - _removeTimer(timers, t); - - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - // Invoke the callback now. - if (doStopCb) - (*(t->stopCb))(t, t->closure); - - // No release of the timer here. The user is still responsible to call - // natsTimer_Destroy(). - - natsMutex_Lock(timers->lock); - } - - natsMutex_Unlock(timers->lock); - - natsLib_Release(); -} - -static void -_asyncCbsThread(void *arg) -{ - natsLibAsyncCbs *asyncCbs = &(gLib.asyncCbs); - natsAsyncCbInfo *cb = NULL; - natsConnection *nc = NULL; -#if defined(NATS_HAS_STREAMING) - stanConnection *sc = NULL; -#endif - - WAIT_LIB_INITIALIZED; - - natsMutex_Lock(asyncCbs->lock); - - // We want all callbacks to be invoked even on shutdown - while (true) - { - while (((cb = asyncCbs->head) == NULL) && !asyncCbs->shutdown) - natsCondition_Wait(asyncCbs->cond, asyncCbs->lock); - - if ((cb == NULL) && asyncCbs->shutdown) - break; - - asyncCbs->head = cb->next; - - if (asyncCbs->tail == cb) - asyncCbs->tail = NULL; - - cb->next = NULL; - - natsMutex_Unlock(asyncCbs->lock); - - nc = cb->nc; -#if defined(NATS_HAS_STREAMING) - sc = cb->sc; -#endif - - switch (cb->type) - { - case ASYNC_CLOSED: - { - (*(nc->opts->closedCb))(nc, nc->opts->closedCbClosure); - if (nc->opts->microClosedCb != NULL) - (*(nc->opts->microClosedCb))(nc, NULL); - break; - } - - case ASYNC_DISCONNECTED: - (*(nc->opts->disconnectedCb))(nc, nc->opts->disconnectedCbClosure); - break; - case ASYNC_RECONNECTED: - (*(nc->opts->reconnectedCb))(nc, nc->opts->reconnectedCbClosure); - break; - case ASYNC_CONNECTED: - (*(nc->opts->connectedCb))(nc, nc->opts->connectedCbClosure); - break; - case ASYNC_DISCOVERED_SERVERS: - (*(nc->opts->discoveredServersCb))(nc, nc->opts->discoveredServersClosure); - break; - case ASYNC_LAME_DUCK_MODE: - (*(nc->opts->lameDuckCb))(nc, nc->opts->lameDuckClosure); - break; - case ASYNC_ERROR: - { - if (cb->errTxt != NULL) - nats_setErrStatusAndTxt(cb->err, cb->errTxt); - - (*(nc->opts->asyncErrCb))(nc, cb->sub, cb->err, nc->opts->asyncErrCbClosure); - if (nc->opts->microAsyncErrCb != NULL) - (*(nc->opts->microAsyncErrCb))(nc, cb->sub, cb->err, NULL); - break; - } -#if defined(NATS_HAS_STREAMING) - case ASYNC_STAN_CONN_LOST: - (*(sc->opts->connectionLostCB))(sc, sc->connLostErrTxt, sc->opts->connectionLostCBClosure); - break; -#endif - default: - break; - } - - natsAsyncCb_Destroy(cb); - nats_clearLastError(); - - natsMutex_Lock(asyncCbs->lock); - } - - natsMutex_Unlock(asyncCbs->lock); - - natsLib_Release(); -} - -natsStatus -nats_postAsyncCbInfo(natsAsyncCbInfo *info) -{ - natsMutex_Lock(gLib.asyncCbs.lock); - - if (gLib.asyncCbs.shutdown) - { - natsMutex_Unlock(gLib.asyncCbs.lock); - return NATS_NOT_INITIALIZED; - } - - info->next = NULL; - - if (gLib.asyncCbs.head == NULL) - gLib.asyncCbs.head = info; - - if (gLib.asyncCbs.tail != NULL) - gLib.asyncCbs.tail->next = info; - - gLib.asyncCbs.tail = info; - - natsCondition_Signal(gLib.asyncCbs.cond); - - natsMutex_Unlock(gLib.asyncCbs.lock); - - return NATS_OK; -} - -static void -_garbageCollector(void *closure) -{ - natsGCList *gc = &(gLib.gc); - natsGCItem *item; - natsGCItem *list; - - WAIT_LIB_INITIALIZED; - - natsMutex_Lock(gc->lock); - - // Process all elements in the list, even on shutdown - while (true) - { - // Go into wait until we are notified to shutdown - // or there is something to garbage collect - gc->inWait = true; - - while (!(gc->shutdown) && (gc->head == NULL)) - { - natsCondition_Wait(gc->cond, gc->lock); - } - - // Out of the wait. Setting this boolean avoids unnecessary - // signaling when an item is added to the collector. - gc->inWait = false; - - // Do not break out on shutdown here, we want to clear the list, - // even on exit so that valgrind and the like are happy. - - // Under the lock, we will switch to a local list and reset the - // GC's list (so that others can add to the list without contention - // (at least from the GC itself). - do - { - list = gc->head; - gc->head = NULL; - - natsMutex_Unlock(gc->lock); - - // Now that we are outside of the lock, we can empty the list. - while ((item = list) != NULL) - { - // Pops item from the beginning of the list. - list = item->next; - item->next = NULL; - - // Invoke the freeCb associated with this object - (*(item->freeCb))((void*) item); - } - - natsMutex_Lock(gc->lock); - } - while (gc->head != NULL); - - // If we were ask to shutdown and since the list is now empty, exit - if (gc->shutdown) - break; - } - - natsMutex_Unlock(gc->lock); - - natsLib_Release(); -} - -bool -natsGC_collect(natsGCItem *item) -{ - natsGCList *gc; - bool signal; - - // If the object was not setup for garbage collection, return false - // so the caller frees the object. - if (item->freeCb == NULL) - return false; - - gc = &(gLib.gc); - - natsMutex_Lock(gc->lock); - - // We will signal only if the GC is in the condition wait. - signal = gc->inWait; - - // Add to the front of the list. - item->next = gc->head; - - // Update head. - gc->head = item; - - if (signal) - natsCondition_Signal(gc->cond); - - natsMutex_Unlock(gc->lock); - - return true; -} - -static void -_libTearDown(void) -{ - int i; - - for (i=0; ithread != NULL) - natsThread_Join(worker->thread); - } - - if (gLib.timers.thread != NULL) - natsThread_Join(gLib.timers.thread); - - if (gLib.asyncCbs.thread != NULL) - natsThread_Join(gLib.asyncCbs.thread); - - if (gLib.gc.thread != NULL) - natsThread_Join(gLib.gc.thread); - - natsLib_Release(); -} - -natsStatus -nats_Open(int64_t lockSpinCount) -{ - natsStatus s = NATS_OK; - - if (!nats_InitOnce(&gInitOnce, _doInitOnce)) - return NATS_FAILED_TO_INITIALIZE; - - natsMutex_Lock(gLib.lock); - - if (gLib.closed || gLib.initialized || gLib.initializing) - { - if (gLib.closed) - s = NATS_FAILED_TO_INITIALIZE; - else if (gLib.initializing) - s = NATS_ILLEGAL_STATE; - - natsMutex_Unlock(gLib.lock); - return s; - } - - gLib.initializing = true; - gLib.initAborted = false; - -#if !defined(_WIN32) - signal(SIGPIPE, SIG_IGN); -#endif - - srand((unsigned int) nats_NowInNanoSeconds()); - - gLib.refs = 1; - - // If the caller specifies negative value, then we use the default - if (lockSpinCount >= 0) - gLockSpinCount = lockSpinCount; - - nats_Base32_Init(); - - s = natsCondition_Create(&(gLib.cond)); - - if (s == NATS_OK) - s = natsCrypto_Init(); - - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.timers.lock)); - if (s == NATS_OK) - s = natsCondition_Create(&(gLib.timers.cond)); - if (s == NATS_OK) - { - s = natsThread_Create(&(gLib.timers.thread), _timerThread, NULL); - if (s == NATS_OK) - gLib.refs++; - } - - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.asyncCbs.lock)); - if (s == NATS_OK) - s = natsCondition_Create(&(gLib.asyncCbs.cond)); - if (s == NATS_OK) - { - s = natsThread_Create(&(gLib.asyncCbs.thread), _asyncCbsThread, NULL); - if (s == NATS_OK) - gLib.refs++; - } - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.gc.lock)); - if (s == NATS_OK) - s = natsCondition_Create(&(gLib.gc.cond)); - if (s == NATS_OK) - { - s = natsThread_Create(&(gLib.gc.thread), _garbageCollector, NULL); - if (s == NATS_OK) - gLib.refs++; - } - if (s == NATS_OK) - s = natsNUID_init(); - - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.dlvWorkers.lock)); - if (s == NATS_OK) - { - char *defaultWriteDeadlineStr = getenv("NATS_DEFAULT_LIB_WRITE_DEADLINE"); - - if (defaultWriteDeadlineStr != NULL) - gLib.libDefaultWriteDeadline = (int64_t) atol(defaultWriteDeadlineStr); - - gLib.libHandlingMsgDeliveryByDefault = (getenv("NATS_DEFAULT_TO_LIB_MSG_DELIVERY") != NULL ? true : false); - gLib.dlvWorkers.maxSize = 1; - gLib.dlvWorkers.workers = NATS_CALLOC(gLib.dlvWorkers.maxSize, sizeof(natsMsgDlvWorker*)); - if (gLib.dlvWorkers.workers == NULL) - s = NATS_NO_MEMORY; - } - if (s == NATS_OK) - s = natsMutex_Create(&gLib.service_callback_mu); - if (s == NATS_OK) - s = natsHash_Create(&gLib.all_services_to_callback, 8); - - if (s == NATS_OK) - gLib.initialized = true; - - // In case of success or error, broadcast so that lib's threads - // can proceed. - if (gLib.cond != NULL) - { - if (s != NATS_OK) - { - gLib.initAborted = true; - gLib.timers.shutdown = true; - gLib.asyncCbs.shutdown = true; - gLib.gc.shutdown = true; - } - natsCondition_Broadcast(gLib.cond); - } - - gLib.initializing = false; - gLib.wasOpenedOnce = true; - - natsMutex_Unlock(gLib.lock); - - if (s != NATS_OK) - _libTearDown(); - - return s; -} - -natsStatus -natsInbox_Create(natsInbox **newInbox) -{ - natsStatus s; - char *inbox = NULL; - const int size = NATS_DEFAULT_INBOX_PRE_LEN + NUID_BUFFER_LEN + 1; - - s = nats_Open(-1); - if (s != NATS_OK) - return s; - - inbox = NATS_MALLOC(size); - if (inbox == NULL) - return nats_setDefaultError(NATS_NO_MEMORY); - - memcpy(inbox, NATS_DEFAULT_INBOX_PRE, NATS_DEFAULT_INBOX_PRE_LEN); - s = natsNUID_Next(inbox + NATS_DEFAULT_INBOX_PRE_LEN, NUID_BUFFER_LEN + 1); - if (s == NATS_OK) - { - inbox[size-1] = '\0'; - *newInbox = (natsInbox*) inbox; - } - else - NATS_FREE(inbox); - return NATS_UPDATE_ERR_STACK(s); -} - -void -natsInbox_Destroy(natsInbox *inbox) -{ - if (inbox == NULL) - return; - - NATS_FREE(inbox); -} - - -static natsStatus -_close(bool wait, int64_t timeout) -{ - natsStatus s = NATS_OK; - natsCondition *cond = NULL; - bool complete = false; - int i; - - // This is to protect against a call to nats_Close() while there - // was no prior call to nats_Open(), either directly or indirectly. - if (!nats_InitOnce(&gInitOnce, _doInitOnce)) - return NATS_ERR; - - natsMutex_Lock(gLib.lock); - - if (gLib.closed || !gLib.initialized) - { - bool closed = gLib.closed; - - natsMutex_Unlock(gLib.lock); - - if (closed) - return NATS_ILLEGAL_STATE; - return NATS_NOT_INITIALIZED; - } - if (wait) - { - if (natsThreadLocal_Get(gLib.natsThreadKey) != NULL) - s = NATS_ILLEGAL_STATE; - if (s == NATS_OK) - s = natsCondition_Create(&cond); - if (s != NATS_OK) - { - natsMutex_Unlock(gLib.lock); - return s; - } - gLib.closeCompleteCond = cond; - gLib.closeCompleteBool = &complete; - gLib.closeCompleteSignal = true; - } - - gLib.closed = true; - - natsMutex_Lock(gLib.timers.lock); - gLib.timers.shutdown = true; - natsCondition_Signal(gLib.timers.cond); - natsMutex_Unlock(gLib.timers.lock); - - natsMutex_Lock(gLib.asyncCbs.lock); - gLib.asyncCbs.shutdown = true; - natsCondition_Signal(gLib.asyncCbs.cond); - natsMutex_Unlock(gLib.asyncCbs.lock); - - natsMutex_Lock(gLib.gc.lock); - gLib.gc.shutdown = true; - natsCondition_Signal(gLib.gc.cond); - natsMutex_Unlock(gLib.gc.lock); - - natsMutex_Lock(gLib.dlvWorkers.lock); - for (i=0; ilock); - worker->shutdown = true; - natsCondition_Signal(worker->cond); - natsMutex_Unlock(worker->lock); - } - natsMutex_Unlock(gLib.dlvWorkers.lock); - - natsMutex_Unlock(gLib.lock); - - nats_ReleaseThreadMemory(); - _libTearDown(); - - if (wait) - { - natsMutex_Lock(gLib.lock); - while ((s != NATS_TIMEOUT) && !complete) - { - if (timeout <= 0) - natsCondition_Wait(cond, gLib.lock); - else - s = natsCondition_TimedWait(cond, gLib.lock, timeout); - } - if (s != NATS_OK) - gLib.closeCompleteSignal = false; - natsMutex_Unlock(gLib.lock); - - natsCondition_Destroy(cond); - } - - return s; -} - -void -nats_Close(void) -{ - _close(false, true); -} - -natsStatus -nats_CloseAndWait(int64_t timeout) -{ - return _close(true, timeout); -} - -const char* -nats_GetVersion(void) -{ - return LIB_NATS_VERSION_STRING; -} - -uint32_t -nats_GetVersionNumber(void) -{ - return LIB_NATS_VERSION_NUMBER; -} - -static void -_versionGetString(char *buffer, size_t bufLen, uint32_t verNumber) -{ - snprintf(buffer, bufLen, "%u.%u.%u", - ((verNumber >> 16) & 0xF), - ((verNumber >> 8) & 0xF), - (verNumber & 0xF)); -} - -bool -nats_CheckCompatibilityImpl(uint32_t headerReqVerNumber, uint32_t headerVerNumber, - const char *headerVerString) -{ - if ((headerVerNumber < LIB_NATS_VERSION_REQUIRED_NUMBER) - || (headerReqVerNumber > LIB_NATS_VERSION_NUMBER)) - { - char reqVerString[10]; - char libReqVerString[10]; - - _versionGetString(reqVerString, sizeof(reqVerString), headerReqVerNumber); - _versionGetString(libReqVerString, sizeof(libReqVerString), NATS_VERSION_REQUIRED_NUMBER); - - printf("Incompatible versions:\n" \ - "Header : %s (requires %s)\n" \ - "Library: %s (requires %s)\n", - headerVerString, reqVerString, - NATS_VERSION_STRING, libReqVerString); - exit(1); - } - - return true; -} - -static natsTLError* -_getTLError(void) -{ - natsTLError *errTL = NULL; - bool needFree = false; - - // The library should already be initialized, but let's protect against - // situations where foo() invokes bar(), which invokes baz(), which - // invokes nats_Open(). If that last call fails, when we un-wind down - // to foo(), it may be difficult to know that nats_Open() failed and - // that we should not try to invoke natsLib_setError. So we check again - // here that the library has been initialized properly, and if not, we - // simply don't set the error. - if (nats_Open(-1) != NATS_OK) - return NULL; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if (errTL == NULL) - { - errTL = (natsTLError*) NATS_CALLOC(1, sizeof(natsTLError)); - if (errTL != NULL) - errTL->framesCount = -1; - needFree = (errTL != NULL); - - } - - if ((errTL != NULL) - && (natsThreadLocal_SetEx(gLib.errTLKey, - (const void*) errTL, false) != NATS_OK)) - { - if (needFree) - NATS_FREE(errTL); - - errTL = NULL; - } - - return errTL; -} - -static char* -_getErrorShortFileName(const char* fileName) -{ - char *file = strstr(fileName, "src"); - - if (file != NULL) - file = (file + 4); - else - file = (char*) fileName; - - return file; -} - -static void -_updateStack(natsTLError *errTL, const char *funcName, natsStatus errSts, - bool calledFromSetError) -{ - int idx; - - idx = errTL->framesCount; - if ((idx >= 0) - && (idx < MAX_FRAMES) - && (strcmp(errTL->func[idx], funcName) == 0)) - { - return; - } - - // In case no error was already set... - if ((errTL->framesCount == -1) && !calledFromSetError) - errTL->sts = errSts; - - idx = ++(errTL->framesCount); - - if (idx >= MAX_FRAMES) - return; - - errTL->func[idx] = funcName; -} - -#if !defined(_WIN32) -__attribute__ ((format (printf, 5, 6))) -#endif -natsStatus -nats_setErrorReal(const char *fileName, const char *funcName, int line, natsStatus errSts, const char *errTxtFmt, ...) -{ - natsTLError *errTL = _getTLError(); - char tmp[256]; - va_list ap; - int n; - - if ((errTL == NULL) || errTL->skipUpdate) - return errSts; - - errTL->sts = errSts; - errTL->framesCount = -1; - - tmp[0] = '\0'; - - va_start(ap, errTxtFmt); - nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); - va_end(ap); +#include "conn.h" - if (strlen(tmp) > 0) +#if defined(_WIN32) && _WIN32 +#ifndef NATS_STATIC +BOOL WINAPI DllMain(HINSTANCE hinstDLL, // DLL module handle + DWORD fdwReason, // reason called + LPVOID lpvReserved) // reserved +{ + switch (fdwReason) { - n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", - _getErrorShortFileName(fileName), line, tmp); - if ((n < 0) || (n >= (int) sizeof(errTL->text))) + // For applications linking dynamically NATS library, + // release thread-local memory for user-created threads. + // For portable applications, the user should manually call + // nats_ReleaseThreadMemory() before the thread returns so + // that no memory is leaked regardless if they link statically + // or dynamically. It is safe to call nats_ReleaseThreadMemory() + // twice for the same threads. + case DLL_THREAD_DETACH: { - int pos = ((int) strlen(errTL->text)) - 1; - int i; - - for (i=0; i<3; i++) - errTL->text[pos--] = '.'; + nats_ReleaseThreadMemory(); + break; } + default: + break; } - _updateStack(errTL, funcName, errSts, true); - - return errSts; + return TRUE; + UNREFERENCED_PARAMETER(hinstDLL); + UNREFERENCED_PARAMETER(lpvReserved); } - -#if !defined(_WIN32) -__attribute__ ((format (printf, 4, 5))) #endif -void -nats_updateErrTxt(const char *fileName, const char *funcName, int line, const char *errTxtFmt, ...) -{ - natsTLError *errTL = _getTLError(); - char tmp[256]; - va_list ap; - int n; - - if ((errTL == NULL) || errTL->skipUpdate) - return; - - tmp[0] = '\0'; - - va_start(ap, errTxtFmt); - nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); - va_end(ap); - - if (strlen(tmp) > 0) - { - n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", - _getErrorShortFileName(fileName), line, tmp); - if ((n < 0) || (n >= (int) sizeof(errTL->text))) - { - int pos = ((int) strlen(errTL->text)) - 1; - int i; +#endif - for (i=0; i<3; i++) - errTL->text[pos--] = '.'; - } - } +static void _overwriteInt64(const char *envVar, int64_t *val) +{ + char *str = getenv(envVar); + if (str != NULL) + *val = atoll(str); } -void -nats_setErrStatusAndTxt(natsStatus err, const char *errTxt) +static void _overwriteInt(const char *envVar, int *val) { - natsTLError *errTL = _getTLError(); - - if ((errTL == NULL) || errTL->skipUpdate) - return; - - errTL->sts = err; - snprintf(errTL->text, sizeof(errTL->text), "%s", errTxt); - errTL->framesCount = -1; + char *str = getenv(envVar); + if (str != NULL) + *val = atoi(str); } -natsStatus -nats_updateErrStack(natsStatus err, const char *func) +static void _overwriteBool(const char *envVar, bool *val) { - natsTLError *errTL = _getTLError(); - - if ((errTL == NULL) || errTL->skipUpdate) - return err; - - _updateStack(errTL, func, err, false); - - return err; + char *str = getenv(envVar); + if (str != NULL) + *val = (strcasecmp(str, "true") == 0) || + (strcasecmp(str, "on") == 0) || + (atoi(str) != 0); } -void -nats_clearLastError(void) +static void _overrideWithEnv(natsClientConfig *config) { - natsTLError *errTL = _getTLError(); - - if ((errTL == NULL) || errTL->skipUpdate) - return; - - errTL->sts = NATS_OK; - errTL->text[0] = '\0'; - errTL->framesCount = -1; + _overwriteInt64("NATS_DEFAULT_LIB_WRITE_DEADLINE", &config->DefaultWriteDeadline); + _overwriteBool("NATS_USE_THREAD_POOL", &config->DefaultToThreadPool); + _overwriteInt("NATS_THREAD_POOL_MAX", &config->ThreadPoolMax); + _overwriteBool("NATS_USE_THREAD_POOL_FOR_REPLIES", &config->DefaultRepliesToThreadPool); + _overwriteInt("NATS_REPLY_THREAD_POOL_MAX", &config->ReplyThreadPoolMax); } -void -nats_doNotUpdateErrStack(bool skipStackUpdate) +// environment variables will override the default options. +natsStatus +nats_OpenWithConfig(natsClientConfig *config) { - natsTLError *errTL = _getTLError(); - - if (errTL == NULL) - return; - - if (skipStackUpdate) - { - errTL->skipUpdate++; - } - else - { - errTL->skipUpdate--; - assert(errTL->skipUpdate >= 0); - } + return nats_openLib(config); } -const char* -nats_GetLastError(natsStatus *status) +natsStatus +nats_Open(int64_t lockSpinCount) { - natsStatus s; - natsTLError *errTL = NULL; - - if (status != NULL) - *status = NATS_OK; - - // Ensure the library is loaded - s = nats_Open(-1); - if (s != NATS_OK) - return NULL; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if ((errTL == NULL) || (errTL->sts == NATS_OK)) - return NULL; - - if (status != NULL) - *status = errTL->sts; + bool defaultToSharedDispatchers = (getenv("NATS_DEFAULT_TO_LIB_MSG_DELIVERY") != NULL ? true : false); + + natsClientConfig config = { + .LockSpinCount = lockSpinCount, + .DefaultToThreadPool = defaultToSharedDispatchers, + .ThreadPoolMax = 1, + .DefaultRepliesToThreadPool = false, + .ReplyThreadPoolMax = 0, + }; - return errTL->text; + return nats_openLib(&config); } natsStatus -nats_GetLastErrorStack(char *buffer, size_t bufLen) +natsInbox_Create(natsInbox **newInbox) { - natsTLError *errTL = NULL; - int offset = 0; - int i, max, n, len; - - if ((buffer == NULL) || (bufLen == 0)) - return NATS_INVALID_ARG; - - buffer[0] = '\0'; - len = (int) bufLen; - - // Ensure the library is loaded - if (nats_Open(-1) != NATS_OK) - return NATS_FAILED_TO_INITIALIZE; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) - return NATS_OK; + natsStatus s; + char *inbox = NULL; + const int size = NATS_DEFAULT_INBOX_PRE_LEN + NUID_BUFFER_LEN + 1; - max = errTL->framesCount; - if (max >= MAX_FRAMES) - max = MAX_FRAMES - 1; + s = nats_Open(-1); + if (s != NATS_OK) + return s; - for (i=0; (i<=max) && (len > 0); i++) - { - n = snprintf(buffer + offset, len, "%s%s", - errTL->func[i], - (i < max ? "\n" : "")); - // On Windows, n will be < 0 if len is not big enough. - if (n < 0) - { - len = 0; - } - else - { - offset += n; - len -= n; - } - } + inbox = NATS_MALLOC(size); + if (inbox == NULL) + return nats_setDefaultError(NATS_NO_MEMORY); - if ((max != errTL->framesCount) && (len > 0)) + memcpy(inbox, NATS_DEFAULT_INBOX_PRE, NATS_DEFAULT_INBOX_PRE_LEN); + s = natsNUID_Next(inbox + NATS_DEFAULT_INBOX_PRE_LEN, NUID_BUFFER_LEN + 1); + if (s == NATS_OK) { - n = snprintf(buffer + offset, len, "\n%d more...", - errTL->framesCount - max); - // On Windows, n will be < 0 if len is not big enough. - if (n < 0) - len = 0; - else - len -= n; + inbox[size-1] = '\0'; + *newInbox = (natsInbox*) inbox; } - - if (len <= 0) - return NATS_INSUFFICIENT_BUFFER; - - return NATS_OK; + else + NATS_FREE(inbox); + return NATS_UPDATE_ERR_STACK(s); } void -nats_PrintLastErrorStack(FILE *file) +natsInbox_Destroy(natsInbox *inbox) { - natsTLError *errTL = NULL; - int i, max; - - // Ensure the library is loaded - if (nats_Open(-1) != NATS_OK) - return; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) + if (inbox == NULL) return; - fprintf(file, "Error: %u - %s", - errTL->sts, natsStatus_GetText(errTL->sts)); - if (errTL->text[0] != '\0') - fprintf(file, " - %s", errTL->text); - fprintf(file, "\n"); - fprintf(file, "Stack: (library version: %s)\n", nats_GetVersion()); - - max = errTL->framesCount; - if (max >= MAX_FRAMES) - max = MAX_FRAMES - 1; - - for (i=0; i<=max; i++) - fprintf(file, " %02d - %s\n", (i+1), errTL->func[i]); - - if (max != errTL->framesCount) - fprintf(file, " %d more...\n", errTL->framesCount - max); - - fflush(file); + NATS_FREE(inbox); } void -nats_sslRegisterThreadForCleanup(void) +nats_Close(void) { -#if defined(NATS_HAS_TLS) - // Set anything. The goal is that at thread exit, the thread local key - // will have something non NULL associated, which will trigger the - // destructor that we have registered. - (void) natsThreadLocal_Set(gLib.sslTLKey, (void*) 1); -#endif + nats_closeLib(false, true); } natsStatus -nats_sslInit(void) +nats_CloseAndWait(int64_t timeout) { - natsStatus s = NATS_OK; + return nats_closeLib(true, timeout); +} - // Ensure the library is loaded - s = nats_Open(-1); - if (s != NATS_OK) - return s; +const char* +nats_GetVersion(void) +{ + return LIB_NATS_VERSION_STRING; +} + +uint32_t +nats_GetVersionNumber(void) +{ + return LIB_NATS_VERSION_NUMBER; +} - natsMutex_Lock(gLib.lock); +static void +_versionGetString(char *buffer, size_t bufLen, uint32_t verNumber) +{ + snprintf(buffer, bufLen, "%u.%u.%u", + ((verNumber >> 16) & 0xF), + ((verNumber >> 8) & 0xF), + (verNumber & 0xF)); +} - if (!(gLib.sslInitialized)) +bool +nats_CheckCompatibilityImpl(uint32_t headerReqVerNumber, uint32_t headerVerNumber, + const char *headerVerString) +{ + if ((headerVerNumber < LIB_NATS_VERSION_REQUIRED_NUMBER) + || (headerReqVerNumber > LIB_NATS_VERSION_NUMBER)) { - // Regardless of success, mark as initialized so that we - // can do cleanup on exit. - gLib.sslInitialized = true; + char reqVerString[10]; + char libReqVerString[10]; -#if defined(NATS_HAS_TLS) -#if !defined(NATS_USE_OPENSSL_1_1) - // Initialize SSL. - SSL_library_init(); - SSL_load_error_strings(); -#endif -#endif - s = natsThreadLocal_CreateKey(&(gLib.sslTLKey), _cleanupThreadSSL); - } + _versionGetString(reqVerString, sizeof(reqVerString), headerReqVerNumber); + _versionGetString(libReqVerString, sizeof(libReqVerString), NATS_VERSION_REQUIRED_NUMBER); - natsMutex_Unlock(gLib.lock); + printf("Incompatible versions:\n" \ + "Header : %s (requires %s)\n" \ + "Library: %s (requires %s)\n", + headerVerString, reqVerString, + NATS_VERSION_STRING, libReqVerString); + exit(1); + } - return NATS_UPDATE_ERR_STACK(s); + return true; } -static void -_deliverMsgs(void *arg) -{ - natsMsgDlvWorker *dlv = (natsMsgDlvWorker*) arg; - natsConnection *nc; - natsSubscription *sub; - natsMsgHandler mcb; - void *mcbClosure; - uint64_t delivered; - uint64_t max; - natsMsg *msg; - bool timerNeedReset = false; - jsSub *jsi; - char *fcReply; - - natsMutex_Lock(dlv->lock); +void +nats_deliverMsgsPoolf(void *arg) +{ + natsDispatcher *d = (natsDispatcher *)arg; + natsConnection *nc; + natsSubscription *sub; + natsMsgHandler mcb; + void *mcbClosure; + uint64_t delivered; + uint64_t max; + natsMsg *msg; + bool timerNeedReset = false; + jsSub *jsi; + char *fcReply; + + natsMutex_Lock(d->mu); while (true) { - while (((msg = dlv->msgList.head) == NULL) && !dlv->shutdown) - natsCondition_Wait(dlv->cond, dlv->lock); + while (((msg = d->queue.head) == NULL) && !d->shutdown) + natsCondition_Wait(d->cond, d->mu); // Break out only when list is empty - if ((msg == NULL) && dlv->shutdown) + if ((msg == NULL) && d->shutdown) { break; } // Remove message from list now... - dlv->msgList.head = msg->next; - if (dlv->msgList.tail == msg) - dlv->msgList.tail = NULL; + d->queue.head = msg->next; + if (d->queue.tail == msg) + d->queue.tail = NULL; msg->next = NULL; // Get subscription reference from message @@ -1704,16 +246,12 @@ _deliverMsgs(void *arg) // Is this a control message? if (msg->subject[0] == '\0') { - bool closed = sub->closed; - bool timedOut = sub->timedOut; - bool draining = sub->libDlvDraining; - - // Switch off this flag. - if (draining) - sub->libDlvDraining = false; + bool closed = (msg == sub->control->sub.close); + bool timedOut = (msg == sub->control->sub.timeout); + bool draining = (msg == sub->control->sub.drain); // We need to release this lock... - natsMutex_Unlock(dlv->lock); + natsMutex_Unlock(d->mu); // Release the message natsMsg_Destroy(msg); @@ -1728,15 +266,15 @@ _deliverMsgs(void *arg) } else if (closed) { - natsOnCompleteCB cb = NULL; - void *closure = NULL; + natsOnCompleteCB cb = NULL; + void *closure = NULL; // Call this in case the subscription was draining. natsSub_setDrainCompleteState(sub); // Check for completion callback natsSub_Lock(sub); - cb = sub->onCompleteCB; + cb = sub->onCompleteCB; closure = sub->onCompleteCBClosure; natsSub_Unlock(sub); @@ -1753,7 +291,7 @@ _deliverMsgs(void *arg) } // Grab the lock, we go back to beginning of loop. - natsMutex_Lock(dlv->lock); + natsMutex_Lock(d->mu); if (!draining && !closed && timedOut) { @@ -1768,9 +306,11 @@ _deliverMsgs(void *arg) continue; } - // Update before checking closed state. - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_dataAndHdrLen(msg); + // Update the sub's stats before checking closed state. (We now post + // control messages to the sub's queue, because hbTimer processing is + // expecting it, so need to clear the stats for them, too) + sub->ownDispatcher.queue.msgs--; + sub->ownDispatcher.queue.bytes -= natsMsg_dataAndHdrLen(msg); // Need to check for closed subscription again here. // The subscription could have been unsubscribed from a callback @@ -1796,15 +336,15 @@ _deliverMsgs(void *arg) // If we are dealing with the last pending message for this sub, // we will reset the timer after the user callback returns. - if (sub->msgList.msgs == 0) + if (sub->ownDispatcher.queue.msgs == 0) timerNeedReset = true; } - natsMutex_Unlock(dlv->lock); + natsMutex_Unlock(d->mu); if ((max == 0) || (delivered <= max)) { - (*mcb)(nc, sub, msg, mcbClosure); + (*mcb)(nc, sub, msg, mcbClosure); } else { @@ -1828,7 +368,7 @@ _deliverMsgs(void *arg) natsConn_removeSubscription(nc, sub); } - natsMutex_Lock(dlv->lock); + natsMutex_Lock(d->mu); // Check if timer need to be reset for subscriptions that can timeout. if (!sub->closed && (sub->timeout != 0) && timerNeedReset) @@ -1847,7 +387,7 @@ _deliverMsgs(void *arg) } } - natsMutex_Unlock(dlv->lock); + natsMutex_Unlock(d->mu); natsLib_Release(); } @@ -1856,194 +396,12 @@ natsStatus nats_SetMessageDeliveryPoolSize(int max) { natsStatus s = NATS_OK; - natsLibDlvWorkers *workers; // Ensure the library is loaded s = nats_Open(-1); if (s != NATS_OK) return s; - workers = &gLib.dlvWorkers; - - natsMutex_Lock(workers->lock); - - if (max <= 0) - { - natsMutex_Unlock(workers->lock); - return nats_setError(NATS_ERR, "%s", "Pool size cannot be negative or zero"); - } - - // Do not error on max < workers->maxSize in case we allow shrinking - // the pool in the future. - if (max > workers->maxSize) - { - natsMsgDlvWorker **newArray = NATS_CALLOC(max, sizeof(natsMsgDlvWorker*)); - if (newArray == NULL) - s = nats_setDefaultError(NATS_NO_MEMORY); - if (s == NATS_OK) - { - int i; - for (i=0; isize; i++) - newArray[i] = workers->workers[i]; - - NATS_FREE(workers->workers); - workers->workers = newArray; - workers->maxSize = max; - } - } - - natsMutex_Unlock(workers->lock); - - return NATS_UPDATE_ERR_STACK(s); -} - -// Post a control message to the worker's queue. -natsStatus -natsLib_msgDeliveryPostControlMsg(natsSubscription *sub) -{ - natsStatus s; - natsMsg *controlMsg = NULL; - natsMsgDlvWorker *worker = (sub->libDlvWorker); - - // Create a "end" message and post it to the delivery worker - s = natsMsg_create(&controlMsg, NULL, 0, NULL, 0, NULL, 0, -1); - if (s == NATS_OK) - { - nats_MsgList *l; - bool signal = false; - - natsMutex_Lock(worker->lock); - - controlMsg->sub = sub; - - l = &(worker->msgList); - if (l->head == NULL) - { - l->head = controlMsg; - signal = true; - } - else - l->tail->next = controlMsg; - l->tail = controlMsg; - - if (signal) - natsCondition_Signal(worker->cond); - - natsMutex_Unlock(worker->lock); - } - return NATS_UPDATE_ERR_STACK(s); -} - -natsStatus -natsLib_msgDeliveryAssignWorker(natsSubscription *sub) -{ - natsStatus s = NATS_OK; - natsLibDlvWorkers *workers = &(gLib.dlvWorkers); - natsMsgDlvWorker *worker = NULL; - - natsMutex_Lock(workers->lock); - - if (workers->maxSize == 0) - { - natsMutex_Unlock(workers->lock); - return nats_setError(NATS_FAILED_TO_INITIALIZE, "%s", "Message delivery thread pool size is 0!"); - } - - worker = workers->workers[workers->idx]; - if (worker == NULL) - { - worker = NATS_CALLOC(1, sizeof(natsMsgDlvWorker)); - if (worker == NULL) - s = nats_setDefaultError(NATS_NO_MEMORY); - if (s == NATS_OK) - s = natsMutex_Create(&worker->lock); - if (s == NATS_OK) - s = natsCondition_Create(&worker->cond); - if (s == NATS_OK) - { - natsLib_Retain(); - s = natsThread_Create(&worker->thread, _deliverMsgs, (void*) worker); - if (s != NATS_OK) - natsLib_Release(); - } - if (s == NATS_OK) - { - workers->workers[workers->idx] = worker; - workers->size++; - } - else - { - _freeDlvWorker(worker); - } - } - if (s == NATS_OK) - { - sub->libDlvWorker = worker; - if (++(workers->idx) == workers->maxSize) - workers->idx = 0; - } - - natsMutex_Unlock(workers->lock); - - return NATS_UPDATE_ERR_STACK(s); -} - -bool -natsLib_isLibHandlingMsgDeliveryByDefault(void) -{ - return gLib.libHandlingMsgDeliveryByDefault; -} - -int64_t -natsLib_defaultWriteDeadline(void) -{ - return gLib.libDefaultWriteDeadline; -} - -void -natsLib_getMsgDeliveryPoolInfo(int *maxSize, int *size, int *idx, natsMsgDlvWorker ***workersArray) -{ - natsLibDlvWorkers *workers = &gLib.dlvWorkers; - - natsMutex_Lock(workers->lock); - *maxSize = workers->maxSize; - *size = workers->size; - *idx = workers->idx; - *workersArray = workers->workers; - natsMutex_Unlock(workers->lock); -} - -natsStatus -natsLib_startServiceCallbacks(microService *m) -{ - natsStatus s; - - natsMutex_Lock(gLib.service_callback_mu); - s = natsHash_Set(gLib.all_services_to_callback, (int64_t)m, (void *)m, NULL); - natsMutex_Unlock(gLib.service_callback_mu); - + s = nats_setMessageDispatcherPoolCap(max); return NATS_UPDATE_ERR_STACK(s); } - -void -natsLib_stopServiceCallbacks(microService *m) -{ - if (m == NULL) - return; - - natsMutex_Lock(gLib.service_callback_mu); - natsHash_Remove(gLib.all_services_to_callback, (int64_t)m); - natsMutex_Unlock(gLib.service_callback_mu); -} - -natsMutex* -natsLib_getServiceCallbackMutex(void) -{ - return gLib.service_callback_mu; -} - -natsHash* -natsLib_getAllServicesToCallback(void) -{ - return gLib.all_services_to_callback; -} diff --git a/src/nats.h b/src/nats.h index 295af3a30..24e88c190 100644 --- a/src/nats.h +++ b/src/nats.h @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -186,6 +186,28 @@ typedef struct __natsOptions natsOptions; */ typedef char natsInbox; + +/** \brief An initial configuration for NATS client. Provides control over the + * threading model, and sets many default option values. + * + * @see natsOptions_Create + */ +typedef struct __natsClientConfig +{ + int64_t DefaultWriteDeadline; + + int64_t LockSpinCount; + + // Subscription message delivery thread control + bool DefaultToThreadPool; + int ThreadPoolMax; + + // Reply message delivery thread control + bool DefaultRepliesToThreadPool; + bool UseSeparatePoolForReplies; + int ReplyThreadPoolMax; +} natsClientConfig; + /** \brief A list of NATS messages. * * Used by some APIs which return a list of #natsMsg objects. @@ -1773,6 +1795,23 @@ typedef void (*stanConnectionLostHandler)( * @{ */ +/** \brief Initializes the library. + * + * This initializes the library, with more control over how threads and/or + * thread pools are used to deliver messages to the application. + * + * It is invoked automatically when creating a connection, with the default + * settings (same as nats_Open(-1)). + * + * \warning You must not call #nats_Open[WithConfig] and #nats_Close + * concurrently. + * + * @param config points to a natsClientConfig. A copy of the settings is made, + * so the config can be freed after initializing the NATS client. + */ +natsStatus +nats_OpenWithConfig(natsClientConfig *config); + /** \brief Initializes the library. * * This initializes the library. diff --git a/src/natsp.h b/src/natsp.h index 1f7894ecd..84bf00612 100644 --- a/src/natsp.h +++ b/src/natsp.h @@ -1,4 +1,4 @@ -// Copyright 2015-2022 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -41,6 +41,7 @@ #include "url.h" #include "srvpool.h" #include "msg.h" +#include "dispatch.h" #include "asynccb.h" #include "hash.h" #include "stats.h" @@ -273,7 +274,16 @@ struct __natsOptions void *evLoop; natsEvLoopCallbacks evCbs; - bool libMsgDelivery; + // If set to false, the client will start a per-subscription dedicated + // thread to deliver messages to the user callbacks. If true, a shared + // thread out of a thread pool is used. natsClientConfig controls the pool + // size. + bool useSharedDispatcher; + + // If set to false, the client will start a per-connection dedicated thread + // to deliver reply messages to the user callbacks. If true, a shared thread + // out of a thread pool is used. natsClientConfig controls the pool size. + bool useSharedReplyDispatcher; int orderIP; // possible values: 0,4,6,46,64 @@ -334,26 +344,6 @@ struct __natsOptions // Custom message payload padding size int payloadPaddingSize; }; - -typedef struct __nats_MsgList -{ - natsMsg *head; - natsMsg *tail; - int msgs; - int bytes; - -} nats_MsgList; - -typedef struct __natsMsgDlvWorker -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - bool shutdown; - nats_MsgList msgList; - -} natsMsgDlvWorker; - typedef struct __pmInfo { char *subject; @@ -407,7 +397,6 @@ typedef struct __jsSub int64_t hbi; bool active; natsTimer *hbTimer; - natsMsg *mhMsg; char *cmeta; uint64_t sseq; @@ -497,6 +486,21 @@ struct __kvWatcher }; +typedef struct __natsSubscriptionControlMessages +{ + struct + { + natsMsg *timeout; + natsMsg *close; + natsMsg *drain; + } sub; + struct + { + natsMsg *expired; + natsMsg *missedHeartbeat; + } batch; +} natsSubscriptionControlMessages; + struct __natsSubscription { natsMutex *mu; @@ -506,22 +510,28 @@ struct __natsSubscription // This is non-zero when auto-unsubscribe is used. uint64_t max; - // This is updated in the delivery thread (or NextMsg) and indicates - // how many message have been presented to the callback (or returned - // from NextMsg). Like 'msgs', this is also used to determine if we - // have reached the max number of messages. + // We always have a dispatcher to keep track of things, even if the + // subscription is sync. The dispatcher is set up at the subscription + // creation time, and may point to a dedicated thread using sub's own + // dispatchQueue, or a shared worker using its own dispatch queue, which + // dispatcher->queue then points to. + natsDispatcher *dispatcher; + natsDispatcher ownDispatcher; + + // These are a signals to the sub's async dispatcher thread that something + // happened - draining or closing the subscription, or some sort of a + // timeout. Since these are optional, we only allocate them when starting an + // async dispatcher. + natsSubscriptionControlMessages *control; + + // This is updated in the delivery thread (or NextMsg) and indicates how + // many message have been presented to the callback (or returned from + // NextMsg). Together with the messages pending dispatch in + // dispatch->queue, this is also used to determine if we have reached the + // max number of messages. uint64_t delivered; - - // The list of messages waiting to be delivered to the callback (or - // returned from NextMsg). - nats_MsgList msgList; - - // True if msgList.count is over pendingMax + // True if ownDispatcher.queue.msgs is over pendingMax bool slowConsumer; - - // Condition variable used to wait for message delivery. - natsCondition *cond; - // The subscriber is closed (or closing). bool closed; @@ -537,11 +547,7 @@ struct __natsSubscription int64_t drainTimeout; // This is set if the flush failed and will prevent the connection for pushing further messages. bool drainSkip; - - // Same than draining but for the global delivery situation. - // This boolean will be switched off when processed, as opposed - // to draining that once set does not get reset. - bool libDlvDraining; + natsCondition *drainCond; // If true, the subscription is closed, but because the connection // was closed, not because of subscription (auto-)unsubscribe. @@ -562,13 +568,6 @@ struct __natsSubscription // Reference to the connection that created this subscription. struct __natsConnection *conn; - // Delivery thread (for async subscription). - natsThread *deliverMsgsThread; - - // If message delivery is done by the library instead, this is the - // reference to the worker handling this subscription. - natsMsgDlvWorker *libDlvWorker; - // Message callback and closure (for async subscription). natsMsgHandler msgCb; void *msgCbClosure; @@ -591,7 +590,6 @@ struct __natsSubscription // For JetStream jsSub *jsi; - }; typedef struct __natsPong @@ -751,62 +749,9 @@ struct __natsConnection } srvVersion; }; -// -// Library -// - -void -natsSys_Init(void); - -void -natsLib_Retain(void); - -void -natsLib_Release(void); - -int64_t -nats_setTargetTime(int64_t timeout); - -void -nats_resetTimer(natsTimer *t, int64_t newInterval); - -void -nats_stopTimer(natsTimer *t); - -// Returns the number of timers that have been created and not stopped. -int -nats_getTimersCount(void); - -// Returns the number of timers actually in the list. This should be -// equal to nats_getTimersCount() or nats_getTimersCount() - 1 when a -// timer thread is invoking a timer's callback. -int -nats_getTimersCountInList(void); - -natsStatus -nats_postAsyncCbInfo(natsAsyncCbInfo *info); - void nats_sslRegisterThreadForCleanup(void); -natsStatus -nats_sslInit(void); - -natsStatus -natsLib_msgDeliveryPostControlMsg(natsSubscription *sub); - -natsStatus -natsLib_msgDeliveryAssignWorker(natsSubscription *sub); - -bool -natsLib_isLibHandlingMsgDeliveryByDefault(void); - -int64_t -natsLib_defaultWriteDeadline(void); - -void -natsLib_getMsgDeliveryPoolInfo(int *maxSize, int *size, int *idx, natsMsgDlvWorker ***workersArray); - void nats_setNATSThreadKey(void); @@ -939,4 +884,18 @@ jsSub_resetOrderedConsumer(natsSubscription *sub, uint64_t sseq); bool natsMsg_isJSCtrl(natsMsg *msg, int *ctrlType); +static inline void nats_lockDispatcher(natsDispatcher *d) +{ + if (d->mu != NULL) + natsMutex_Lock(d->mu); +} + +static inline void nats_unlockDispatcher(natsDispatcher *d) +{ + if (d->mu != NULL) + natsMutex_Unlock(d->mu); +} + +void nats_deliverMsgsPoolf(void *arg); + #endif /* NATSP_H_ */ diff --git a/src/natstime.c b/src/natstime.c index 8e87d7568..9712d9c8f 100644 --- a/src/natstime.c +++ b/src/natstime.c @@ -85,3 +85,12 @@ natsDeadline_GetTimeout(natsDeadline *deadline) return timeout; } + +int64_t +nats_setTargetTime(int64_t timeout) +{ + int64_t target = nats_Now() + timeout; + if (target < 0) + target = 0x7FFFFFFFFFFFFFFF; + return target; +} diff --git a/src/natstime.h b/src/natstime.h index b7041d2ea..66da73b93 100644 --- a/src/natstime.h +++ b/src/natstime.h @@ -33,5 +33,7 @@ natsDeadline_GetTimeout(natsDeadline *deadline); void natsDeadline_Clear(natsDeadline *deadline); +int64_t +nats_setTargetTime(int64_t timeout); #endif /* NATSTIME_H_ */ diff --git a/src/opts.c b/src/opts.c index 078691a38..da72a3d99 100644 --- a/src/opts.c +++ b/src/opts.c @@ -1,4 +1,4 @@ -// Copyright 2015-2021 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,6 +19,7 @@ #include "opts.h" #include "util.h" #include "conn.h" +#include "glib/glib.h" natsStatus natsOptions_SetURL(natsOptions *opts, const char* url) @@ -306,7 +307,7 @@ _getSSLCtx(natsOptions *opts) { natsStatus s; - s = nats_sslInit(); + s = nats_initSSL(); if ((s == NATS_OK) && (opts->sslCtx != NULL)) { bool createNew = false; @@ -1065,7 +1066,7 @@ natsOptions_UseGlobalMessageDelivery(natsOptions *opts, bool global) // Sets if the subscriptions created from the connection will // create their own delivery thread or use the one(s) from // the library. - opts->libMsgDelivery = global; + opts->useSharedDispatcher = global; UNLOCK_OPTS(opts); @@ -1536,7 +1537,8 @@ natsOptions_Create(natsOptions **newOpts) if (opts == NULL) return nats_setDefaultError(NATS_NO_MEMORY); - if (natsMutex_Create(&(opts->mu)) != NATS_OK) + IFOK(s, natsMutex_Create(&opts->mu)); + if(s != NATS_OK) { NATS_FREE(opts); return NATS_UPDATE_ERR_STACK(NATS_NO_MEMORY); @@ -1552,13 +1554,14 @@ natsOptions_Create(natsOptions **newOpts) opts->maxPendingMsgs = NATS_OPTS_DEFAULT_MAX_PENDING_MSGS; opts->maxPendingBytes = -1; opts->timeout = NATS_OPTS_DEFAULT_TIMEOUT; - opts->libMsgDelivery = natsLib_isLibHandlingMsgDeliveryByDefault(); - opts->writeDeadline = natsLib_defaultWriteDeadline(); opts->reconnectBufSize = NATS_OPTS_DEFAULT_RECONNECT_BUF_SIZE; opts->reconnectJitter = NATS_OPTS_DEFAULT_RECONNECT_JITTER; opts->reconnectJitterTLS = NATS_OPTS_DEFAULT_RECONNECT_JITTER_TLS; opts->asyncErrCb = natsConn_defaultErrHandler; + // Override with values from the config (or from environment variables) + nats_overrideDefaultOptionsWithConfig(opts); + *newOpts = opts; return NATS_OK; diff --git a/src/stan/conn.c b/src/stan/conn.c index 50aaec982..4bfe90eb6 100644 --- a/src/stan/conn.c +++ b/src/stan/conn.c @@ -18,6 +18,7 @@ #include "../asynccb.h" #include "../conn.h" #include "../sub.h" +#include "../glib/glib.h" // Client send connID in ConnectRequest and PubMsg, and server // listens and responds to client PINGs. The validity of the diff --git a/src/sub.c b/src/sub.c index 3ff9472bc..561867b2b 100644 --- a/src/sub.c +++ b/src/sub.c @@ -1,4 +1,4 @@ -// Copyright 2015-2021 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,54 +23,99 @@ #include "util.h" #include "js.h" #include "opts.h" +#include "glib/glib.h" #ifdef DEV_MODE -static void _retain(natsSubscription *sub) { sub->refs++; } -static void _release(natsSubscription *sub) { sub->refs--; } - -void natsSub_Lock(natsSubscription *sub) { natsMutex_Lock(sub->mu); } -void natsSub_Unlock(natsSubscription *sub) { natsMutex_Unlock(sub->mu); } +static inline int _retain(natsSubscription *sub) { return ++(sub->refs); } +static inline int _release(natsSubscription *sub) { return --(sub->refs); } #else -#define _retain(s) ((s)->refs++) -#define _release(s) ((s)->refs--) +#define _retain(s) (++((s)->refs)) +#define _release(s) (--((s)->refs)) #endif // DEV_MODE -#define SUB_DLV_WORKER_LOCK(s) if ((s)->libDlvWorker != NULL) \ - natsMutex_Lock((s)->libDlvWorker->lock) - -#define SUB_DLV_WORKER_UNLOCK(s) if ((s)->libDlvWorker != NULL) \ - natsMutex_Unlock((s)->libDlvWorker->lock) - bool testDrainAutoUnsubRace = false; -static void -_freeSubscription(natsSubscription *sub) +static inline void _destroyControlMessage(natsMsg *msg) { - natsMsg *m; + if (msg != NULL) + { + natsMsg_clearNoDestroy(msg); + natsMsg_Destroy(msg); + } +} - if (sub == NULL) +static inline void _freeControlMessages(natsSubscription *sub) +{ + if (sub->control == NULL) return; - while ((m = sub->msgList.head) != NULL) + _destroyControlMessage(sub->control->sub.timeout); + _destroyControlMessage(sub->control->sub.close); + _destroyControlMessage(sub->control->sub.drain); + _destroyControlMessage(sub->control->batch.expired); + _destroyControlMessage(sub->control->batch.missedHeartbeat); + NATS_FREE(sub->control); +} + +static inline natsStatus _createControlMessage(natsMsg **msg, natsSubscription *sub) +{ + natsStatus s = natsMsg_create(msg, NULL, 0, NULL, 0, NULL, 0, -1); + if (s == NATS_OK) { - sub->msgList.head = m->next; - natsMsg_Destroy(m); + natsMsg_setNoDestroy(*msg); + (*msg)->sub = sub; } + return s; +} - NATS_FREE(sub->subject); - NATS_FREE(sub->queue); +// Sets up the default (user-thread NextMsg-only) dispatcher for a sub. Avoid +// the use of term "pull" since it's not to be confused with JetStream consumer +// pull requests. +static natsStatus +_initOwnDispatcher(natsSubscription *sub) +{ + natsStatus s = NATS_OK; + + if (sub->ownDispatcher.dedicatedTo != NULL) + return nats_setDefaultError(NATS_ILLEGAL_STATE); + + sub->ownDispatcher.dedicatedTo = sub; + sub->ownDispatcher.mu = sub->mu; + s = natsCondition_Create(&sub->ownDispatcher.cond); + return NATS_UPDATE_ERR_STACK(s); +} + +static inline void _cleanupOwnDispatcher(natsSubscription *sub) +{ + nats_destroyQueuedMessages(&sub->ownDispatcher.queue); - if (sub->deliverMsgsThread != NULL) + if (sub->ownDispatcher.thread != NULL) { - natsThread_Detach(sub->deliverMsgsThread); - natsThread_Destroy(sub->deliverMsgsThread); + natsThread_Join(sub->ownDispatcher.thread); + natsThread_Destroy(sub->ownDispatcher.thread); + sub->ownDispatcher.thread = NULL; } + + natsCondition_Destroy(sub->ownDispatcher.cond); +} + +void _freeSub(natsSubscription *sub) +{ + if (sub == NULL) + return; + + _freeControlMessages(sub); + _cleanupOwnDispatcher(sub); + + NATS_FREE(sub->subject); + NATS_FREE(sub->queue); + + natsCondition_Destroy(sub->drainCond); natsTimer_Destroy(sub->timeoutTimer); - natsCondition_Destroy(sub->cond); natsMutex_Destroy(sub->mu); jsSub_free(sub->jsi); @@ -79,18 +124,7 @@ _freeSubscription(natsSubscription *sub) NATS_FREE(sub); } -void -natsSub_retain(natsSubscription *sub) -{ - natsSub_Lock(sub); - - sub->refs++; - - natsSub_Unlock(sub); -} - -void -natsSub_release(natsSubscription *sub) +void natsSub_release(natsSubscription *sub) { int refs = 0; @@ -99,53 +133,29 @@ natsSub_release(natsSubscription *sub) natsSub_Lock(sub); - refs = --(sub->refs); + refs = _release(sub); natsSub_Unlock(sub); if (refs == 0) - _freeSubscription(sub); -} - -void -natsSubAndLdw_Lock(natsSubscription *sub) -{ - natsMutex_Lock(sub->mu); - SUB_DLV_WORKER_LOCK(sub); -} - -void -natsSubAndLdw_LockAndRetain(natsSubscription *sub) -{ - natsMutex_Lock(sub->mu); - sub->refs++; - SUB_DLV_WORKER_LOCK(sub); -} - -void -natsSubAndLdw_Unlock(natsSubscription *sub) -{ - SUB_DLV_WORKER_UNLOCK(sub); - natsMutex_Unlock(sub->mu); + _freeSub(sub); } -void -natsSubAndLdw_UnlockAndRelease(natsSubscription *sub) +void natsSub_unlockRelease(natsSubscription *sub) { int refs = 0; - SUB_DLV_WORKER_UNLOCK(sub); + refs = _release(sub); - refs = --(sub->refs); - natsMutex_Unlock(sub->mu); + natsSub_Unlock(sub); if (refs == 0) - _freeSubscription(sub); + _freeSub(sub); } // Runs under the subscription lock but will release it for a JS subscription // if the JS consumer needs to be deleted. -static void +static inline void _setDrainCompleteState(natsSubscription *sub) { // It is possible that we are here without being in "drain in progress" @@ -178,12 +188,12 @@ _setDrainCompleteState(natsSubscription *sub) sub->drainStatus = NATS_INVALID_SUBSCRIPTION; } sub->drainState |= SUB_DRAIN_COMPLETE; - natsCondition_Broadcast(sub->cond); + + natsCondition_Broadcast(sub->drainCond); } } -void -natsSub_setDrainCompleteState(natsSubscription *sub) +void natsSub_setDrainCompleteState(natsSubscription *sub) { natsSub_Lock(sub); _setDrainCompleteState(sub); @@ -191,24 +201,23 @@ natsSub_setDrainCompleteState(natsSubscription *sub) } // _deliverMsgs is used to deliver messages to asynchronous subscribers. -void -natsSub_deliverMsgs(void *arg) +void natsSub_deliverMsgs(void *arg) { - natsSubscription *sub = (natsSubscription*) arg; - natsConnection *nc = sub->conn; - natsMsgHandler mcb = sub->msgCb; - void *mcbClosure = sub->msgCbClosure; - uint64_t delivered; - uint64_t max; - natsMsg *msg; - int64_t timeout; - natsStatus s = NATS_OK; - bool draining = false; - bool rmSub = false; - natsOnCompleteCB onCompleteCB = NULL; - void *onCompleteCBClosure = NULL; - char *fcReply = NULL; - jsSub *jsi = NULL; + natsSubscription *sub = (natsSubscription *)arg; + natsConnection *nc = sub->conn; + natsMsgHandler mcb = sub->msgCb; + void *mcbClosure = sub->msgCbClosure; + uint64_t delivered; + uint64_t max; + natsMsg *msg; + int64_t timeout; + natsStatus s = NATS_OK; + bool draining = false; + bool rmSub = false; + natsOnCompleteCB onCompleteCB = NULL; + void *onCompleteCBClosure = NULL; + char *fcReply = NULL; + jsSub *jsi = NULL; // This just serves as a barrier for the creation of this thread. natsConn_Lock(nc); @@ -224,12 +233,12 @@ natsSub_deliverMsgs(void *arg) natsSub_Lock(sub); s = NATS_OK; - while (((msg = sub->msgList.head) == NULL) && !(sub->closed) && !(sub->draining) && (s != NATS_TIMEOUT)) + while (((msg = sub->ownDispatcher.queue.head) == NULL) && !(sub->closed) && !(sub->draining) && (s != NATS_TIMEOUT)) { if (timeout != 0) - s = natsCondition_TimedWait(sub->cond, sub->mu, timeout); + s = natsCondition_TimedWait(sub->ownDispatcher.cond, sub->mu, timeout); else - natsCondition_Wait(sub->cond, sub->mu); + natsCondition_Wait(sub->ownDispatcher.cond, sub->mu); } if (sub->closed) @@ -256,13 +265,13 @@ natsSub_deliverMsgs(void *arg) delivered = ++(sub->delivered); - sub->msgList.head = msg->next; + sub->ownDispatcher.queue.head = msg->next; - if (sub->msgList.tail == msg) - sub->msgList.tail = NULL; + if (sub->ownDispatcher.queue.tail == msg) + sub->ownDispatcher.queue.tail = NULL; - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_dataAndHdrLen(msg); + sub->ownDispatcher.queue.msgs--; + sub->ownDispatcher.queue.bytes -= natsMsg_dataAndHdrLen(msg); msg->next = NULL; @@ -276,7 +285,7 @@ natsSub_deliverMsgs(void *arg) if ((max == 0) || (delivered <= max)) { - (*mcb)(nc, sub, msg, mcbClosure); + (*mcb)(nc, sub, msg, mcbClosure); } else { @@ -301,7 +310,7 @@ natsSub_deliverMsgs(void *arg) } natsSub_Lock(sub); - onCompleteCB = sub->onCompleteCB; + onCompleteCB = sub->onCompleteCB; onCompleteCBClosure = sub->onCompleteCBClosure; _setDrainCompleteState(sub); natsSub_Unlock(sub); @@ -315,17 +324,27 @@ natsSub_deliverMsgs(void *arg) natsSub_release(sub); } -bool -natsSub_setMax(natsSubscription *sub, uint64_t max) +// Should be called only during the subscription creation process, no need to lock +static inline natsStatus +_runOwnDispatcher(natsSubscription *sub, bool forReplies) +{ + natsStatus s = NATS_OK; + if (sub->ownDispatcher.thread != NULL) + return nats_setDefaultError(NATS_ILLEGAL_STATE); // already running + + sub->dispatcher = &sub->ownDispatcher; + s = natsThread_Create(&sub->ownDispatcher.thread, natsSub_deliverMsgs, (void *) sub); + return NATS_UPDATE_ERR_STACK(s); +} + +bool natsSub_setMax(natsSubscription *sub, uint64_t max) { bool accepted = false; - natsSub_Lock(sub); - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); sub->max = (max <= sub->delivered ? 0 : max); accepted = sub->max != 0; - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return accepted; } @@ -350,12 +369,9 @@ natsSubscription_SetOnCompleteCB(natsSubscription *sub, natsOnCompleteCB cb, voi return s; } -void -natsSub_close(natsSubscription *sub, bool connectionClosed) +void natsSub_close(natsSubscription *sub, bool connectionClosed) { - natsSub_Lock(sub); - - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); if (!(sub->closed)) { @@ -365,40 +381,40 @@ natsSub_close(natsSubscription *sub, bool connectionClosed) if ((sub->jsi != NULL) && (sub->jsi->hbTimer != NULL)) natsTimer_Stop(sub->jsi->hbTimer); - if (sub->libDlvWorker != NULL) + // If this is a subscription with timeout, stop the timer. + if (sub->timeout != 0) + natsTimer_Stop(sub->timeoutTimer); + + if (sub->dispatcher != &sub->ownDispatcher) { - // If this is a subscription with timeout, stop the timer. - if (sub->timeout != 0) - natsTimer_Stop(sub->timeoutTimer); - - // Post a control message to wake-up the worker which will - // ensure that all pending messages for this subscription - // are removed and the subscription will ultimately be - // released in the worker thread. - natsLib_msgDeliveryPostControlMsg(sub); + // Post a control message to wake-up the worker which will ensure + // that all pending messages for this subscription are removed, + // release the subscription and self-destroy. + natsSub_enqueueMessage(sub, sub->control->sub.close); } else - natsCondition_Broadcast(sub->cond); + { + // Notify any pending natsSub_NextMsg() that the subscription is + // closed. + natsCondition_Broadcast(sub->ownDispatcher.cond); + } } - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } static void -_asyncTimeoutCb(natsTimer *timer, void* closure) +_asyncTimeoutCb(natsTimer *timer, void *closure) { - natsSubscription *sub = (natsSubscription*) closure; + natsSubscription *sub = (natsSubscription *)closure; // Should not happen, but in case - if (sub->libDlvWorker == NULL) + if (sub->dispatcher == NULL) return; - SUB_DLV_WORKER_LOCK(sub); - - // If the subscription is closed, or if we are prevented from posting - // a "timeout" control message, do nothing. + nats_lockSubAndDispatcher(sub); + // If the subscription has already timed out and has not reset, is closed or + // draining - do nothing. if (!sub->closed && !sub->timedOut && !sub->timeoutSuspended) { // Prevent from scheduling another control message while we are not @@ -407,32 +423,50 @@ _asyncTimeoutCb(natsTimer *timer, void* closure) // Set the timer to a very high value, it will be reset from the // worker thread. - natsTimer_Reset(sub->timeoutTimer, 60*60*1000); + natsTimer_Reset(sub->timeoutTimer, 60 * 60 * 1000); // Post a control message to the worker thread. - natsLib_msgDeliveryPostControlMsg(sub); + natsSub_enqueueMessage(sub, sub->control->sub.timeout); } - - SUB_DLV_WORKER_UNLOCK(sub); + nats_unlockSubAndDispatcher(sub); } static void -_asyncTimeoutStopCb(natsTimer *timer, void* closure) +_asyncTimeoutStopCb(natsTimer *timer, void *closure) { - natsSubscription *sub = (natsSubscription*) closure; + natsSubscription *sub = (natsSubscription *)closure; natsSub_release(sub); } +natsStatus nats_createControlMessages(natsSubscription *sub) +{ + natsStatus s = NATS_OK; + + if (sub->control != NULL) + return NATS_OK; + sub->control = NATS_CALLOC(1, sizeof(natsSubscriptionControlMessages)); + if (sub->control == NULL) + return nats_setDefaultError(NATS_NO_MEMORY); + IFOK(s, _createControlMessage(&(sub->control->sub.timeout), sub)); + IFOK(s, _createControlMessage(&sub->control->sub.close, sub)); + IFOK(s, _createControlMessage(&sub->control->sub.drain, sub)); + IFOK(s, _createControlMessage(&sub->control->batch.expired, sub)); + IFOK(s, _createControlMessage(&sub->control->batch.missedHeartbeat, sub)); + + // no need to free on failure, sub's free will clean it up. + return NATS_UPDATE_ERR_STACK(s); +} + natsStatus natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, const char *queueGroup, int64_t timeout, natsMsgHandler cb, void *cbClosure, - bool preventUseOfLibDlvPool, jsSub *jsi) + bool forReplies, jsSub *jsi) { - natsStatus s = NATS_OK; - natsSubscription *sub = NULL; + natsStatus s = NATS_OK; + natsSubscription *sub = NULL; - sub = (natsSubscription*) NATS_CALLOC(1, sizeof(natsSubscription)); + sub = (natsSubscription *)NATS_CALLOC(1, sizeof(natsSubscription)); if (sub == NULL) return nats_setDefaultError(NATS_NO_MEMORY); @@ -464,38 +498,45 @@ natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, if (sub->queue == NULL) s = nats_setDefaultError(NATS_NO_MEMORY); } + if (s == NATS_OK) - s = natsCondition_Create(&(sub->cond)); - if ((s == NATS_OK) && (cb != NULL)) - { - if (!(nc->opts->libMsgDelivery) || preventUseOfLibDlvPool) - { - // Let's not rely on the created thread acquiring the lock that - // would make it safe to retain only on success. - _retain(sub); + s = natsCondition_Create(&sub->drainCond); + if (s == NATS_OK) + s = _initOwnDispatcher(sub); - // If we have an async callback, start up a sub specific - // thread to deliver the messages. - s = natsThread_Create(&(sub->deliverMsgsThread), natsSub_deliverMsgs, - (void*) sub); - if (s != NATS_OK) - _release(sub); + bool useShared = (forReplies ? nc->opts->useSharedReplyDispatcher : nc->opts->useSharedDispatcher); + bool useAsyncThread = (cb != NULL); + if ((s == NATS_OK) && useAsyncThread) + s = nats_createControlMessages(sub); + if (s == NATS_OK) + { + _retain(sub); + if (!useAsyncThread) + { + sub->dispatcher = &sub->ownDispatcher; + _release(sub); } - else + else if (useShared && !forReplies) { - _retain(sub); - s = natsLib_msgDeliveryAssignWorker(sub); + s = nats_assignSubToDispatch(sub); + + // If we are using a shared dispatcher, we need to start the + // timeout timer. Own dispatcher uses a timed wait on the + // condition, and does not need the timer. if ((s == NATS_OK) && (timeout > 0)) { _retain(sub); - s = natsTimer_Create(&sub->timeoutTimer, _asyncTimeoutCb, - _asyncTimeoutStopCb, timeout, (void*) sub); + s = natsTimer_Create(&sub->timeoutTimer, _asyncTimeoutCb, _asyncTimeoutStopCb, timeout, sub); if (s != NATS_OK) _release(sub); } - if (s != NATS_OK) - _release(sub); } + else + { + s = _runOwnDispatcher(sub, forReplies); + } + if (s != NATS_OK) + _release(sub); } if (s == NATS_OK) @@ -547,7 +588,6 @@ natsConnection_SubscribeTimeout(natsSubscription **sub, natsConnection *nc, cons return NATS_UPDATE_ERR_STACK(s); } - /* * natsSubscribeSync is syntactic sugar for natsSubscribe(&sub, nc, subject, NULL). */ @@ -569,8 +609,8 @@ natsConnection_SubscribeSync(natsSubscription **sub, natsConnection *nc, const c */ natsStatus natsConnection_QueueSubscribe(natsSubscription **sub, natsConnection *nc, - const char *subject, const char *queueGroup, - natsMsgHandler cb, void *cbClosure) + const char *subject, const char *queueGroup, + natsMsgHandler cb, void *cbClosure) { natsStatus s; @@ -591,13 +631,12 @@ natsConnection_QueueSubscribe(natsSubscription **sub, natsConnection *nc, */ natsStatus natsConnection_QueueSubscribeTimeout(natsSubscription **sub, natsConnection *nc, - const char *subject, const char *queueGroup, - int64_t timeout, natsMsgHandler cb, void *cbClosure) + const char *subject, const char *queueGroup, + int64_t timeout, natsMsgHandler cb, void *cbClosure) { natsStatus s; - if ((queueGroup == NULL) || (strlen(queueGroup) == 0) || (cb == NULL) - || (timeout <= 0)) + if ((queueGroup == NULL) || (strlen(queueGroup) == 0) || (cb == NULL) || (timeout <= 0)) { return nats_setDefaultError(NATS_INVALID_ARG); } @@ -612,7 +651,7 @@ natsConnection_QueueSubscribeTimeout(natsSubscription **sub, natsConnection *nc, */ natsStatus natsConnection_QueueSubscribeSync(natsSubscription **sub, natsConnection *nc, - const char *subject, const char *queueGroup) + const char *subject, const char *queueGroup) { natsStatus s; @@ -645,13 +684,13 @@ natsSubscription_NoDeliveryDelay(natsSubscription *sub) natsStatus natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool pullSubInternal) { - natsStatus s = NATS_OK; - natsConnection *nc = NULL; - natsMsg *msg = NULL; - bool removeSub = false; - int64_t target = 0; - jsSub *jsi = NULL; - char *fcReply = NULL; + natsStatus s = NATS_OK; + natsConnection *nc = NULL; + natsMsg *msg = NULL; + bool removeSub = false; + int64_t target = 0; + jsSub *jsi = NULL; + char *fcReply = NULL; if ((sub == NULL) || (nextMsg == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); @@ -705,19 +744,16 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool } nc = sub->conn; - jsi= sub->jsi; + jsi = sub->jsi; if (timeout > 0) { - while ((sub->msgList.msgs == 0) - && (s != NATS_TIMEOUT) - && !(sub->closed) - && !(sub->draining)) + while ((sub->ownDispatcher.queue.msgs == 0) && (s != NATS_TIMEOUT) && !(sub->closed) && !(sub->draining)) { if (target == 0) target = nats_setTargetTime(timeout); - s = natsCondition_AbsoluteTimedWait(sub->cond, sub->mu, target); + s = natsCondition_AbsoluteTimedWait(sub->ownDispatcher.cond, sub->mu, target); if (s != NATS_OK) s = nats_setDefaultError(s); } @@ -729,14 +765,14 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool } else { - s = (sub->msgList.msgs == 0 ? NATS_TIMEOUT : NATS_OK); + s = (sub->ownDispatcher.queue.msgs == 0 ? NATS_TIMEOUT : NATS_OK); if ((s != NATS_OK) && !pullSubInternal) s = nats_setDefaultError(s); } if (s == NATS_OK) { - msg = sub->msgList.head; + msg = sub->ownDispatcher.queue.head; if ((msg == NULL) && sub->draining) { removeSub = true; @@ -744,13 +780,13 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool } else { - sub->msgList.head = msg->next; + sub->ownDispatcher.queue.head = msg->next; - if (sub->msgList.tail == msg) - sub->msgList.tail = NULL; + if (sub->ownDispatcher.queue.tail == msg) + sub->ownDispatcher.queue.tail = NULL; - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_dataAndHdrLen(msg); + sub->ownDispatcher.queue.msgs--; + sub->ownDispatcher.queue.bytes -= natsMsg_dataAndHdrLen(msg); msg->next = NULL; @@ -765,7 +801,7 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool removeSub = true; } - if (sub->draining && (sub->msgList.msgs == 0)) + if (sub->draining && (sub->ownDispatcher.queue.msgs == 0)) removeSub = true; } if (removeSub) @@ -815,10 +851,10 @@ natsSubscription_NextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeo static natsStatus _unsubscribe(natsSubscription *sub, int max, bool drainMode, int64_t timeout) { - natsStatus s = NATS_OK; - natsConnection *nc = NULL; - bool dc = false; - jsSub *jsi; + natsStatus s = NATS_OK; + natsConnection *nc = NULL; + bool dc = false; + jsSub *jsi; if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); @@ -853,6 +889,7 @@ _unsubscribe(natsSubscription *sub, int max, bool drainMode, int64_t timeout) natsStatus natsSubscription_Unsubscribe(natsSubscription *sub) { + natsStatus s = _unsubscribe(sub, 0, false, 0); return NATS_UPDATE_ERR_STACK(s); } @@ -864,43 +901,34 @@ natsSubscription_AutoUnsubscribe(natsSubscription *sub, int max) return NATS_UPDATE_ERR_STACK(s); } -void -natsSub_drain(natsSubscription *sub) +void natsSub_drain(natsSubscription *sub) { - natsSub_Lock(sub); - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return; } sub->draining = true; - if (sub->libDlvWorker != NULL) - { - // If this is a subscription with timeout, stop the timer. - if (sub->timeout != 0) - { - natsTimer_Stop(sub->timeoutTimer); - // Prevent code to reset this timer - sub->timeoutSuspended = true; - } - // Set this to true. It will be set to false in the - // worker delivery thread when the control message is - // processed. - sub->libDlvDraining = true; + // If this is a subscription with timeout, stop the timer. + if (sub->timeout != 0) + { + natsTimer_Stop(sub->timeoutTimer); + // Prevent code to reset this timer + sub->timeoutSuspended = true; + } - // Post a control message to wake-up the worker which will - // ensure that all pending messages for this subscription - // are removed and the subscription will ultimately be - // released in the worker thread. - natsLib_msgDeliveryPostControlMsg(sub); + if (sub->dispatcher != &sub->ownDispatcher) + { + natsSub_enqueueMessage(sub, sub->control->sub.drain); } else - natsCondition_Broadcast(sub->cond); - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + { + natsCondition_Broadcast(sub->ownDispatcher.cond); + } + + nats_unlockSubAndDispatcher(sub); } static void @@ -911,8 +939,7 @@ _updateDrainStatus(natsSubscription *sub, natsStatus s) sub->drainStatus = s; } -void -natsSub_updateDrainStatus(natsSubscription *sub, natsStatus s) +void natsSub_updateDrainStatus(natsSubscription *sub, natsStatus s) { natsSub_Lock(sub); _updateDrainStatus(sub, s); @@ -920,33 +947,30 @@ natsSub_updateDrainStatus(natsSubscription *sub, natsStatus s) } // Mark the subscription such that connection stops to try to push messages into its list. -void -natsSub_setDrainSkip(natsSubscription *sub, natsStatus s) +void natsSub_setDrainSkip(natsSubscription *sub, natsStatus s) { - natsSub_Lock(sub); - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); _updateDrainStatus(sub, s); sub->drainSkip = true; - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } static void _flushAndDrain(void *closure) { - natsSubscription *sub = (natsSubscription*) closure; - natsConnection *nc = NULL; - natsThread *t = NULL; - int64_t timeout = 0; - int64_t deadline = 0; - bool sync = false; - natsStatus s; + natsSubscription *sub = (natsSubscription *)closure; + natsConnection *nc = NULL; + natsThread *t = NULL; + int64_t timeout = 0; + int64_t deadline = 0; + bool sync = false; + natsStatus s; natsSub_Lock(sub); - nc = sub->conn; - t = sub->drainThread; + nc = sub->conn; + t = sub->drainThread; timeout = sub->drainTimeout; - sync = (sub->msgCb == NULL ? true : false); + sync = (sub->msgCb == NULL ? true : false); natsSub_Unlock(sub); // Make sure that negative value is considered no timeout. @@ -982,14 +1006,15 @@ _flushAndDrain(void *closure) // already called NextMsg() for all pending messages before the sub // was marked as "draining", so if we detect this situation, we need // to switch status to complete here. - if (sync && !natsSub_drainComplete(sub) && (sub->msgList.msgs == 0)) + + if (sync && !natsSub_drainComplete(sub) && (sub->ownDispatcher.queue.msgs == 0)) { _setDrainCompleteState(sub); } else { while ((s != NATS_TIMEOUT) && !natsSub_drainComplete(sub)) - s = natsCondition_AbsoluteTimedWait(sub->cond, sub->mu, deadline); + s = natsCondition_AbsoluteTimedWait(sub->drainCond, sub->mu, deadline); } natsSub_Unlock(sub); @@ -1006,8 +1031,7 @@ _flushAndDrain(void *closure) } // Switch subscription's drain state to "started". -void -natsSub_initDrain(natsSubscription *sub) +void natsSub_initDrain(natsSubscription *sub) { natsSub_Lock(sub); sub->drainState |= SUB_DRAIN_STARTED; @@ -1034,7 +1058,7 @@ natsSub_startDrain(natsSubscription *sub, int64_t timeout) // to make sure that this call will not block. s = natsConn_enqueueUnsubProto(sub->conn, sub->sid); if (s == NATS_OK) - s = natsThread_Create(&(sub->drainThread), _flushAndDrain, (void*) sub); + s = natsThread_Create(&(sub->drainThread), _flushAndDrain, (void *)sub); if (s == NATS_OK) { sub->drainTimeout = timeout; @@ -1067,9 +1091,9 @@ natsSubscription_DrainTimeout(natsSubscription *sub, int64_t timeout) natsStatus natsSubscription_WaitForDrainCompletion(natsSubscription *sub, int64_t timeout) { - natsStatus s = NATS_OK; - int64_t deadline = 0; - bool dc = false; + natsStatus s = NATS_OK; + int64_t deadline = 0; + bool dc = false; if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); @@ -1090,9 +1114,9 @@ natsSubscription_WaitForDrainCompletion(natsSubscription *sub, int64_t timeout) while ((s != NATS_TIMEOUT) && !natsSub_drainComplete(sub)) { if (timeout > 0) - s = natsCondition_AbsoluteTimedWait(sub->cond, sub->mu, deadline); + s = natsCondition_AbsoluteTimedWait(sub->drainCond, sub->mu, deadline); else - natsCondition_Wait(sub->cond, sub->mu); + natsCondition_Wait(sub->drainCond, sub->mu); } natsSub_Unlock(sub); @@ -1130,21 +1154,21 @@ natsSubscription_DrainCompletionStatus(natsSubscription *sub) natsStatus natsSubscription_QueuedMsgs(natsSubscription *sub, uint64_t *queuedMsgs) { - natsStatus s; - int msgs = 0; + natsStatus s; + int msgs = 0; if (queuedMsgs == NULL) return nats_setDefaultError(NATS_INVALID_ARG); s = natsSubscription_GetPending(sub, &msgs, NULL); if (s == NATS_OK) - *queuedMsgs = (uint64_t) msgs; + *queuedMsgs = (uint64_t)msgs; return s; } int64_t -natsSubscription_GetID(natsSubscription* sub) +natsSubscription_GetID(natsSubscription *sub) { int64_t id = 0; @@ -1166,10 +1190,10 @@ natsSubscription_GetID(natsSubscription* sub) return id; } -const char* -natsSubscription_GetSubject(natsSubscription* sub) +const char * +natsSubscription_GetSubject(natsSubscription *sub) { - const char* subject = NULL; + const char *subject = NULL; if (sub == NULL) return NULL; @@ -1182,38 +1206,36 @@ natsSubscription_GetSubject(natsSubscription* sub) return NULL; } - subject = (const char*)sub->subject; + subject = (const char *)sub->subject; natsSub_Unlock(sub); return subject; } +// This works for both shared and dedicated dispatchers since we maintain the +// per-sub stats. natsStatus natsSubscription_GetPending(natsSubscription *sub, int *msgs, int *bytes) { if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - if (msgs != NULL) - *msgs = sub->msgList.msgs; + *msgs = sub->ownDispatcher.queue.msgs; if (bytes != NULL) - *bytes = sub->msgList.bytes; + *bytes = sub->ownDispatcher.queue.bytes; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1226,24 +1248,20 @@ natsSubscription_SetPendingLimits(natsSubscription *sub, int msgLimit, int bytes if ((msgLimit == 0) || (bytesLimit == 0)) return nats_setError(NATS_INVALID_ARG, "%s", - "Limits must be either > 0 or negative to specify no limit"); + "Limits must be either > 0 or negative to specify no limit"); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - sub->msgsLimit = msgLimit; sub->bytesLimit = bytesLimit; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1254,25 +1272,21 @@ natsSubscription_GetPendingLimits(natsSubscription *sub, int *msgLimit, int *byt if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - if (msgLimit != NULL) *msgLimit = sub->msgsLimit; if (bytesLimit != NULL) *bytesLimit = sub->bytesLimit; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1283,21 +1297,17 @@ natsSubscription_GetDelivered(natsSubscription *sub, int64_t *msgs) if ((sub == NULL) || (msgs == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - - *msgs = (int64_t) sub->delivered; + *msgs = (int64_t)sub->delivered; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1308,21 +1318,17 @@ natsSubscription_GetDropped(natsSubscription *sub, int64_t *msgs) if ((sub == NULL) || (msgs == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - *msgs = sub->dropped; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1333,25 +1339,21 @@ natsSubscription_GetMaxPending(natsSubscription *sub, int *msgs, int *bytes) if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - if (msgs != NULL) *msgs = sub->msgsMax; if (bytes != NULL) *bytes = sub->bytesMax; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1362,53 +1364,47 @@ natsSubscription_ClearMaxPending(natsSubscription *sub) if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); - + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - sub->msgsMax = 0; sub->bytesMax = 0; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } natsStatus natsSubscription_GetStats(natsSubscription *sub, - int *pendingMsgs, - int *pendingBytes, - int *maxPendingMsgs, - int *maxPendingBytes, - int64_t *deliveredMsgs, - int64_t *droppedMsgs) + int *pendingMsgs, + int *pendingBytes, + int *maxPendingMsgs, + int *maxPendingBytes, + int64_t *deliveredMsgs, + int64_t *droppedMsgs) { if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - + // messages and bytes are up to date even with a shared dispatcher. if (pendingMsgs != NULL) - *pendingMsgs = sub->msgList.msgs; + *pendingMsgs = sub->ownDispatcher.queue.msgs; if (pendingBytes != NULL) - *pendingBytes = sub->msgList.bytes; + *pendingBytes = sub->ownDispatcher.queue.bytes; if (maxPendingMsgs != NULL) *maxPendingMsgs = sub->msgsMax; @@ -1417,14 +1413,12 @@ natsSubscription_GetStats(natsSubscription *sub, *maxPendingBytes = sub->bytesMax; if (deliveredMsgs != NULL) - *deliveredMsgs = (int) sub->delivered; + *deliveredMsgs = (int)sub->delivered; if (droppedMsgs != NULL) *droppedMsgs = sub->dropped; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1434,8 +1428,7 @@ natsSubscription_GetStats(natsSubscription *sub, * This will return false if the subscription has already been closed, * or auto unsubscribed. */ -bool -natsSubscription_IsValid(natsSubscription *sub) +bool natsSubscription_IsValid(natsSubscription *sub) { bool valid = false; @@ -1455,8 +1448,7 @@ natsSubscription_IsValid(natsSubscription *sub) * Destroys the subscription object, freeing up memory. * If not already done, this call will removes interest on the subject. */ -void -natsSubscription_Destroy(natsSubscription *sub) +void natsSubscription_Destroy(natsSubscription *sub) { bool doUnsub = false; @@ -1480,7 +1472,7 @@ natsSubscription_Destroy(natsSubscription *sub) natsSub_Unlock(sub); if (doUnsub) - (void) natsSubscription_Unsubscribe(sub); + (void)natsSubscription_Unsubscribe(sub); natsSub_release(sub); } diff --git a/src/sub.h b/src/sub.h index 876fabe3b..52f89023f 100644 --- a/src/sub.h +++ b/src/sub.h @@ -1,4 +1,4 @@ -// Copyright 2015-2021 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,32 +16,79 @@ #include "natsp.h" -#ifdef DEV_MODE -// For type safety... +#define SUB_DRAIN_STARTED ((uint8_t)1) +#define SUB_DRAIN_COMPLETE ((uint8_t)2) -void natsSub_Lock(natsSubscription *sub); -void natsSub_Unlock(natsSubscription *sub); - -#else - -#define natsSub_Lock(s) natsMutex_Lock((s)->mu) -#define natsSub_Unlock(s) natsMutex_Unlock((s)->mu) - -#endif // DEV_MODE - -#define SUB_DRAIN_STARTED ((uint8_t) 1) -#define SUB_DRAIN_COMPLETE ((uint8_t) 2) - -#define natsSub_drainStarted(s) (((s)->drainState & SUB_DRAIN_STARTED) != 0) -#define natsSub_drainComplete(s) (((s)->drainState & SUB_DRAIN_COMPLETE) != 0) +#define natsSub_drainStarted(s) (((s)->drainState & SUB_DRAIN_STARTED) != 0) +#define natsSub_drainComplete(s) (((s)->drainState & SUB_DRAIN_COMPLETE) != 0) extern bool testDrainAutoUnsubRace; -void -natsSub_retain(natsSubscription *sub); +// lock/unlock sub +static inline void natsSub_Lock(natsSubscription *sub) +{ + natsMutex_Lock(sub->mu); +} + +static inline void natsSub_Unlock(natsSubscription *sub) +{ + natsMutex_Unlock(sub->mu); +} + +// lock/unlock with retain/release +static inline void natsSub_lockRetain(natsSubscription *sub) +{ + natsSub_Lock(sub); + + sub->refs++; +} +void natsSub_unlockRelease(natsSubscription *sub); + + +// retain/release, lock is obtained +static inline void natsSub_retain(natsSubscription *sub) +{ + natsSub_lockRetain(sub); + natsSub_Unlock(sub); +} +void natsSub_release(natsSubscription *sub); + +// lock/unlock sub's dispatcher ONLY +static inline void natsSub_lockDispatcher(natsSubscription *sub) +{ + if (sub->dispatcher != &sub->ownDispatcher) + nats_lockDispatcher(sub->dispatcher); +} +static inline void natsSub_unlockDispatcher(natsSubscription *sub) +{ + if (sub->dispatcher != &sub->ownDispatcher) + nats_unlockDispatcher(sub->dispatcher); +} + +// lock/unlock sub and its dispatcher, together +static inline void nats_lockSubAndDispatcher(natsSubscription *sub) +{ + natsSub_Lock(sub); + natsSub_lockDispatcher(sub); +} +static inline void nats_unlockSubAndDispatcher(natsSubscription *sub) +{ + natsSub_unlockDispatcher(sub); + natsSub_Unlock(sub); +} + +// lock/unlock sub and its dispatcher, and retain/release the sub +static inline void nats_lockRetainSubAndDispatcher(natsSubscription *sub) +{ + natsSub_lockRetain(sub); + natsSub_lockDispatcher(sub); +} +static inline void nats_unlockReleaseSubAndDispatcher(natsSubscription *sub) +{ + natsSub_unlockDispatcher(sub); + natsSub_unlockRelease(sub); +} -void -natsSub_release(natsSubscription *sub); natsStatus natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, @@ -72,19 +119,9 @@ natsSub_drain(natsSubscription *sub); natsStatus natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool pullSubInternal); -void -natsSubAndLdw_Lock(natsSubscription *sub); - -void -natsSubAndLdw_Unlock(natsSubscription *sub); - -void -natsSubAndLdw_LockAndRetain(natsSubscription *sub); - -void -natsSubAndLdw_UnlockAndRelease(natsSubscription *sub); - void natsSub_close(natsSubscription *sub, bool connectionClosed); +natsStatus nats_createControlMessages(natsSubscription *sub); + #endif /* SUB_H_ */ diff --git a/src/timer.c b/src/timer.c index 7a449456f..ba2548aa2 100644 --- a/src/timer.c +++ b/src/timer.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,6 +14,7 @@ #include "natsp.h" #include "mem.h" #include "util.h" +#include "glib/glib.h" static void _freeTimer(natsTimer *t) @@ -75,6 +76,9 @@ natsTimer_Create(natsTimer **timer, natsTimerCb timerCb, natsTimerStopCb stopCb, void natsTimer_Stop(natsTimer *timer) { + if (timer == NULL) + return; + // Proxy for this call: nats_stopTimer(timer); } @@ -82,6 +86,8 @@ natsTimer_Stop(natsTimer *timer) void natsTimer_Reset(natsTimer *timer, int64_t interval) { + if (timer == NULL) + return; // Proxy for this call: nats_resetTimer(timer, interval); } diff --git a/src/unix/sock.c b/src/unix/sock.c index 94e512c21..4bfcfa39c 100644 --- a/src/unix/sock.c +++ b/src/unix/sock.c @@ -16,7 +16,7 @@ #include "../comsock.h" void -natsSys_Init(void) +nats_initForOS(void) { // Would do anything that needs to be initialized when // the library loads, specific to unix. diff --git a/src/unix/thread.c b/src/unix/thread.c index 2f7773844..624f66080 100644 --- a/src/unix/thread.c +++ b/src/unix/thread.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -13,6 +13,7 @@ #include "../natsp.h" #include "../mem.h" +#include "../glib/glib.h" bool nats_InitOnce(natsInitOnceType *control, natsInitOnceCb cb) diff --git a/src/win/sock.c b/src/win/sock.c index ea2845e07..74a808264 100644 --- a/src/win/sock.c +++ b/src/win/sock.c @@ -20,7 +20,7 @@ #include "../comsock.h" void -natsSys_Init(void) +nats_initForOS(void) { WSADATA wsaData; int errorno; diff --git a/src/win/thread.c b/src/win/thread.c index c85e828ae..b50de09a4 100644 --- a/src/win/thread.c +++ b/src/win/thread.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,6 +16,7 @@ #include #include "../mem.h" +#include "../glib/glib.h" static BOOL CALLBACK _initHandleFunction(PINIT_ONCE InitOnce, PVOID Parameter, PVOID *lpContext) diff --git a/test/bench_sub_async.c b/test/bench_sub_async.c index 374f0178b..47c58c7ae 100644 --- a/test/bench_sub_async.c +++ b/test/bench_sub_async.c @@ -413,83 +413,16 @@ static natsStatus _inject(natsConnection *nc, const char *subject, ENV *env) snprintf(buf, sizeof(buf), "%d", i); s = natsMsg_Create(&m, subject, NULL, buf, (int)strlen(buf)); - IFOK(s, _enqueueToSub(env->subs[n].sub, m)); + natsSubscription *sub = env->subs[n].sub; + nats_lockSubAndDispatcher(sub); + IFOK(s, natsSub_enqueueUserMessage(sub, m)); + nats_unlockSubAndDispatcher(sub); } } return s; } -static natsStatus _enqueueToSub(natsSubscription *sub, natsMsg *m) -{ - natsStatus s = NATS_OK; - natsCondition *cond = NULL; - nats_MsgList *list = NULL; - natsMsgDlvWorker *ldw = NULL; - - // Pick condition variable and list based on if the sub is - // part of a global delivery thread pool or not. - // Note about `list`: this is used only to link messages, but - // sub->msgList needs to be used to update/check number of pending - // messages, since in case of delivery thread pool, `list` will have - // messages from many different subscriptions. - if ((ldw = sub->libDlvWorker) != NULL) - { - cond = ldw->cond; - list = &(ldw->msgList); - } - else - { - cond = sub->cond; - list = &(sub->msgList); - } - - natsSubAndLdw_LockAndRetain(sub); - - sub->msgList.msgs++; - sub->msgList.bytes += natsMsg_GetDataLength(m); - if (((sub->msgsLimit > 0) && (sub->msgList.msgs > sub->msgsLimit)) || ((sub->bytesLimit > 0) && (sub->msgList.bytes > sub->bytesLimit))) - { - natsMsg_Destroy(m); - - sub->dropped++; - sub->slowConsumer = true; - - // Undo stats from above. - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_GetDataLength(m); - } - else - { - bool signal = false; - - if (sub->msgList.msgs > sub->msgsMax) - sub->msgsMax = sub->msgList.msgs; - - if (sub->msgList.bytes > sub->bytesMax) - sub->bytesMax = sub->msgList.bytes; - - sub->slowConsumer = false; - - m->sub = sub; - if (list->head == NULL) - { - list->head = m; - signal = true; - } - else - list->tail->next = m; - - list->tail = m; - - if (signal) - natsCondition_Signal(cond); - } - - natsSubAndLdw_UnlockAndRelease(sub); - return s; -} - static uint64_t _expectedSum(int N) { uint64_t sum = 0; diff --git a/test/list_test.txt b/test/list_test.txt index 4b2bd6cf0..b53816d54 100644 --- a/test/list_test.txt +++ b/test/list_test.txt @@ -1,3 +1,4 @@ +_test(AssignSubToDispatch) _test(AsyncErrHandlerMaxPendingBytes) _test(AsyncErrHandlerMaxPendingMsgs) _test(AsyncErrHandlerSubDestroyed) @@ -124,7 +125,6 @@ _test(KeyValueRePublish) _test(KeyValueWatch) _test(KeyValueWatchMulti) _test(LameDuckMode) -_test(LibMsgDelivery) _test(MessageBufferPadding) _test(MicroAddService) _test(MicroAsyncErrorHandlerMaxPendingBytes) diff --git a/test/test.c b/test/test.c index 48cf966f8..25993a167 100644 --- a/test/test.c +++ b/test/test.c @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -39,6 +39,7 @@ #include "js.h" #include "kv.h" #include "microp.h" +#include "glib/glibp.h" #if defined(NATS_HAS_STREAMING) @@ -71,9 +72,8 @@ testInfo allTests[] = }; #undef _TEST_LIST - static int tests = 0; -bool failed = false; +bool failed = false; bool keepServerOutput = false; static bool valgrind = false; @@ -86,7 +86,9 @@ static const char *natsStreamingServerExe = "nats-streaming-server"; natsMutex *slMu = NULL; natsHash *slMap = NULL; -#define test(s) { printf("#%02d ", ++tests); printf("%s", (s)); fflush(stdout); } +#define test(s) { printf("#%02d ", ++tests); printf("%s\n", (s)); fflush(stdout); } +#define testf(s, ...) { printf("#%02d ", ++tests); printf((s "\n"), __VA_ARGS__); fflush(stdout); } + #ifdef _WIN32 #define testCond(c) if(c) { printf("PASSED\n"); fflush(stdout); } else { printf("FAILED\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; return; } #define testCondNoReturn(c) if(c) { printf("PASSED\n"); fflush(stdout); } else { printf("FAILED\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; } @@ -123,6 +125,7 @@ struct threadArg natsStrHash *inboxes; natsStatus status; const char* string; + int N; bool connected; bool disconnected; int64_t disconnectedAt[4]; @@ -2611,7 +2614,7 @@ void test_natsOptions(void) && (opts->token == NULL) && (opts->tokenCb == NULL) && (opts->orderIP == 0) - && (opts->writeDeadline == natsLib_defaultWriteDeadline()) + && (opts->writeDeadline == nats_lib()->config.DefaultWriteDeadline) && !opts->noEcho && !opts->retryOnFailedConnect && !opts->ignoreDiscoveredServers @@ -5828,6 +5831,7 @@ _recvTestString(natsConnection *nc, natsSubscription *sub, natsMsg *msg, natsMutex_Lock(arg->m); + // up to 12 now switch (arg->control) { case 0: @@ -5862,15 +5866,26 @@ _recvTestString(natsConnection *nc, natsSubscription *sub, natsMsg *msg, } case 3: case 9: + case 12: { doSignal = false; arg->sum++; - if ((arg->control != 9) && (arg->sum == 10)) + if ((arg->control == 3) && (arg->sum == 10)) { arg->status = natsSubscription_Unsubscribe(sub); doSignal = true; } + else if ((arg->control == 12) && (arg->sum == arg->N)) + { + arg->status = NATS_OK; + doSignal = true; + } + else if ((arg->control == 12) && (arg->sum > arg->N)) + { + doSignal = true; + arg->status = NATS_ERR; + } break; } case 11: @@ -7680,148 +7695,211 @@ _dummyMsgHandler(natsConnection *nc, natsSubscription *sub, natsMsg *msg, natsMsg_Destroy(msg); } -void test_LibMsgDelivery(void) +static int _numRunningThreads(natsDispatcherPool *pool) { - natsStatus s; - natsPid serverPid = NATS_INVALID_PID; - natsOptions *opts = NULL; - natsConnection *nc = NULL; - natsSubscription *s1 = NULL; - natsSubscription *s2 = NULL; - natsSubscription *s3 = NULL; - natsSubscription *s4 = NULL; - natsSubscription *s5 = NULL; - natsMsgDlvWorker *lmd1 = NULL; - natsMsgDlvWorker *lmd2 = NULL; - natsMsgDlvWorker *lmd3 = NULL; - natsMsgDlvWorker *lmd4 = NULL; - natsMsgDlvWorker *lmd5 = NULL; - natsMsgDlvWorker **pwks = NULL; - int psize = 0; - int pmaxSize = 0; - int pidx = 0; + int i, n; + for (n = 0, i = 0; i < pool->cap; i++) + n += (pool->dispatchers[i].thread != NULL); + return n; +} - // First, close the library and re-open, to reset things - nats_Close(); +void test_AssignSubToDispatch(void) +{ + natsStatus s; +#define MAX_SUBS 500 + natsPid serverPid = NATS_INVALID_PID; + natsConnection *nc = NULL; + natsSubscription *subs[MAX_SUBS]; + natsClientConfig config4 = { + .DefaultToThreadPool = true, + .ThreadPoolMax = 4, + }; + natsClientConfig config50 = { + .DefaultToThreadPool = true, + .ThreadPoolMax = 50, + }; + natsDispatcherPool *pool = NULL; + natsDispatcherPool *rpool = NULL; + natsClientConfig c; + struct threadArg arg; + char subSubj[32]; + char pubSubj[32]; + int i=0, n=0; - nats_Sleep(100); + const int numMsgs = 5; - nats_Open(-1); + typedef struct + { + int numSubs; + natsClientConfig *config; + int expectedMax; + int expectedDispatchers; + bool expectedDefaultPool; + } TC; - // Check some pre-conditions that need to be met for the test to work. - test("Check initial values: ") - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - testCond((pmaxSize == 1) && (psize == 0) && (pidx == 0)); + TC tcs[] = { + { + .numSubs = 10, + .expectedMax = 1, + .expectedDispatchers = 0, // all should be dispatched with dedicated threads + }, + { + .config = &config4, + .numSubs = 3, + .expectedMax = 4, + .expectedDispatchers = 3, + .expectedDefaultPool = true, + }, + { + .config = &config4, + .numSubs = 23, + .expectedMax = 4, + .expectedDispatchers = 4, + .expectedDefaultPool = true, + }, + { + .numSubs = 77, + .config = &config50, + .expectedMax = 50, + .expectedDispatchers = 50, + .expectedDefaultPool = true, + }, + }; + TC *tc, *end = tcs + (sizeof(tcs) / sizeof(tcs[0])); - test("Check pool size not negative: ") + test("Check pool size not negative: "); s = nats_SetMessageDeliveryPoolSize(-1); testCond(s != NATS_OK); - test("Check pool size not zero: ") + test("Check pool size not zero: "); s = nats_SetMessageDeliveryPoolSize(0); testCond(s != NATS_OK); // Reset stack since we know the above generated errors. nats_clearLastError(); - test("Increase size to 2: ") - s = nats_SetMessageDeliveryPoolSize(2); - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - testCond((s == NATS_OK) && (pmaxSize == 2) && (psize == 0)); - - test("Check pool size decreased (no error): ") - s = nats_SetMessageDeliveryPoolSize(1); - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - testCond((s == NATS_OK) && (pmaxSize == 2) && (psize == 0)); + s = _createDefaultThreadArgsForCbTests(&arg); + if (s != NATS_OK) + FAIL("Unable to setup test"); serverPid = _startServer("nats://127.0.0.1:4222", NULL, true); CHECK_SERVER_STARTED(serverPid); - s = natsOptions_Create(&opts); - IFOK(s, natsOptions_UseGlobalMessageDelivery(opts, true)); - IFOK(s, natsConnection_Connect(&nc, opts)); - IFOK(s, natsConnection_Subscribe(&s1, nc, "foo", _dummyMsgHandler, NULL)); - if (s == NATS_OK) - { - natsMutex_Lock(s1->mu); - lmd1 = s1->libDlvWorker; - natsMutex_Unlock(s1->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 1st sub assigned 1st worker: ") - testCond((s == NATS_OK) && (psize == 1) && (lmd1 != NULL) - && (pidx == 1) && (pwks != NULL) && (lmd1 == pwks[0])); + // First, close the library and re-open, to reset things + test("Reset the library's global state: "); + nats_CloseAndWait(100); + testCond(true); - s = natsConnection_Subscribe(&s2, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) + for (tc = tcs; tc < end; tc++, n++) { - natsMutex_Lock(s2->mu); - lmd2 = s2->libDlvWorker; - natsMutex_Unlock(s2->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 2nd sub assigned 2nd worker: ") - testCond((s == NATS_OK) && (psize == 2) && (lmd2 != lmd1) - && (pidx == 0) && (pwks != NULL) && (lmd2 == pwks[1])); + testf("%d subs over %d threads: Re-open, check settings, ensure no shared dispatchers created: ", tc->numSubs, tc->expectedDispatchers); + s = nats_OpenWithConfig(tc->config); + IFOK(s, (c = nats_lib()->config, NATS_OK)); + if (s == NATS_OK) + { + pool = &nats_lib()->messageDispatchers; + rpool = &nats_lib()->replyDispatchers; + } + natsMutex_Lock(pool->lock); + natsMutex_Lock(rpool->lock); + testCond((s == NATS_OK) && + (c.DefaultToThreadPool == tc->expectedDefaultPool) && + (c.ThreadPoolMax == tc->expectedMax) && + (c.DefaultRepliesToThreadPool == false) && + (c.ReplyThreadPoolMax == 0) && + (c.UseSeparatePoolForReplies == false) && + (_numRunningThreads(pool) == 0) && + (pool->cap == tc->expectedMax) && + (pool->dispatchers != NULL) && + (_numRunningThreads(rpool) == 0) && + (rpool->dispatchers == NULL)); + natsMutex_Unlock(rpool->lock); + natsMutex_Unlock(pool->lock); + + testf("%d subs over %d threads: Initial connect: ", tc->numSubs, tc->expectedDispatchers); + s = natsConnection_ConnectTo(&nc, NATS_DEFAULT_URL); + testCond(s == NATS_OK); - s = natsConnection_Subscribe(&s3, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) - { - natsMutex_Lock(s3->mu); - lmd3 = s3->libDlvWorker; - natsMutex_Unlock(s3->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 3rd sub assigned 1st worker: ") - testCond((s == NATS_OK) && (psize == 2) && (lmd3 == lmd1) - && (pidx == 1) && (pwks != NULL) && (lmd3 == pwks[0])); + testf("%d subs over %d threads: Create %d subscriptions: ", tc->numSubs, tc->expectedDispatchers, tc->numSubs); + if (tc->numSubs > MAX_SUBS) + FAIL("Too many subscriptions for this test's buffer"); - // Bump the pool size to 4 - s = nats_SetMessageDeliveryPoolSize(4); - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check increase of pool size: "); - testCond((s == NATS_OK) && (psize == 2) && (pidx == 1) - && (pmaxSize == 4) && (pwks != NULL)); + arg.string = "test"; + arg.status = NATS_OK; + arg.control = 12; + arg.N = tc->numSubs * numMsgs; + arg.sum = 0; + snprintf(subSubj, sizeof(subSubj), "foo.%d.*", n); + for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) + { + s = natsConnection_Subscribe(&subs[i], nc, subSubj, _recvTestString, &arg); + } + testCond(s == NATS_OK); - s = natsConnection_Subscribe(&s4, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) - { - natsMutex_Lock(s4->mu); - lmd4 = s4->libDlvWorker; - natsMutex_Unlock(s4->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 4th sub assigned 2nd worker: ") - testCond((s == NATS_OK) && (psize == 2) && (lmd4 == lmd2) - && (pidx == 2) && (pwks != NULL) && (lmd4 == pwks[1])); + testf("%d subs over %d threads: Check shared dispatchers assigned as expected: ", tc->numSubs, tc->expectedDispatchers); + natsMutex_Lock(pool->lock); + natsMutex_Lock(rpool->lock); + testCond((pool->dispatchers != NULL) && + (_numRunningThreads(pool) == tc->expectedDispatchers) && + (_numRunningThreads(rpool) == 0)); - s = natsConnection_Subscribe(&s5, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) - { - natsMutex_Lock(s5->mu); - lmd5 = s5->libDlvWorker; - natsMutex_Unlock(s5->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 5th sub assigned 3rd worker: ") - testCond((s == NATS_OK) && (psize == 3) && (lmd5 != lmd2) - && (pidx == 3) && (pwks != NULL) && (lmd5 == pwks[2])); + if (tc->config != NULL) + { + testf("%d subs over %d threads: Verify that the dispatchers have been assigned: ", tc->numSubs, tc->expectedDispatchers); + for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) + { + natsSub_Lock(subs[i]); + if (subs[i]->dispatcher != &pool->dispatchers[i % tc->expectedDispatchers]) + s = NATS_ERR; + natsSub_Unlock(subs[i]); + } + testCond(s == NATS_OK); + } + natsMutex_Unlock(rpool->lock); + natsMutex_Unlock(pool->lock); - natsSubscription_Destroy(s5); - natsSubscription_Destroy(s4); - natsSubscription_Destroy(s3); - natsSubscription_Destroy(s2); - natsSubscription_Destroy(s1); - natsConnection_Destroy(nc); - natsOptions_Destroy(opts); - _stopServer(serverPid); + testf("%d subs over %d threads: Publish %d messages and verify received %d times: ", tc->numSubs, tc->expectedDispatchers, numMsgs, arg.N); + for (i = 0; (s == NATS_OK) && (i < numMsgs); i++) + { + snprintf(pubSubj, sizeof(pubSubj), "foo.%d.%d", n, i); + s = natsConnection_PublishString(nc, pubSubj, "test"); + } + if (s == NATS_OK) + { + natsMutex_Lock(arg.m); + while ((s != NATS_TIMEOUT) && !arg.msgReceived) + s = natsCondition_TimedWait(arg.c, arg.m, 1500); + arg.msgReceived = false; + arg.status = NATS_OK; + natsMutex_Unlock(arg.m); + if (s == NATS_OK) + s = arg.status; + } + testCond(s == NATS_OK); - // Close the library and re-open, to reset things - nats_Close(); + testf("%d subs over %d threads: Drain the subs: ", tc->numSubs, tc->expectedDispatchers); + for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) + { + s = natsSubscription_Drain(subs[i]); + natsSubscription_Destroy(subs[i]); + } + testCond(s == NATS_OK); - nats_Sleep(100); + testf("%d subs over %d threads: Make sure we didn't get unexpected messages: ", tc->numSubs, tc->expectedDispatchers); + natsMutex_Lock(arg.m); + while ((s != NATS_TIMEOUT) && !arg.msgReceived) + s = natsCondition_TimedWait(arg.c, arg.m, 50); + natsMutex_Unlock(arg.m); + testCond((s == NATS_TIMEOUT) && !arg.msgReceived && arg.status != NATS_ERR); - nats_Open(-1); + testf("%d subs over %d threads: cleanup before next test: ", tc->numSubs, tc->expectedDispatchers); + natsConnection_Destroy(nc); + nats_Close(); + testCond(true); + } + _stopServer(serverPid); + + _destroyDefaultThreadArgs(&arg); } void test_DefaultConnection(void) @@ -10928,7 +11006,6 @@ void test_AsyncSubscribeTimeout(void) struct threadArg arg; bool useLibDlv = false; int i; - char testText[128]; int64_t timeout = 100; _asyncTimeoutInfo ai; @@ -10950,10 +11027,9 @@ void test_AsyncSubscribeTimeout(void) serverPid = _startServer("nats://127.0.0.1:4222", NULL, true); CHECK_SERVER_STARTED(serverPid); - snprintf(testText, sizeof(testText), "Test async %ssubscriber timeout%s: ", + testf("Test async %ssubscriber timeout%s: ", ((i == 1 || i == 3) ? "queue " : ""), (i > 1 ? " (lib msg delivery)" : "")); - test(testText); s = natsConnection_Connect(&nc, opts); if (s == NATS_OK) { @@ -11588,11 +11664,11 @@ void test_DoubleUnsubscribe(void) void test_SubRemovedWhileProcessingMsg(void) { - natsStatus s; - natsConnection *nc = NULL; - natsOptions *opts = NULL; - natsSubscription *sub = NULL; - natsPid serverPid = NATS_INVALID_PID; + natsStatus s; + natsConnection *nc = NULL; + natsOptions *opts = NULL; + natsSubscription *sub = NULL; + natsPid serverPid = NATS_INVALID_PID; serverPid = _startServer("nats://127.0.0.1:4222", NULL, true); CHECK_SERVER_STARTED(serverPid); @@ -11617,7 +11693,7 @@ void test_SubRemovedWhileProcessingMsg(void) test("Check msg not given: "); natsSub_Lock(sub); - testCond(sub->msgList.msgs == 0); + testCond(sub->ownDispatcher.queue.msgs == 0); natsSub_Unlock(sub); natsSubscription_Destroy(sub); @@ -11636,24 +11712,20 @@ void test_SubRemovedWhileProcessingMsg(void) IFOK(s, natsConnection_Subscribe(&sub, nc, "foo", _dummyMsgHandler, NULL)); testCond(s == NATS_OK); - natsSub_Lock(sub); - natsMutex_Lock(sub->libDlvWorker->lock); + nats_lockSubAndDispatcher(sub); test("Send message: "); s = natsConnection_PublishString(nc, "foo", "hello"); testCond(s == NATS_OK); test("Close sub: "); - natsMutex_Unlock(sub->libDlvWorker->lock); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); natsSub_close(sub, false); testCond(s == NATS_OK); test("Check msg not given: "); - natsSub_Lock(sub); - natsMutex_Lock(sub->libDlvWorker->lock); - testCond(sub->msgList.msgs == 0); - natsMutex_Unlock(sub->libDlvWorker->lock); - natsSub_Unlock(sub); + nats_lockSubAndDispatcher(sub); + testCond(sub->ownDispatcher.queue.msgs == 0); + nats_unlockSubAndDispatcher(sub); natsSubscription_Destroy(sub); natsConnection_Destroy(nc); @@ -13194,12 +13266,14 @@ void test_SlowAsyncSubscriber(void) CHECK_SERVER_STARTED(serverPid); s = natsConnection_Connect(&nc, opts); - IFOK(s, natsConnection_Subscribe(&sub, nc, "foo", _recvTestString, (void*) &arg)); + IFOK(s, natsConnection_Subscribe(&sub, nc, "foo.*", _recvTestString, (void*) &arg)); for (int i=0; - (s == NATS_OK) && (i < (total + 100)); i++) + (s == NATS_OK) && (i < (2 * total)); i++) { - s = natsConnection_PublishString(nc, "foo", "hello"); + char subj[32]; + snprintf(subj, sizeof(subj), "foo.%d", i); + s = natsConnection_PublishString(nc, subj, "hello"); } test("Check Publish does not fail due to SlowConsumer: "); @@ -13232,21 +13306,21 @@ void test_SlowAsyncSubscriber(void) // Release the sub natsMutex_Lock(arg.m); - // Unblock the wait - arg.closed = true; - // And destroy the subscription here so that the next msg callback // is not invoked. natsSubscription_Destroy(sub); + // Unblock the wait, let the callback finish + test("Unblock and wait for the callback to finish: "); + arg.closed = true; natsCondition_Signal(arg.c); arg.msgReceived = false; natsMutex_Unlock(arg.m); - // Let the callback finish natsMutex_Lock(arg.m); while (!arg.msgReceived) natsCondition_TimedWait(arg.c, arg.m, 5000); + testCond(arg.msgReceived); natsMutex_Unlock(arg.m); natsOptions_Destroy(opts); @@ -17483,6 +17557,7 @@ void test_DrainSub(void) arg.closed = true; natsCondition_Signal(arg.c); natsMutex_Unlock(arg.m); + testCond(s == NATS_OK); test("Wait for Drain to complete: "); s = natsSubscription_WaitForDrainCompletion(sub, -1); @@ -25375,11 +25450,11 @@ void test_JetStreamPublishAsync(void) _waitSubPending(rsub, 0); natsSub_Lock(rsub); - rsub->msgList.head = msg; - rsub->msgList.tail = msg; - rsub->msgList.msgs = 1; - rsub->msgList.bytes = natsMsg_dataAndHdrLen(msg); - natsCondition_Signal(rsub->cond); + rsub->ownDispatcher.queue.head = msg; + rsub->ownDispatcher.queue.tail = msg; + rsub->ownDispatcher.queue.msgs = 1; + rsub->ownDispatcher.queue.bytes = natsMsg_dataAndHdrLen(msg); + natsCondition_Signal(rsub->ownDispatcher.cond); natsSub_Unlock(rsub); // Message is owned by subscription, do not destroy it here. @@ -27479,9 +27554,9 @@ void test_JetStreamSubscribeIdleHearbeat(void) test("Check HB received: "); nats_Sleep(300); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); s = (sub->jsi->mismatch.dseq == 1 ? NATS_OK : NATS_ERR); - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); testCond(s == NATS_OK); test("Check HB is not given to app: "); @@ -27521,10 +27596,10 @@ void test_JetStreamSubscribeIdleHearbeat(void) // server state. #define PUBLISH_FAKE_JS_MSG_WITH_SEQ(_reply, _msg) \ { \ - natsSub_Lock(sub); \ + nats_lockSubAndDispatcher(sub); \ inbox = sub->subject; \ sub->jsi->ackNone = true; \ - natsSub_Unlock(sub); \ + nats_unlockSubAndDispatcher(sub); \ \ natsConn_setFilterWithClosure(nc, _setMsgReply, (void *)(_reply)); \ s = natsConnection_PublishString(nc, inbox, (_msg)); \ @@ -27572,9 +27647,9 @@ void test_JetStreamSubscribeIdleHearbeat(void) // Send real message so that all clears up s = js_Publish(NULL, js, "foo", "msg3", 4, NULL, &jerr); nats_Sleep(300); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); s = (sub->jsi->ssmn == false ? NATS_OK : NATS_ERR); - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); testCond(s == NATS_OK); test("Skip again: "); @@ -27676,9 +27751,9 @@ void test_JetStreamSubscribeIdleHearbeat(void) // Send real message so that all clears up s = js_Publish(NULL, js, "foo", "msg4", 4, NULL, &jerr); nats_Sleep(300); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); s = (sub->jsi->ssmn == false && sub->jsi->sm == false ? NATS_OK : NATS_ERR); - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); testCond(s == NATS_OK); test("Skip again: "); @@ -35411,7 +35486,7 @@ _subDlvThreadPooled(natsSubscription *sub) { bool pooled; natsSub_Lock(sub); - pooled = (sub->libDlvWorker != NULL); + pooled = (sub->dispatcher->dedicatedTo == NULL); natsSub_Unlock(sub); return pooled; } From 01cfb28581f237183c17255ee6349421da8b5f84 Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Thu, 8 Aug 2024 07:30:38 -0600 Subject: [PATCH 09/13] Fixed flappers and test_SSLHandshakeFirst by skipping if server is < 2.10.0 (#784) * Fixed SSLHandshakeFirst test by skipping if server is < 2.10.0 Signed-off-by: Ivan Kozlovic * More flappers Added missing `-a 127.0.0.1` in some tests to prevent the server from sending other IP addresses to the client. Signed-off-by: Ivan Kozlovic --------- Signed-off-by: Ivan Kozlovic --- test/test.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/test/test.c b/test/test.c index 25993a167..a7aa56118 100644 --- a/test/test.c +++ b/test/test.c @@ -7847,7 +7847,7 @@ void test_AssignSubToDispatch(void) { testf("%d subs over %d threads: Verify that the dispatchers have been assigned: ", tc->numSubs, tc->expectedDispatchers); for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) - { + { natsSub_Lock(subs[i]); if (subs[i]->dispatcher != &pool->dispatchers[i % tc->expectedDispatchers]) s = NATS_ERR; @@ -18352,7 +18352,7 @@ void test_GetClientID(void) testCond(true); return; } - pid1 = _startServer("nats://127.0.0.1:4222", "-cluster nats://127.0.0.1:6222 -cluster_name abc", true); + pid1 = _startServer("nats://127.0.0.1:4222", "-a 127.0.0.1 -p 4222 -cluster nats://127.0.0.1:6222 -cluster_name abc", true); CHECK_SERVER_STARTED(pid1); test("Create nc1: "); @@ -18371,7 +18371,7 @@ void test_GetClientID(void) testCond((s == NATS_OK) && (cid != 0)); test("Wait for discovered callback: "); - pid2 = _startServer("nats://127.0.0.1:4223", "-p 4223 -cluster nats://127.0.0.1:6223 -cluster_name abc -routes nats://127.0.0.1:6222", true); + pid2 = _startServer("nats://127.0.0.1:4223", "-a 127.0.0.1 -p 4223 -cluster nats://127.0.0.1:6223 -cluster_name abc -routes nats://127.0.0.1:6222", true); CHECK_SERVER_STARTED(pid2); natsMutex_Lock(arg.m); @@ -21289,6 +21289,16 @@ void test_SSLHandshakeFirst(void) natsOptions *opts = NULL; natsPid serverPid = NATS_INVALID_PID; + if (!serverVersionAtLeast(2, 10, 0)) + { + char txt[200]; + + snprintf(txt, sizeof(txt), "Skipping since requires server version of at least 2.10.0, got %s: ", serverVersion); + test(txt); + testCond(true); + return; + } + serverPid = _startServer("nats://127.0.0.1:4443", "-config tlsfirst.conf", true); CHECK_SERVER_STARTED(serverPid); @@ -21411,18 +21421,15 @@ void test_SSLReconnectWithAuthError(void) IFOK(s, natsOptions_SetTimeout(opts, 250)); IFOK(s, natsOptions_SetMaxReconnect(opts, 1000)); IFOK(s, natsOptions_SetReconnectWait(opts, 100)); - IFOK(s, natsOptions_SetReconnectJitter(opts, 0, 0)); IFOK(s, natsOptions_SetClosedCB(opts, _closedCb, (void*) &args)); IFOK(s, natsOptions_SetServers(opts, (const char*[2]){"tls://user:pwd@127.0.0.1:4443", "tls://bad:pwd@127.0.0.1:4444"}, 2)); - IFOK(s, natsOptions_SetNoRandomize(opts, true)); - IFOK(s, natsOptions_SetIgnoreDiscoveredServers(opts, true)); if (opts == NULL) FAIL("Unable to create reconnect options!"); - pid1 = _startServer("nats://127.0.0.1:4443", "-p 4443 -cluster_name abc -cluster nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); + pid1 = _startServer("nats://127.0.0.1:4443", "-a 127.0.0.1 -p 4443 -cluster_name abc -cluster nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); CHECK_SERVER_STARTED(pid1); - pid2 = _startServer("nats://127.0.0.1:4444", "-p 4444 -cluster_name abc -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); + pid2 = _startServer("nats://127.0.0.1:4444", "-a 127.0.0.1 -p 4444 -cluster_name abc -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); CHECK_SERVER_STARTED(pid2); test("Connect to server1: "); @@ -21636,7 +21643,7 @@ void test_ReconnectImplicitUserInfo(void) "}\n"\ "no_auth_user: b\n"); test("Start server1: "); - snprintf(cmdLine, sizeof(cmdLine), "-cluster_name \"local\" -cluster nats://127.0.0.1:6222 -c %s", conf); + snprintf(cmdLine, sizeof(cmdLine), "-a 127.0.0.1 -p 4222 -cluster_name \"local\" -cluster nats://127.0.0.1:6222 -c %s", conf); pid1 = _startServer("nats://127.0.0.1:4222", cmdLine, true); CHECK_SERVER_STARTED(pid1); testCond(true); @@ -21652,7 +21659,7 @@ void test_ReconnectImplicitUserInfo(void) testCond(s == NATS_OK); test("Start server2: "); - snprintf(cmdLine, sizeof(cmdLine), "-p 4223 -cluster_name \"local\" -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -c %s", conf); + snprintf(cmdLine, sizeof(cmdLine), "-a 127.0.0.1 -p 4223 -cluster_name \"local\" -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -c %s", conf); pid2 = _startServer("nats://127.0.0.1:4223", cmdLine, true); CHECK_SERVER_STARTED(pid2); testCond(true); From c4a26a5d76c560807c88f91fcdbddc0680e9c08e Mon Sep 17 00:00:00 2001 From: Thierry Bastian Date: Mon, 12 Aug 2024 18:47:04 +0200 Subject: [PATCH 10/13] Adding SNI extension if available (#787) * Adding SNI extension if available * Adding a unit test to check the SNI is present * code review --- src/conn.c | 7 +++++ test/list_test.txt | 1 + test/test.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/src/conn.c b/src/conn.c index 54bb697dd..37fcc1790 100644 --- a/src/conn.c +++ b/src/conn.c @@ -737,6 +737,13 @@ _makeTLSConn(natsConnection *nc) SSL_set_verify(ssl, SSL_VERIFY_PEER, _collectSSLErr); } } +#if defined(NATS_USE_OPENSSL_1_1) + // add the host name in the SNI extension + if ((s == NATS_OK) && (nc->cur != NULL) && (!SSL_set_tlsext_host_name(ssl, nc->cur->url->host))) + { + s = nats_setError(NATS_SSL_ERROR, "unable to set SNI extension for hostname '%s'", nc->cur->url->host); + } +#endif if ((s == NATS_OK) && (SSL_do_handshake(ssl) != 1)) { s = nats_setError(NATS_SSL_ERROR, diff --git a/test/list_test.txt b/test/list_test.txt index b53816d54..496d5b3f8 100644 --- a/test/list_test.txt +++ b/test/list_test.txt @@ -247,6 +247,7 @@ _test(SSLCertAndKeyFromMemory) _test(SSLCiphers) _test(SSLConnectVerboseOption) _test(SSLHandshakeFirst) +_test(SSLServerNameIndication) _test(SSLLoadCAFromMemory) _test(SSLMultithreads) _test(SSLReconnectWithAuthError) diff --git a/test/test.c b/test/test.c index a7aa56118..5f2965a61 100644 --- a/test/test.c +++ b/test/test.c @@ -21347,6 +21347,84 @@ void test_SSLHandshakeFirst(void) #endif } +void test_SSLServerNameIndication(void) +{ +#if defined(NATS_HAS_TLS) + natsStatus s = NATS_OK; + natsSock sock = NATS_SOCK_INVALID; + natsThread *t = NULL; + struct threadArg arg; + natsSockCtx ctx; + static const char *server = "tls://localhost:4222"; + + memset(&ctx, 0, sizeof(natsSockCtx)); + + s = _createDefaultThreadArgsForCbTests(&arg); + IFOK(s, natsOptions_Create(&(arg.opts))); + IFOK(s, natsOptions_SetSecure(arg.opts, true)); + IFOK(s, natsOptions_TLSHandshakeFirst(arg.opts)); + IFOK(s, natsOptions_SetServers(arg.opts, &server, 1)); + if (s != NATS_OK) + FAIL("@@ Unable to setup test!"); + + test("Start server and connect client: ") + + arg.control = 3; + + s = _startMockupServer(&sock, "localhost", "4222"); + + // Start the thread that will try to connect to our server... + IFOK(s, natsThread_Create(&t, _connectToMockupServer, (void*) &arg)); + + if ((s == NATS_OK) + && (((ctx.fd = accept(sock, NULL, NULL)) == NATS_SOCK_INVALID) + || natsSock_SetCommonTcpOptions(ctx.fd) != NATS_OK)) + { + s = NATS_SYS_ERROR; + } + + testCond((s == NATS_OK) && (ctx.fd > 0)); + + test("Read ClientHello from client: "); + char buffer[1024]; + memset(buffer, 0, sizeof(buffer)); + + int size = recv(ctx.fd, buffer, sizeof(buffer), 0); + testCond(size > 0); + + // remove all null chars to allow the use of strstr on the result + for (int i = 0; i < size; ++i) { + if (buffer[i] == 0) + buffer[i] = '0'; + } + + test("Check hostname is found in ClientHello: "); + bool found = strstr(buffer, "localhost"); +#if defined(NATS_USE_OPENSSL_1_1) + testCond(found == true); +#else + testCond(found == false); +#endif + + // Need to close those for the client side to unblock. + natsSock_Close(ctx.fd); + natsSock_Close(sock); + + // Wait for the client to finish. + if (t != NULL) + { + natsThread_Join(t); + natsThread_Destroy(t); + } + + _destroyDefaultThreadArgs(&arg); + +#else + test("Skipped when built with no SSL support: "); + testCond(true); +#endif +} + #if defined(NATS_HAS_TLS) static natsStatus _elDummyAttach(void **userData, void *loop, natsConnection *nc, natsSock socket) { return NATS_OK; } From e7d767eca0a65cf1d1af0c917b8b1cc4a53f018c Mon Sep 17 00:00:00 2001 From: Thierry Bastian Date: Mon, 12 Aug 2024 18:48:10 +0200 Subject: [PATCH 11/13] Fixing setting handshake_first without setting secure in the natsOptions (#789) * Fixing setting handshake_first without setting secure in the natsOptions some tests seem unstable... * code review + rebuild * more code review unstable tests... --------- Co-authored-by: Lev <1187448+levb@users.noreply.github.com> --- src/opts.c | 7 +++++-- test/test.c | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/opts.c b/src/opts.c index da72a3d99..32776afb8 100644 --- a/src/opts.c +++ b/src/opts.c @@ -371,8 +371,11 @@ natsOptions_TLSHandshakeFirst(natsOptions *opts) LOCK_AND_CHECK_OPTIONS(opts, 0); - opts->tlsHandshakeFirst = true; - opts->secure = true; + s = natsOptions_SetSecure(opts, true); + if (s == NATS_OK) + { + opts->tlsHandshakeFirst = true; + } UNLOCK_OPTS(opts); diff --git a/test/test.c b/test/test.c index 5f2965a61..021c983a0 100644 --- a/test/test.c +++ b/test/test.c @@ -21324,6 +21324,22 @@ void test_SSLHandshakeFirst(void) s = natsOptions_TLSHandshakeFirst(opts); testCond(s == NATS_OK); + test("Set TLSHandshakeFirst option without setting secure: "); + { + // we start with a new natsOptions struct so that we can test + // that it does not crash with a minimal config + natsOptions *no_secure_opts = NULL; + s = natsOptions_Create(&no_secure_opts); + IFOK(s, natsOptions_SetURL(no_secure_opts, "nats://127.0.0.1:4443")); + IFOK(s, natsOptions_SetTimeout(no_secure_opts, 500)); + IFOK(s, natsOptions_TLSHandshakeFirst(no_secure_opts)); + IFOK(s, natsConnection_Connect(&nc, no_secure_opts)); + // expecting an error because cert valiation will fail; the goal here is to avoid a crash + testCond(s == NATS_SSL_ERROR); + natsOptions_Destroy(no_secure_opts); + nats_clearLastError(); + } + test("Check that connect succeeds: "); s = natsConnection_Connect(&nc, opts); testCond(s == NATS_OK); From 4f19da67f801c61560c652bb84680ba021e042d7 Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Tue, 13 Aug 2024 10:34:08 -0600 Subject: [PATCH 12/13] Fixed test SSLVerifyHostName when build NATS_FORCE_HOST_VERIFICATION=OFF (#788) * Fixed test SSLVerifyHostName when build NATS_FORCE_HOST_VERIFICATION=OFF This test would fail unless we force host verification, so adapt test to take into consideration the expected result based on the build environment variable. Signed-off-by: Ivan Kozlovic * Added a GHA job for NATS_FORCE_HOST_VERIFICATION * fixed default: ON * job: no-host-sanitize * job names * [CI only] Streamlined/fixed sanitize/coverage matrices (#790) --------- Signed-off-by: Ivan Kozlovic Co-authored-by: Lev Brouk Co-authored-by: Lev <1187448+levb@users.noreply.github.com> --- .github/workflows/build-test.yml | 32 ++++++++----- .github/workflows/on-pr-debug.yml | 74 +++++++++++++++++++------------ test/test.c | 13 +++++- 3 files changed, 79 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e4eda5c70..cf96b75e6 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,12 +13,18 @@ on: dev_mode: type: string default: "OFF" - lib_msg_delivery: + pool_dispatch: type: string - default: "OFF" - lib_write_deadline: + default: "NO-pool" + write_deadline: type: string - default: "OFF" + default: "NO-write_deadline" + tls: + type: string + default: "TLS" + verify_host: + type: string + default: "verify_host" repeat: type: string default: "1" @@ -32,9 +38,6 @@ on: streaming: type: string default: "ON" - tls: - type: string - default: "ON" type: type: string description: "Debug or Release." @@ -85,11 +88,20 @@ jobs: flags: -DNATS_BUILD_ARCH=${{ inputs.arch }} -DCMAKE_BUILD_TYPE=${{ inputs.type }} -DNATS_BUILD_STREAMING=${{ inputs.streaming }} - -DNATS_BUILD_WITH_TLS=${{ inputs.tls }} -DNATS_PROTOBUF_DIR=${{ github.workspace}}/deps/pbuf -DNATS_BUILD_USE_SODIUM=ON -DNATS_SODIUM_DIR=${{ github.workspace}}/deps/sodium run: | + if [[ "${{ inputs.tls }}" == "TLS" ]]; then + flags="$flags -DNATS_BUILD_WITH_TLS=ON" + if [[ "${{ inputs.verify_host }}" == "verify_host" ]]; then + flags="$flags -DNATS_BUILD_TLS_FORCE_HOST_VERIFY=ON" + else + flags="$flags -DNATS_BUILD_TLS_FORCE_HOST_VERIFY=OFF" + fi + else + flags="$flags -DNATS_BUILD_WITH_TLS=OFF" + fi if [[ -n "${{ inputs.sanitize }}" ]]; then flags="$flags -DNATS_SANITIZE=ON -DCMAKE_C_FLAGS=-fsanitize=${{ inputs.sanitize }}" fi @@ -110,10 +122,10 @@ jobs: if [[ -n "${{ inputs.sanitize }}" ]]; then echo "NATS_TEST_VALGRIND=yes" >> $GITHUB_ENV fi - if [[ "${{ inputs.lib_msg_delivery }}" == "ON" ]]; then + if [[ "${{ inputs.pool_dispatch }}" == "pool" ]]; then echo "NATS_DEFAULT_TO_LIB_MSG_DELIVERY=yes" >> $GITHUB_ENV fi - if [[ "${{ inputs.lib_write_deadline }}" == "ON" ]]; then + if [[ "${{ inputs.write_deadline }}" == "write_deadline" ]]; then echo "NATS_DEFAULT_LIB_WRITE_DEADLINE=2000" >> $GITHUB_ENV fi echo "CC=${{ inputs.compiler }}" >> $GITHUB_ENV diff --git a/.github/workflows/on-pr-debug.yml b/.github/workflows/on-pr-debug.yml index 6ec95f12f..fe1a1d0e7 100644 --- a/.github/workflows/on-pr-debug.yml +++ b/.github/workflows/on-pr-debug.yml @@ -18,27 +18,6 @@ jobs: server_version: main type: Debug - coverage: - name: "Coverage" - uses: ./.github/workflows/build-test.yml - with: - coverage: ON - server_version: main - type: Debug - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - coverage-pooled: - name: "Coverage (pooled delivery)" - uses: ./.github/workflows/build-test.yml - with: - coverage: ON - server_version: main - type: Debug - lib_msg_delivery: ON - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - dev-mode: name: "DEV_MODE" uses: ./.github/workflows/build-test.yml @@ -56,24 +35,61 @@ jobs: matrix: compiler: [gcc, clang] sanitize: [address, thread] - pooled_delivery: [ON, OFF] + pooled_dispatch: [pool, NO-pool] uses: ./.github/workflows/build-test.yml with: server_version: main - type: Debug + type: RelWithDebInfo compiler: ${{ matrix.compiler }} sanitize: ${{ matrix.sanitize }} - lib_msg_delivery: ${{ matrix.pooled_delivery }} + pool_dispatch: ${{ matrix.pooled_dispatch }} - san-addr-deadline: - name: "Sanitize address (lib_write_deadline)" + coverage-TLS: + name: "Coverage: TLS" + strategy: + fail-fast: false + matrix: + pooled_dispatch: [pool, NO-pool] + write_deadline: [write_deadline, NO-write_deadline] uses: ./.github/workflows/build-test.yml with: - type: Debug - sanitize: address + coverage: ON + type: RelWithDebInfo server_version: main - lib_write_deadline: ON + compiler: gcc + tls: TLS + verify_host: verify_host + pool_dispatch: ${{ matrix.pooled_dispatch }} + write_deadline: ${{ matrix.write_deadline }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + coverage-NO-verify_host: + name: "Coverage: NO-verify_host" + uses: ./.github/workflows/build-test.yml + with: + coverage: ON + type: RelWithDebInfo + server_version: main + compiler: gcc + tls: TLS + verify_host: NO-verify_host + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + coverage-NO-TLS: + name: "Coverage NO-TLS" + uses: ./.github/workflows/build-test.yml + with: + coverage: ON + type: RelWithDebInfo + server_version: main + compiler: gcc + tls: NO-TLS + verify_host: NO-verify_host + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + bench: name: "Benchmark" uses: ./.github/workflows/build-test.yml diff --git a/test/test.c b/test/test.c index 021c983a0..5c46669a4 100644 --- a/test/test.c +++ b/test/test.c @@ -20895,20 +20895,31 @@ void test_SSLVerifyHostname(void) serverPid = _startServer("nats://127.0.0.1:4443", "-config tls_noip.conf", true); CHECK_SERVER_STARTED(serverPid); +#if defined(NATS_FORCE_HOST_VERIFICATION) test("Check that connect fails if url is IP: "); +#else + test("Check that connect succeeds if url is IP: "); +#endif s = natsOptions_SetURL(opts, "nats://127.0.0.1:4443"); IFOK(s, natsOptions_SetSecure(opts, true)); - // For test purposes, we provide the CA trusted certs IFOK(s, natsOptions_LoadCATrustedCertificates(opts, "certs/ca.pem")); IFOK(s, natsOptions_SetReconnectedCB(opts, _reconnectedCb, &args)); IFOK(s, natsConnection_Connect(&nc, opts)); +#if defined(NATS_FORCE_HOST_VERIFICATION) testCond(s == NATS_SSL_ERROR); + nats_clearLastError(); +#else + testCond(s == NATS_OK); + natsConnection_Destroy(nc); + nc = NULL; +#endif test("Check that connect fails if wrong expected hostname: "); s = natsOptions_SetURL(opts, "nats://localhost:4443"); IFOK(s, natsOptions_SetExpectedHostname(opts, "foo")); IFOK(s, natsConnection_Connect(&nc, opts)); testCond(s == NATS_SSL_ERROR); + nats_clearLastError(); test("Check that connect succeeds if hostname ok and no expected hostname set: "); s = natsOptions_SetURL(opts, "nats://localhost:4443"); From 381f01cb75acb6ff0860a599ff29b17127d21dac Mon Sep 17 00:00:00 2001 From: mathi-m <36664203+mathi-m@users.noreply.github.com> Date: Fri, 16 Aug 2024 19:23:59 +0200 Subject: [PATCH 13/13] nats.c is available on conan.io/center (#791) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fa081026d..13d7bff45 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ This NATS Client implementation is heavily based on the [NATS GO Client](https:/ There are several package managers with NATS C client library available. If you know one that is not in this list, please submit a PR to add it! +- [conan](https://conan.io/center) The "cnats" recipe is [here](https://github.com/conan-io/conan-center-index/tree/master/recipes/cnats) - [Homebrew](https://github.com/Homebrew/homebrew-core) The "cnats" formula is [here](https://github.com/Homebrew/homebrew-core/blob/master/Formula/c/cnats.rb) - [vcpkg](https://vcpkg.io) The "cnats" port is [here](https://github.com/microsoft/vcpkg/tree/master/ports/cnats)