diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9a1bf957c..cf96b75e6 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,12 +10,21 @@ on: coverage: type: string default: "OFF" - lib_msg_delivery: + dev_mode: type: string default: "OFF" - lib_write_deadline: + pool_dispatch: type: string - default: "OFF" + default: "NO-pool" + write_deadline: + type: string + default: "NO-write_deadline" + tls: + type: string + default: "TLS" + verify_host: + type: string + default: "verify_host" repeat: type: string default: "1" @@ -29,9 +38,6 @@ on: streaming: type: string default: "ON" - tls: - type: string - default: "ON" type: type: string description: "Debug or Release." @@ -40,6 +46,16 @@ on: type: string description: "Ubuntu version to use, e.g. '20.04'" default: "latest" + verbose_test_output: + type: string + default: "OFF" + verbose_make_output: + type: string + default: "OFF" + benchmark: + type: string + default: "OFF" + secrets: CODECOV_TOKEN: description: "Codecov repo token" @@ -72,17 +88,32 @@ jobs: flags: -DNATS_BUILD_ARCH=${{ inputs.arch }} -DCMAKE_BUILD_TYPE=${{ inputs.type }} -DNATS_BUILD_STREAMING=${{ inputs.streaming }} - -DNATS_BUILD_WITH_TLS=${{ inputs.tls }} -DNATS_PROTOBUF_DIR=${{ github.workspace}}/deps/pbuf -DNATS_BUILD_USE_SODIUM=ON -DNATS_SODIUM_DIR=${{ github.workspace}}/deps/sodium run: | + if [[ "${{ inputs.tls }}" == "TLS" ]]; then + flags="$flags -DNATS_BUILD_WITH_TLS=ON" + if [[ "${{ inputs.verify_host }}" == "verify_host" ]]; then + flags="$flags -DNATS_BUILD_TLS_FORCE_HOST_VERIFY=ON" + else + flags="$flags -DNATS_BUILD_TLS_FORCE_HOST_VERIFY=OFF" + fi + else + flags="$flags -DNATS_BUILD_WITH_TLS=OFF" + fi if [[ -n "${{ inputs.sanitize }}" ]]; then flags="$flags -DNATS_SANITIZE=ON -DCMAKE_C_FLAGS=-fsanitize=${{ inputs.sanitize }}" fi if [[ "${{ inputs.coverage }}" == "ON" ]]; then flags="$flags -DNATS_COVERAGE=ON" fi + if [[ "${{ inputs.dev_mode }}" == "ON" ]]; then + flags="$flags -DDEV_MODE=ON" + fi + if [[ "${{ inputs.verbose_make_output }}" == "ON" ]]; then + flags="$flags -DCMAKE_VERBOSE_MAKEFILE=ON" + fi echo "flags=$flags" >> $GITHUB_OUTPUT - id: nats-vars @@ -91,10 +122,10 @@ jobs: if [[ -n "${{ inputs.sanitize }}" ]]; then echo "NATS_TEST_VALGRIND=yes" >> $GITHUB_ENV fi - if [[ "${{ inputs.lib_msg_delivery }}" == "ON" ]]; then + if [[ "${{ inputs.pool_dispatch }}" == "pool" ]]; then echo "NATS_DEFAULT_TO_LIB_MSG_DELIVERY=yes" >> $GITHUB_ENV fi - if [[ "${{ inputs.lib_write_deadline }}" == "ON" ]]; then + if [[ "${{ inputs.write_deadline }}" == "write_deadline" ]]; then echo "NATS_DEFAULT_LIB_WRITE_DEADLINE=2000" >> $GITHUB_ENV fi echo "CC=${{ inputs.compiler }}" >> $GITHUB_ENV @@ -114,15 +145,6 @@ jobs: cmake .. ${{ steps.cmake-flags.outputs.flags }} make rebuild_cache && make - - name: "Rebuild the list of tests to match the compile flags" - working-directory: ./build - run: | - ./bin/testsuite - if [[ $(diff list.txt ../test/list.txt; echo $?) != 0 ]]; then - mv list.txt ../test/list.txt - make rebuild_cache - fi - # testing - name: "Download nats-server version ${{ inputs.server_version }}" @@ -153,21 +175,69 @@ jobs: fi - name: "Test" + if: inputs.benchmark == 'OFF' working-directory: ./build run: | export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH export NATS_TEST_SERVER_VERSION="$(nats-server -v)" flags="" - ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} 2>&1 | tee /tmp/out.txt - if [[ $(grep -q 'ThreadSanitizer: ' /tmp/out.txt; echo $?) == 0 ]]; then - echo "!!! ThreadSanitizer detected WARNING(s) !!!" - exit 1 - fi + ctest -L 'test' --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }} - name: Upload coverage reports to Codecov - if: inputs.coverage == 'ON' + # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301 + # Only upload coverage reports for PRs from the same repo (not forks) + if: inputs.coverage == 'ON' && github.event.pull_request.head.repo.full_name == github.repository uses: codecov/codecov-action@v4 with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} - # verbose: true \ No newline at end of file + # verbose: true + + - name: "Cache for base benchmark for ${{ github.event.pull_request.base.ref }}" + id: cache-base-bench + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref + uses: actions/cache@v4 + with: + key: bench-${{ github.event.pull_request.base.ref }} + path: ./build/bench-${{ github.event.pull_request.base.ref }}.log + + - name: "Benchmark" + if: inputs.benchmark == 'ON' + working-directory: ./build + run: | + export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH + export NATS_TEST_SERVER_VERSION="$(nats-server -v)" + flags="" + ctest -L 'bench' --timeout 600 -VV | tee bench.log + + - name: "Checkout nats.c for ${{ github.event.pull_request.base.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref && steps.cache-base-bench.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + clean: false + + - name: "Benchmark ${{ github.event.pull_request.base.ref }} for comparison" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref && steps.cache-base-bench.outputs.cache-hit != 'true' + run: | + mkdir -p build + cd build + rm -rf CMakeFiles CMakeCache.txt + cmake .. ${{ steps.cmake-flags.outputs.flags }} + make rebuild_cache && make + export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH + export NATS_TEST_SERVER_VERSION="$(nats-server -v)" + flags="" + ctest -L 'bench' --timeout 600 -VV | tee bench-${{ github.event.pull_request.base.ref }}.log + + - name: "Checkout HEAD ${{ github.event.pull_request.head.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref && steps.cache-base-bench.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + clean: false + + - name: "Compare benchmark to ${{ github.event.pull_request.base.ref }}" + if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref + run: | + cd build + go run ../test/diffstat_sub_async.go bench-${{ github.event.pull_request.base.ref }}.log bench.log diff --git a/.github/workflows/on-pr-debug.yml b/.github/workflows/on-pr-debug.yml index 660cccdfe..fe1a1d0e7 100644 --- a/.github/workflows/on-pr-debug.yml +++ b/.github/workflows/on-pr-debug.yml @@ -1,4 +1,4 @@ -name: "Debug" +name: "PR" on: pull_request: @@ -18,46 +18,85 @@ jobs: server_version: main type: Debug - coverage: - name: "Coverage" + dev-mode: + name: "DEV_MODE" uses: ./.github/workflows/build-test.yml with: - coverage: ON + dev_mode: ON server_version: main type: Debug - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + verbose_test_output: ON + verbose_make_output: ON - sanitize-addr: - name: "Sanitize address" + sanitize: + name: "Sanitize" + strategy: + fail-fast: false + matrix: + compiler: [gcc, clang] + sanitize: [address, thread] + pooled_dispatch: [pool, NO-pool] uses: ./.github/workflows/build-test.yml with: - sanitize: address server_version: main - type: Debug + type: RelWithDebInfo + compiler: ${{ matrix.compiler }} + sanitize: ${{ matrix.sanitize }} + pool_dispatch: ${{ matrix.pooled_dispatch }} - sanitize-addr-lib-msg-delivery: - name: "Sanitize address (lib_msg_delivery)" + coverage-TLS: + name: "Coverage: TLS" + strategy: + fail-fast: false + matrix: + pooled_dispatch: [pool, NO-pool] + write_deadline: [write_deadline, NO-write_deadline] uses: ./.github/workflows/build-test.yml with: - sanitize: address + coverage: ON + type: RelWithDebInfo server_version: main - lib_msg_delivery: ON + compiler: gcc + tls: TLS + verify_host: verify_host + pool_dispatch: ${{ matrix.pooled_dispatch }} + write_deadline: ${{ matrix.write_deadline }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - san-addr: - name: "Sanitize address (lib_write_deadline)" + coverage-NO-verify_host: + name: "Coverage: NO-verify_host" uses: ./.github/workflows/build-test.yml with: - sanitize: address + coverage: ON + type: RelWithDebInfo server_version: main - lib_write_deadline: ON - - san-thread: - name: "Sanitize thread" + compiler: gcc + tls: TLS + verify_host: NO-verify_host + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + coverage-NO-TLS: + name: "Coverage NO-TLS" uses: ./.github/workflows/build-test.yml with: - sanitize: thread + coverage: ON + type: RelWithDebInfo server_version: main + compiler: gcc + tls: NO-TLS + verify_host: NO-verify_host + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + bench: + name: "Benchmark" + uses: ./.github/workflows/build-test.yml + with: + server_version: main + benchmark: ON + type: Release Windows: name: "Windows" @@ -77,21 +116,13 @@ jobs: env: VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" run: | - cmake -B build -S . -DCMAKE_C_FLAGS=/W4 - cmake --build build --config Debug + cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -DNATS_BUILD_STREAMING=OFF + cmake --build build - name: Test shell: bash run: | cd build - - ./bin/Debug/testsuite - mv list.txt ../test/list.txt - - # refresh the cache - cmake -S .. -B . - cmake --build . --config Debug - # Download latest nats-server rel="latest" # TODO: parameterize if [ "$rel" = "latest" ]; then @@ -117,43 +148,6 @@ jobs: mkdir -p ../deps/nats-server mv ./nats-server.exe ../deps/nats-server/nats-server.exe fi - export PATH=../deps/nats-server:$PATH export NATS_TEST_SERVER_VERSION="$(nats-server -v)" - - run_ctest() { - local extra_args="$@" - local current_set=$(set +o) - set +e - echo "Running ctest($extra_args)..." - - ctest -C Debug --timeout 60 --output-on-failure $extra_args - - local result=$? - eval "$current_set" - return $result - } - - run_ctest - result=$? - - # Retry if failed - MAX_RETRIES=10 - retries=0 - while [ $result -ne 0 ] && [ $retries -lt $MAX_RETRIES ]; do - tasklist | grep nats-server.exe | awk '{print $2}' | xargs -I {} taskkill /PID {} /F /T - sleep 3 - echo "Tests failed. Retrying failed tests... ($((retries + 1))/$MAX_RETRIES)" - retries=$((retries + 1)) - run_ctest --rerun-failed - result=$? - done - - # Check the final result - if [ $result -ne 0 ]; then - echo "Tests failed after $MAX_RETRIES retries." - exit 1 - else - echo "All tests passed. (retried $retries times)" - exit 0 - fi + ctest -L test -C Debug --timeout 60 --output-on-failure --repeat until-pass:3 diff --git a/.github/workflows/on-push-release-extra.yml b/.github/workflows/on-push-release-extra.yml index f11aaf4fd..487f5d362 100644 --- a/.github/workflows/on-push-release-extra.yml +++ b/.github/workflows/on-push-release-extra.yml @@ -5,7 +5,6 @@ on: - main - release_* - permissions: contents: write # required by build-test to comment on coverage but not used here. @@ -67,3 +66,11 @@ jobs: type: Debug secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + bench: + name: "Benchmark" + uses: ./.github/workflows/build-test.yml + with: + server_version: main + benchmark: ON + type: Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 7df3cc81f..f683a33fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,9 @@ include(FindPackageHandleStandardArgs) # Uncomment to have the build process verbose # set(CMAKE_VERBOSE_MAKEFILE TRUE) +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) + # Set output directories for libraries and executables. # This is important for Windows builds to have the DLLs in the same directory as the executables. set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) @@ -206,9 +209,17 @@ if(UNIX) elseif(WIN32) set(NATS_LIBDIR "lib") set(NATS_INCLUDE_DIR "include") - set(NATS_EXTRA_LIB "Ws2_32") + set(NATS_EXTRA_LIB "ws2_32") set(NATS_OS "_WIN32") set(NATS_PLATFORM_INCLUDE "win") + + # Warning control. + add_compile_options(/W4) # Set warning level to maximum, then disable: + add_compile_options(/wd4100) # unreferenced formal parameter + add_compile_options(/wd4200) # nonstandard extension used: zero-sized array in struct/union + add_compile_options(/wd4130) # logical operation on address of string constant + add_compile_options(/wd4127) # conditional expression is constant + if(sodium_USE_STATIC_LIBS) add_definitions( -DSODIUM_STATIC diff --git a/README.md b/README.md index 83badc0c3..13d7bff45 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ This NATS Client implementation is heavily based on the [NATS GO Client](https:/ There are several package managers with NATS C client library available. If you know one that is not in this list, please submit a PR to add it! +- [conan](https://conan.io/center) The "cnats" recipe is [here](https://github.com/conan-io/conan-center-index/tree/master/recipes/cnats) - [Homebrew](https://github.com/Homebrew/homebrew-core) The "cnats" formula is [here](https://github.com/Homebrew/homebrew-core/blob/master/Formula/c/cnats.rb) - [vcpkg](https://vcpkg.io) The "cnats" port is [here](https://github.com/microsoft/vcpkg/tree/master/ports/cnats) @@ -241,38 +242,15 @@ ctest -T memcheck -V -I 1,4 ``` The above command would run the tests with `valgrind` (`-T memcheck`), with verbose output (`-V`), and run the tests from 1 to 4 (`-I 1,4`). -If you add a test to `test/test.c`, you need to add it into the `allTests` array. Each entry contains a name, and the test function. You can add it anywhere into this array. -Build you changes: +If you add a test to `test/test.c`, you need to add it into the `list_test.txt` +file. Each entry contains just the test name, the function must be named +identically, with a `test_` prefix. The list is in alphabetical order, but it +does not need to be, you can add anywhere. -``` -make -[ 44%] Built target nats -[ 88%] Built target nats_static -[ 90%] Built target nats-publisher -[ 92%] Built target nats-queuegroup -[ 94%] Built target nats-replier -[ 96%] Built target nats-requestor -[ 98%] Built target nats-subscriber -Scanning dependencies of target testsuite -[100%] Building C object test/CMakeFiles/testsuite.dir/test.c.o -Linking C executable testsuite -[100%] Built target testsuite -``` - -Now regenerate the list by invoking the test suite without any argument: - -``` -./test/testsuite -Number of tests: 77 -``` - -This list the number of tests added to the file `list.txt`. Move this file to the source's test directory. - -``` -mv list.txt ../test/ -``` +If you are adding a benchmark, it should be added to the `list_bench.txt`. These +tests are labeled differently (`-L 'bench'`) and executed separately on CI. -Then, refresh the build: +You need to re-run `cmake` for the changes to take effect: ``` cmake .. diff --git a/buildOnTravis.sh b/buildOnTravis.sh index 3a298345d..2ad12d1c9 100755 --- a/buildOnTravis.sh +++ b/buildOnTravis.sh @@ -75,7 +75,7 @@ fi export NATS_TEST_TRAVIS=yes echo "Using NATS server version: $NATS_TEST_SERVER_VERSION" -ctest --timeout 60 --output-on-failure $4 +ctest -L 'test' --timeout 60 --output-on-failure $4 res=$? if [ $res -ne 0 ]; then exit $res diff --git a/examples/examples.h b/examples/examples.h index 643e0e48f..ba30d38ff 100644 --- a/examples/examples.h +++ b/examples/examples.h @@ -45,7 +45,7 @@ volatile int64_t elapsed = 0; bool print = false; int64_t timeout = 10000; // 10 seconds. -natsOptions *opts = NULL; +natsOptions *gOpts = NULL; const char *certFile = NULL; const char *keyFile = NULL; @@ -154,7 +154,7 @@ printUsageAndExit(const char *progName, const char *usage) "%s\n", progName, usage); - natsOptions_Destroy(opts); + natsOptions_Destroy(gOpts); nats_Close(); exit(1); @@ -214,7 +214,7 @@ parseArgs(int argc, char **argv, const char *usage) bool urlsSet = false; int i; - if (natsOptions_Create(&opts) != NATS_OK) + if (natsOptions_Create(&gOpts) != NATS_OK) s = NATS_NO_MEMORY; for (i=1; (idata) #define natsBuf_Capacity(b) ((b)->capacity) #define natsBuf_Len(b) ((b)->len) diff --git a/src/conn.c b/src/conn.c index 1b405c852..37fcc1790 100644 --- a/src/conn.c +++ b/src/conn.c @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -35,6 +35,7 @@ #include "nkeys.h" #include "crypto.h" #include "js.h" +#include "glib/glib.h" #define DEFAULT_SCRATCH_SIZE (512) #define MAX_INFO_MESSAGE_SIZE (32768) @@ -736,6 +737,13 @@ _makeTLSConn(natsConnection *nc) SSL_set_verify(ssl, SSL_VERIFY_PEER, _collectSSLErr); } } +#if defined(NATS_USE_OPENSSL_1_1) + // add the host name in the SNI extension + if ((s == NATS_OK) && (nc->cur != NULL) && (!SSL_set_tlsext_host_name(ssl, nc->cur->url->host))) + { + s = nats_setError(NATS_SSL_ERROR, "unable to set SNI extension for hostname '%s'", nc->cur->url->host); + } +#endif if ((s == NATS_OK) && (SSL_do_handshake(ssl) != 1)) { s = nats_setError(NATS_SSL_ERROR, @@ -786,7 +794,12 @@ _checkForSecure(natsConnection *nc) } if ((s == NATS_OK) && nc->opts->secure) - s = _makeTLSConn(nc); + { + // If TLS handshake first is true, we have already done + // the handshake, so do it only if false. + if (!nc->opts->tlsHandshakeFirst) + s = _makeTLSConn(nc); + } return NATS_UPDATE_ERR_STACK(s); } @@ -1136,19 +1149,19 @@ _resendSubscriptions(natsConnection *nc) sub = subs[i]; adjustedMax = 0; - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); // If JS ordered consumer, trigger a reset. Don't check the error // condition here. If there is a failure, it will be retried // at the next HB interval. if ((sub->jsi != NULL) && (sub->jsi->ordered)) { jsSub_resetOrderedConsumer(sub, sub->jsi->sseq+1); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); continue; } if (natsSub_drainStarted(sub)) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); continue; } if (sub->max > 0) @@ -1160,7 +1173,7 @@ _resendSubscriptions(natsConnection *nc) // messages have reached the max, if so, unsubscribe. if (adjustedMax == 0) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); s = natsConn_sendUnsubProto(nc, sub->sid, 0); continue; } @@ -1172,7 +1185,7 @@ _resendSubscriptions(natsConnection *nc) // Hold the lock up to that point so we are sure not to resend // any SUB/UNSUB for a subscription that is in draining mode. - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } NATS_FREE(subs); @@ -1948,8 +1961,14 @@ _processConnInit(natsConnection *nc) nc->status = NATS_CONN_STATUS_CONNECTING; + // If we need to have a TLS connection and want the TLS handshake to occur + // first, do it now. + if (nc->opts->secure && nc->opts->tlsHandshakeFirst) + s = _makeTLSConn(nc); + // Process the INFO protocol that we should be receiving - s = _processExpectedInfo(nc); + if (s == NATS_OK) + s = _processExpectedInfo(nc); // Send the CONNECT and PING protocol, and wait for the PONG. if (s == NATS_OK) @@ -2647,11 +2666,8 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) natsStatus s = NATS_OK; natsSubscription *sub = NULL; natsMsg *msg = NULL; - natsMsgDlvWorker *ldw = NULL; bool sc = false; bool sm = false; - nats_MsgList *list = NULL; - natsCondition *cond = NULL; // For JetStream cases jsSub *jsi = NULL; bool ctrlMsg = false; @@ -2697,34 +2713,17 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) // We need to retain the subscription since as soon as we release the // nc->subsMu lock, the subscription could be destroyed and we would // reference freed memory. - natsSubAndLdw_LockAndRetain(sub); + nats_lockRetainSubAndDispatcher(sub); natsMutex_Unlock(nc->subsMu); if (sub->closed || sub->drainSkip) { - natsSubAndLdw_UnlockAndRelease(sub); + nats_unlockReleaseSubAndDispatcher(sub); natsMsg_Destroy(msg); return NATS_OK; } - // Pick condition variable and list based on if the sub is - // part of a global delivery thread pool or not. - // Note about `list`: this is used only to link messages, but - // sub->msgList needs to be used to update/check number of pending - // messages, since in case of delivery thread pool, `list` will have - // messages from many different subscriptions. - if ((ldw = sub->libDlvWorker) != NULL) - { - cond = ldw->cond; - list = &(ldw->msgList); - } - else - { - cond = sub->cond; - list = &(sub->msgList); - } - jsi = sub->jsi; // For JS subscriptions (but not pull ones), handle hearbeat and flow control here. if (jsi && !jsi->pull) @@ -2744,7 +2743,7 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) s = jsSub_checkOrderedMsg(sub, msg, &replaced); if ((s != NATS_OK) || replaced) { - natsSubAndLdw_UnlockAndRelease(sub); + nats_unlockReleaseSubAndDispatcher(sub); natsMsg_Destroy(msg); return s; } @@ -2753,58 +2752,27 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) if (!ctrlMsg) { - sub->msgList.msgs++; - sub->msgList.bytes += bufLen; - - if (((sub->msgsLimit > 0) && (sub->msgList.msgs > sub->msgsLimit)) - || ((sub->bytesLimit > 0) && (sub->msgList.bytes > sub->bytesLimit))) - { - natsMsg_Destroy(msg); - - sub->dropped++; - - sc = !sub->slowConsumer; - sub->slowConsumer = true; - - // Undo stats from above. - sub->msgList.msgs--; - sub->msgList.bytes -= bufLen; - } - else + s = natsSub_enqueueUserMessage(sub, msg); + if (s == NATS_OK) { - bool signal= false; - - if ((jsi != NULL) && jsi->ackNone) - natsMsg_setAcked(msg); - - if (sub->msgList.msgs > sub->msgsMax) - sub->msgsMax = sub->msgList.msgs; - - if (sub->msgList.bytes > sub->bytesMax) - sub->bytesMax = sub->msgList.bytes; - sub->slowConsumer = false; - msg->sub = sub; - - if (list->head == NULL) - { - list->head = msg; - signal = true; - } - else - list->tail->next = msg; - - list->tail = msg; - - if (signal) - natsCondition_Signal(cond); - // Store the ACK metadata from the message to // compare later on with the received heartbeat. if (jsi != NULL) s = jsSub_trackSequences(jsi, msg->reply); } + else + { + // Slow consumer is the only reason to fail here. Handle it and + // reset the status, so we continue. + natsMsg_Destroy(msg); + sub->dropped++; + sc = !sub->slowConsumer; + sub->slowConsumer = true; + + s = NATS_OK; + } } else if ((jct == jsCtrlHeartbeat) && (msg->reply == NULL)) { @@ -2828,9 +2796,9 @@ natsConn_processMsg(natsConnection *nc, char *buf, int bufLen) // If we are going to post to the error handler, do not release yet. if (sc || sm) - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); else - natsSubAndLdw_UnlockAndRelease(sub); + nats_unlockReleaseSubAndDispatcher(sub); if ((s == NATS_OK) && fcReply) s = natsConnection_Publish(nc, fcReply, NULL, 0); @@ -3260,6 +3228,11 @@ natsConn_create(natsConnection **newConn, natsOptions *options) nc->sockCtx.fd = NATS_SOCK_INVALID; nc->opts = options; + // If the TLSHandshakeFirst option is specified, make sure that + // the Secure boolean is true. + if (nc->opts->tlsHandshakeFirst) + nc->opts->secure = true; + nc->errStr[0] = '\0'; s = natsMutex_Create(&(nc->mu)); @@ -3435,7 +3408,7 @@ natsConnection_Reconnect(natsConnection *nc) return nats_setDefaultError(NATS_CONNECTION_CLOSED); } - natsSock_Close(nc->sockCtx.fd); + natsSock_Shutdown(nc->sockCtx.fd); natsConn_Unlock(nc); return NATS_OK; diff --git a/src/crypto.c b/src/crypto.c index fdebf07e9..04391b220 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -61,7 +61,7 @@ typedef unsigned long long u64; typedef long long i64; typedef i64 gf[16]; -static const gf gf0; +static const gf gf0 = {0}; static const gf gf1 = {1}; static const gf D2 = {0xf159, 0x26b2, 0x9b94, 0xebd6, 0xb156, 0x8283, 0x149a, 0x00e0, 0xd130, 0xeef3, 0x80f2, 0x198e, 0xfce7, 0x56df, 0xd9dc, 0x2406}; static const gf X = {0xd51a, 0x8f25, 0x2d60, 0xc956, 0xa7b2, 0x9525, 0xc760, 0x692c, 0xdc5c, 0xfdd6, 0xe231, 0xc0a4, 0x53fe, 0xcd6e, 0x36d3, 0x2169}; diff --git a/src/dispatch.c b/src/dispatch.c new file mode 100644 index 000000000..7284ae5ef --- /dev/null +++ b/src/dispatch.c @@ -0,0 +1,84 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "natsp.h" + +#include +#include + +#include "mem.h" +#include "conn.h" +#include "sub.h" +#include "glib/glib.h" + +// sub and dispatcher locks must be held. +void natsSub_enqueueMessage(natsSubscription *sub, natsMsg *msg) +{ + bool signal = false; + natsDispatchQueue *q = &sub->dispatcher->queue; + + if (q->head == NULL) + { + signal = true; + msg->next = NULL; + q->head = msg; + } + else + { + q->tail->next = msg; + } + q->tail = msg; + q->msgs++; + q->bytes += natsMsg_dataAndHdrLen(msg); + + if (signal) + natsCondition_Signal(sub->dispatcher->cond); +} + +// sub and dispatcher locks must be held. +natsStatus natsSub_enqueueUserMessage(natsSubscription *sub, natsMsg *msg) +{ + natsDispatchQueue *toQ = &sub->dispatcher->queue; + natsDispatchQueue *statsQ = &sub->ownDispatcher.queue; + int newMsgs = statsQ->msgs + 1; + int newBytes = statsQ->bytes + natsMsg_dataAndHdrLen(msg); + + msg->sub = sub; + + if (((sub->msgsLimit > 0) && (newMsgs > sub->msgsLimit)) || + ((sub->bytesLimit > 0) && (newBytes > sub->bytesLimit))) + { + return NATS_SLOW_CONSUMER; + } + sub->slowConsumer = false; + + if (newMsgs > sub->msgsMax) + sub->msgsMax = newMsgs; + if (newBytes > sub->bytesMax) + sub->bytesMax = newBytes; + + if ((sub->jsi != NULL) && sub->jsi->ackNone) + natsMsg_setAcked(msg); + + // Update the subscription stats if separate, the queue stats will be + // updated below. + if (toQ != statsQ) + { + statsQ->msgs++; + statsQ->bytes += natsMsg_dataAndHdrLen(msg); + } + + natsSub_enqueueMessage(sub, msg); + return NATS_OK; +} + diff --git a/src/dispatch.h b/src/dispatch.h new file mode 100644 index 000000000..9662a5533 --- /dev/null +++ b/src/dispatch.h @@ -0,0 +1,55 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DISPATCH_H_ +#define DISPATCH_H_ + +typedef struct __natsDispatchQueue_s +{ + natsMsg *head; + natsMsg *tail; + int msgs; + int bytes; +} natsDispatchQueue; + +typedef struct __natsDispatcher_s +{ + // When created as a dedicated dispatcher for a subscription, we use the + // sub's mutex (for performance? TODO: benchmack), so there is special + // handling for mu in the code. + natsSubscription *dedicatedTo; + natsMutex *mu; + + natsThread *thread; + natsCondition *cond; + natsDispatchQueue queue; + + // Flags. + bool running; + bool shutdown; +} natsDispatcher; + +static inline void nats_destroyQueuedMessages(natsDispatchQueue *queue) +{ + natsMsg *next = NULL; + for (natsMsg *msg = queue->head; msg != NULL; msg = next) + { + next = msg->next; + natsMsg_Destroy(msg); + } +} + +void natsSub_enqueueMessage(natsSubscription *sub, natsMsg *msg); +natsStatus natsSub_enqueueUserMessage(natsSubscription *sub, natsMsg *msg); + +#endif /* DISPATCH_H_ */ diff --git a/src/glib/glib.c b/src/glib/glib.c new file mode 100644 index 000000000..4b8e4bc31 --- /dev/null +++ b/src/glib/glib.c @@ -0,0 +1,491 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +#include "../util.h" +#include "../crypto.h" + +int64_t gLockSpinCount = 2000; + +static natsInitOnceType gInitOnce = NATS_ONCE_STATIC_INIT; +static natsLib gLib; + +natsLib* nats_lib(void) +{ + return &gLib; +} + +static void _freeLib(void); +static void _finalCleanup(void); + +void natsLib_Retain(void) +{ + natsMutex_Lock(gLib.lock); + + gLib.refs++; + + natsMutex_Unlock(gLib.lock); +} + +void natsLib_Release(void) +{ + int refs = 0; + + natsMutex_Lock(gLib.lock); + + refs = --(gLib.refs); + + natsMutex_Unlock(gLib.lock); + + if (refs == 0) + _freeLib(); +} + +static void +_finalCleanup(void) +{ + if (gLib.sslInitialized) + { +#if defined(NATS_HAS_TLS) +#if !defined(NATS_USE_OPENSSL_1_1) + ERR_free_strings(); + EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); + ERR_remove_thread_state(0); +#endif + sk_SSL_COMP_free(SSL_COMP_get_compression_methods()); +#endif + natsThreadLocal_DestroyKey(gLib.sslTLKey); + } + + natsThreadLocal_DestroyKey(gLib.errTLKey); + natsThreadLocal_DestroyKey(gLib.natsThreadKey); + natsMutex_Destroy(gLib.lock); + gLib.lock = NULL; +} + +static void +natsLib_Destructor(void) +{ + int refs = 0; + + if (!(gLib.wasOpenedOnce)) + return; + + // Destroy thread locals for the current thread. + nats_ReleaseThreadMemory(); + + // Do the final cleanup if possible + natsMutex_Lock(gLib.lock); + refs = gLib.refs; + if (refs > 0) + { + // If some thread is still around when the process exits and has a + // reference to the library, then don't do the final cleanup now. + // If the process has not fully exited when the lib's last reference + // is decremented, the final cleanup will be executed from that thread. + gLib.finalCleanup = true; + } + natsMutex_Unlock(gLib.lock); + + if (refs != 0) + return; + + _finalCleanup(); +} + +static void +_freeLib(void) +{ + const unsigned int offset = (unsigned int)offsetof(natsLib, refs); + bool callFinalCleanup = false; + + nats_freeTimers(&gLib); + nats_freeAsyncCbs(&gLib); + nats_freeGC(&gLib); + + nats_freeDispatcherPool(&gLib.messageDispatchers); + nats_freeDispatcherPool(&gLib.replyDispatchers); + + natsNUID_free(); + natsMutex_Destroy(gLib.service_callback_mu); + natsHash_Destroy(gLib.all_services_to_callback); + + natsCondition_Destroy(gLib.cond); + + memset((void *)(offset + (char *)&gLib), 0, sizeof(natsLib) - offset); + + natsMutex_Lock(gLib.lock); + callFinalCleanup = gLib.finalCleanup; + if (gLib.closeCompleteCond != NULL) + { + if (gLib.closeCompleteSignal) + { + *gLib.closeCompleteBool = true; + natsCondition_Signal(gLib.closeCompleteCond); + } + gLib.closeCompleteCond = NULL; + gLib.closeCompleteBool = NULL; + gLib.closeCompleteSignal = false; + } + gLib.closed = false; + gLib.initialized = false; + gLib.finalCleanup = false; + natsMutex_Unlock(gLib.lock); + + if (callFinalCleanup) + _finalCleanup(); +} + +static void +_destroyErrTL(void *localStorage) +{ + natsTLError *err = (natsTLError *)localStorage; + NATS_FREE(err); +} + +static void +_doInitOnce(void) +{ + natsStatus s; + + memset(&gLib, 0, sizeof(natsLib)); + + s = natsMutex_Create(&(gLib.lock)); + if (s == NATS_OK) + s = natsThreadLocal_CreateKey(&(gLib.errTLKey), _destroyErrTL); + if (s == NATS_OK) + s = natsThreadLocal_CreateKey(&(gLib.natsThreadKey), NULL); + if (s != NATS_OK) + { + fprintf(stderr, "FATAL ERROR: Unable to initialize library!\n"); + fflush(stderr); + abort(); + } + + nats_initForOS(); + + // Setup a hook for when the process exits. + atexit(natsLib_Destructor); +} + + +static void +_libTearDown(void) +{ + nats_waitForDispatcherPoolShutdown(&gLib.messageDispatchers); + nats_waitForDispatcherPoolShutdown(&gLib.replyDispatchers); + + if (gLib.timers.thread != NULL) + natsThread_Join(gLib.timers.thread); + + if (gLib.asyncCbs.thread != NULL) + natsThread_Join(gLib.asyncCbs.thread); + + if (gLib.gc.thread != NULL) + natsThread_Join(gLib.gc.thread); + + natsLib_Release(); +} + +// environment variables will override the default options. +natsStatus +nats_openLib(natsClientConfig *config) +{ + natsStatus s = NATS_OK; + + natsClientConfig defaultConfig = { + .LockSpinCount = -1, + .ThreadPoolMax = 1, + }; + if (config == NULL) + config = &defaultConfig; + + if (!nats_InitOnce(&gInitOnce, _doInitOnce)) + return NATS_FAILED_TO_INITIALIZE; + + natsMutex_Lock(gLib.lock); + + if (gLib.closed || gLib.initialized || gLib.initializing) + { + if (gLib.closed) + s = NATS_FAILED_TO_INITIALIZE; + else if (gLib.initializing) + s = NATS_ILLEGAL_STATE; + + natsMutex_Unlock(gLib.lock); + return s; + } + + gLib.initializing = true; + gLib.initAborted = false; + +#if !defined(_WIN32) + signal(SIGPIPE, SIG_IGN); +#endif + + srand((unsigned int)nats_NowInNanoSeconds()); + + gLib.refs = 1; + + // If the caller specifies negative value, then we use the default + if (config->LockSpinCount >= 0) + gLockSpinCount = config->LockSpinCount; + + gLib.config = *config; + nats_Base32_Init(); + + s = natsCondition_Create(&(gLib.cond)); + + if (s == NATS_OK) + s = natsCrypto_Init(); + + if (s == NATS_OK) + s = natsMutex_Create(&(gLib.timers.lock)); + if (s == NATS_OK) + s = natsCondition_Create(&(gLib.timers.cond)); + if (s == NATS_OK) + { + s = natsThread_Create(&(gLib.timers.thread), nats_timerThreadf, &gLib); + if (s == NATS_OK) + gLib.refs++; + } + + if (s == NATS_OK) + s = natsMutex_Create(&(gLib.asyncCbs.lock)); + if (s == NATS_OK) + s = natsCondition_Create(&(gLib.asyncCbs.cond)); + if (s == NATS_OK) + { + s = natsThread_Create(&(gLib.asyncCbs.thread), nats_asyncCbsThreadf, &gLib); + if (s == NATS_OK) + gLib.refs++; + } + if (s == NATS_OK) + s = natsMutex_Create(&(gLib.gc.lock)); + if (s == NATS_OK) + s = natsCondition_Create(&(gLib.gc.cond)); + if (s == NATS_OK) + { + s = natsThread_Create(&(gLib.gc.thread), nats_garbageCollectorThreadf, &gLib); + if (s == NATS_OK) + gLib.refs++; + } + if (s == NATS_OK) + s = natsNUID_init(); + + if (s == NATS_OK) + s = nats_initDispatcherPool(&(gLib.messageDispatchers), config->ThreadPoolMax); + if (s == NATS_OK) + s = nats_initDispatcherPool(&(gLib.replyDispatchers), config->ReplyThreadPoolMax); + + if (s == NATS_OK) + s = natsMutex_Create(&gLib.service_callback_mu); + if (s == NATS_OK) + s = natsHash_Create(&gLib.all_services_to_callback, 8); + + if (s == NATS_OK) + gLib.initialized = true; + + // In case of success or error, broadcast so that lib's threads + // can proceed. + if (gLib.cond != NULL) + { + if (s != NATS_OK) + { + gLib.initAborted = true; + gLib.timers.shutdown = true; + gLib.asyncCbs.shutdown = true; + gLib.gc.shutdown = true; + } + natsCondition_Broadcast(gLib.cond); + } + + gLib.initializing = false; + gLib.wasOpenedOnce = true; + + natsMutex_Unlock(gLib.lock); + + if (s != NATS_OK) + _libTearDown(); + + return s; +} + +natsStatus +nats_closeLib(bool wait, int64_t timeout) +{ + natsStatus s = NATS_OK; + natsCondition *cond = NULL; + bool complete = false; + // int i; + + // This is to protect against a call to nats_Close() while there + // was no prior call to nats_Open(), either directly or indirectly. + if (!nats_InitOnce(&gInitOnce, _doInitOnce)) + return NATS_ERR; + + natsMutex_Lock(gLib.lock); + + if (gLib.closed || !gLib.initialized) + { + bool closed = gLib.closed; + + natsMutex_Unlock(gLib.lock); + + if (closed) + return NATS_ILLEGAL_STATE; + return NATS_NOT_INITIALIZED; + } + if (wait) + { + if (natsThreadLocal_Get(gLib.natsThreadKey) != NULL) + s = NATS_ILLEGAL_STATE; + if (s == NATS_OK) + s = natsCondition_Create(&cond); + if (s != NATS_OK) + { + natsMutex_Unlock(gLib.lock); + return s; + } + gLib.closeCompleteCond = cond; + gLib.closeCompleteBool = &complete; + gLib.closeCompleteSignal = true; + } + + gLib.closed = true; + + natsMutex_Lock(gLib.timers.lock); + gLib.timers.shutdown = true; + natsCondition_Signal(gLib.timers.cond); + natsMutex_Unlock(gLib.timers.lock); + + natsMutex_Lock(gLib.asyncCbs.lock); + gLib.asyncCbs.shutdown = true; + natsCondition_Signal(gLib.asyncCbs.cond); + natsMutex_Unlock(gLib.asyncCbs.lock); + + natsMutex_Lock(gLib.gc.lock); + gLib.gc.shutdown = true; + natsCondition_Signal(gLib.gc.cond); + natsMutex_Unlock(gLib.gc.lock); + + natsMutex_Unlock(gLib.lock); + + nats_signalDispatcherPoolToShutdown(&gLib.messageDispatchers); + nats_signalDispatcherPoolToShutdown(&gLib.replyDispatchers); + + nats_ReleaseThreadMemory(); + _libTearDown(); + + if (wait) + { + natsMutex_Lock(gLib.lock); + while ((s != NATS_TIMEOUT) && !complete) + { + if (timeout <= 0) + natsCondition_Wait(cond, gLib.lock); + else + s = natsCondition_TimedWait(cond, gLib.lock, timeout); + } + if (s != NATS_OK) + gLib.closeCompleteSignal = false; + natsMutex_Unlock(gLib.lock); + + natsCondition_Destroy(cond); + } + + return s; +} + +void nats_setNATSThreadKey(void) +{ + natsThreadLocal_Set(gLib.natsThreadKey, (const void *)1); +} + +void nats_ReleaseThreadMemory(void) +{ + void *tl = NULL; + + if (!(gLib.wasOpenedOnce)) + return; + + tl = natsThreadLocal_Get(gLib.errTLKey); + if (tl != NULL) + { + _destroyErrTL(tl); + natsThreadLocal_SetEx(gLib.errTLKey, NULL, false); + } + + tl = NULL; + + natsMutex_Lock(gLib.lock); + if (gLib.sslInitialized) + { + tl = natsThreadLocal_Get(gLib.sslTLKey); + if (tl != NULL) + { + nats_cleanupThreadSSL(tl); + natsThreadLocal_SetEx(gLib.sslTLKey, NULL, false); + } + } + natsMutex_Unlock(gLib.lock); +} + +natsStatus +natsLib_startServiceCallbacks(microService *m) +{ + natsStatus s; + + natsMutex_Lock(gLib.service_callback_mu); + s = natsHash_Set(gLib.all_services_to_callback, (int64_t)m, (void *)m, NULL); + natsMutex_Unlock(gLib.service_callback_mu); + + return NATS_UPDATE_ERR_STACK(s); +} + +void natsLib_stopServiceCallbacks(microService *m) +{ + if (m == NULL) + return; + + natsMutex_Lock(gLib.service_callback_mu); + natsHash_Remove(gLib.all_services_to_callback, (int64_t)m); + natsMutex_Unlock(gLib.service_callback_mu); +} + +natsMutex * +natsLib_getServiceCallbackMutex(void) +{ + return gLib.service_callback_mu; +} + +natsHash * +natsLib_getAllServicesToCallback(void) +{ + return gLib.all_services_to_callback; +} + +natsClientConfig *nats_testInspectClientConfig(void) +{ + // Immutable after startup + return &gLib.config; +} + +void nats_overrideDefaultOptionsWithConfig(natsOptions *opts) +{ + opts->writeDeadline = gLib.config.DefaultWriteDeadline; + opts->useSharedDispatcher = gLib.config.DefaultToThreadPool; + opts->useSharedReplyDispatcher = gLib.config.DefaultRepliesToThreadPool; +} diff --git a/src/glib/glib.h b/src/glib/glib.h new file mode 100644 index 000000000..96afbe78b --- /dev/null +++ b/src/glib/glib.h @@ -0,0 +1,44 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GLIB_H_ +#define GLIB_H_ + +typedef struct __natsDispatcherPool natsDispatcherPool; + +// Functions that are used internally by the library. + +void natsLib_Retain(void); +void natsLib_Release(void); + +natsStatus nats_assignSubToDispatch(natsSubscription *sub); +natsStatus nats_closeLib(bool wait, int64_t timeout); +natsStatus nats_initSSL(void); +natsStatus nats_openLib(natsClientConfig *config); +natsStatus nats_postAsyncCbInfo(natsAsyncCbInfo *info); +natsStatus nats_setMessageDispatcherPoolCap(int max); + +void nats_initForOS(void); +void nats_overrideDefaultOptionsWithConfig(natsOptions *opts); +void nats_resetTimer(natsTimer *t, int64_t newInterval); +void nats_stopTimer(natsTimer *t); + +// Returns the number of timers that have been created and not stopped. +int nats_getTimersCount(void); + +// Returns the number of timers actually in the list. This should be +// equal to nats_getTimersCount() or nats_getTimersCount() - 1 when a +// timer thread is invoking a timer's callback. +int nats_getTimersCountInList(void); + +#endif // GLIB_H_ diff --git a/src/glib/glib_async_cb.c b/src/glib/glib_async_cb.c new file mode 100644 index 000000000..8b2646539 --- /dev/null +++ b/src/glib/glib_async_cb.c @@ -0,0 +1,148 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +void +nats_freeAsyncCbs(natsLib *lib) +{ + natsLibAsyncCbs *cbs = &(lib->asyncCbs); + + natsThread_Destroy(cbs->thread); + natsCondition_Destroy(cbs->cond); + natsMutex_Destroy(cbs->lock); +} + + +void +nats_asyncCbsThreadf(void *arg) +{ + natsLib *lib = (natsLib*) arg; + natsLibAsyncCbs *asyncCbs = &(lib->asyncCbs); + natsAsyncCbInfo *cb = NULL; + natsConnection *nc = NULL; +#if defined(NATS_HAS_STREAMING) + stanConnection *sc = NULL; +#endif + + WAIT_LIB_INITIALIZED(lib); + + natsMutex_Lock(asyncCbs->lock); + + // We want all callbacks to be invoked even on shutdown + while (true) + { + while (((cb = asyncCbs->head) == NULL) && !asyncCbs->shutdown) + natsCondition_Wait(asyncCbs->cond, asyncCbs->lock); + + if ((cb == NULL) && asyncCbs->shutdown) + break; + + asyncCbs->head = cb->next; + + if (asyncCbs->tail == cb) + asyncCbs->tail = NULL; + + cb->next = NULL; + + natsMutex_Unlock(asyncCbs->lock); + + nc = cb->nc; +#if defined(NATS_HAS_STREAMING) + sc = cb->sc; +#endif + + switch (cb->type) + { + case ASYNC_CLOSED: + { + (*(nc->opts->closedCb))(nc, nc->opts->closedCbClosure); + if (nc->opts->microClosedCb != NULL) + (*(nc->opts->microClosedCb))(nc, NULL); + break; + } + + case ASYNC_DISCONNECTED: + (*(nc->opts->disconnectedCb))(nc, nc->opts->disconnectedCbClosure); + break; + case ASYNC_RECONNECTED: + (*(nc->opts->reconnectedCb))(nc, nc->opts->reconnectedCbClosure); + break; + case ASYNC_CONNECTED: + (*(nc->opts->connectedCb))(nc, nc->opts->connectedCbClosure); + break; + case ASYNC_DISCOVERED_SERVERS: + (*(nc->opts->discoveredServersCb))(nc, nc->opts->discoveredServersClosure); + break; + case ASYNC_LAME_DUCK_MODE: + (*(nc->opts->lameDuckCb))(nc, nc->opts->lameDuckClosure); + break; + case ASYNC_ERROR: + { + if (cb->errTxt != NULL) + nats_setErrStatusAndTxt(cb->err, cb->errTxt); + + (*(nc->opts->asyncErrCb))(nc, cb->sub, cb->err, nc->opts->asyncErrCbClosure); + if (nc->opts->microAsyncErrCb != NULL) + (*(nc->opts->microAsyncErrCb))(nc, cb->sub, cb->err, NULL); + break; + } +#if defined(NATS_HAS_STREAMING) + case ASYNC_STAN_CONN_LOST: + (*(sc->opts->connectionLostCB))(sc, sc->connLostErrTxt, sc->opts->connectionLostCBClosure); + break; +#endif + default: + break; + } + + natsAsyncCb_Destroy(cb); + nats_clearLastError(); + + natsMutex_Lock(asyncCbs->lock); + } + + natsMutex_Unlock(asyncCbs->lock); + + natsLib_Release(); +} + +natsStatus +nats_postAsyncCbInfo(natsAsyncCbInfo *info) +{ + natsLib *lib = nats_lib(); + natsMutex_Lock(lib->asyncCbs.lock); + + if (lib->asyncCbs.shutdown) + { + natsMutex_Unlock(lib->asyncCbs.lock); + return NATS_NOT_INITIALIZED; + } + + info->next = NULL; + + if (lib->asyncCbs.head == NULL) + lib->asyncCbs.head = info; + + if (lib->asyncCbs.tail != NULL) + lib->asyncCbs.tail->next = info; + + lib->asyncCbs.tail = info; + + natsCondition_Signal(lib->asyncCbs.cond); + + natsMutex_Unlock(lib->asyncCbs.lock); + + return NATS_OK; +} + diff --git a/src/glib/glib_dispatch_pool.c b/src/glib/glib_dispatch_pool.c new file mode 100644 index 000000000..5f4be28e4 --- /dev/null +++ b/src/glib/glib_dispatch_pool.c @@ -0,0 +1,180 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" +#include "../sub.h" + +static inline void +_destroyDispatcher(natsDispatcher *d) +{ + if ((d == NULL) || !d->running) + return; + + natsThread_Destroy(d->thread); + nats_destroyQueuedMessages(&d->queue); // there's NEVER anything there, remove? + natsCondition_Destroy(d->cond); + natsMutex_Destroy(d->mu); + memset(d, 0, sizeof(*d)); +} + +static inline natsStatus +_startDispatcher(natsDispatcher *d, void (*threadf)(void *)) +{ + natsStatus s = NATS_OK; + + if (d->running) + return NATS_OK; + + s = natsMutex_Create(&d->mu); + if (s != NATS_OK) + return s; + + natsCondition_Create(&d->cond); + + natsMutex_Lock(d->mu); + natsLib_Retain(); + s = natsThread_Create(&d->thread, threadf, (void *)d); + if (s == NATS_OK) + d->running = true; + natsMutex_Unlock(d->mu); + + if (s != NATS_OK) + { + _destroyDispatcher(d); + natsLib_Release(); + } + return NATS_UPDATE_ERR_STACK(s); +} + +static natsStatus +_growPool(natsDispatcherPool *pool, int cap) +{ + natsStatus s = NATS_OK; + + if (cap <= 0) + return nats_setError(NATS_ERR, "%s", "Pool size cannot be negative or zero"); + + // Do not error on max < workers->maxSize in case we allow shrinking + // the pool in the future. Make it a no-op for now. + if (cap > pool->cap) + { + natsDispatcher *newDispatchers = NATS_CALLOC(cap, sizeof(natsDispatcher)); + if (newDispatchers == NULL) + s = nats_setDefaultError(NATS_NO_MEMORY); + if (s == NATS_OK) + { + memcpy( + newDispatchers, + pool->dispatchers, + pool->cap * sizeof(*newDispatchers)); + NATS_FREE(pool->dispatchers); + pool->dispatchers = newDispatchers; + pool->cap = cap; + } + } + return NATS_UPDATE_ERR_STACK(s); +} + +void nats_freeDispatcherPool(natsDispatcherPool *pool) +{ + for (int i = 0; i < pool->cap; i++) + _destroyDispatcher(&pool->dispatchers[i]); + natsMutex_Destroy(pool->lock); + NATS_FREE(pool->dispatchers); + memset(pool, 0, sizeof(*pool)); +} + +natsStatus +nats_initDispatcherPool(natsDispatcherPool *pool, int cap) +{ + natsStatus s = NATS_OK; + + memset(pool, 0, sizeof(*pool)); + + s = natsMutex_Create(&pool->lock); + if (cap > 0) + IFOK(s, _growPool(pool, cap)); + + if (s != NATS_OK) + nats_freeDispatcherPool(pool); + return NATS_UPDATE_ERR_STACK(s); +} + +void nats_signalDispatcherPoolToShutdown(natsDispatcherPool *pool) +{ + natsCondition *cond = NULL; + + for (int i = 0; i < pool->cap; i++) + { + // These are no-ops for empty slots + nats_lockDispatcher(&pool->dispatchers[i]); + pool->dispatchers[i].shutdown = true; + cond = pool->dispatchers[i].cond; + if (cond != NULL) + natsCondition_Signal(cond); + + nats_unlockDispatcher(&pool->dispatchers[i]); + } +} + +void nats_waitForDispatcherPoolShutdown(natsDispatcherPool *pool) +{ + for (int i = 0; i < pool->cap; i++) + { + if (pool->dispatchers[i].thread != NULL) + natsThread_Join(pool->dispatchers[i].thread); + } +} + +natsStatus nats_setMessageDispatcherPoolCap(int max) +{ + natsLib *lib = nats_lib(); + + natsMutex_Lock(lib->messageDispatchers.lock); + natsStatus s = _growPool(&lib->messageDispatchers, max); + natsMutex_Unlock(lib->messageDispatchers.lock); + + return NATS_UPDATE_ERR_STACK(s); +} + +// no lock on sub->mu needed because we are called during subscription creation. +natsStatus +nats_assignSubToDispatch(natsSubscription *sub) +{ + natsLib *lib = nats_lib(); + natsStatus s = NATS_OK; + natsDispatcher *d = NULL; + natsDispatcherPool *pool = &lib->messageDispatchers; + + natsMutex_Lock(pool->lock); + + if (pool->cap == 0) + s = nats_setError(NATS_FAILED_TO_INITIALIZE, "%s", "No message dispatchers available, the pool is empty."); + + if (s == NATS_OK) + { + // Get the next dispatcher + d = &pool->dispatchers[pool->useNext]; + pool->useNext = (pool->useNext + 1) % pool->cap; + } + if ((s == NATS_OK) && (d->thread == NULL)) + s = _startDispatcher(d, nats_deliverMsgsPoolf); + + // Assign it to the sub. + if (s == NATS_OK) + sub->dispatcher = d; + + natsMutex_Unlock(pool->lock); + + return NATS_UPDATE_ERR_STACK(s); +} diff --git a/src/glib/glib_gc.c b/src/glib/glib_gc.c new file mode 100644 index 000000000..aa82a13cd --- /dev/null +++ b/src/glib/glib_gc.c @@ -0,0 +1,125 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + + +void +nats_freeGC(natsLib *lib) +{ + natsGCList *gc = &(lib->gc); + + natsThread_Destroy(gc->thread); + natsCondition_Destroy(gc->cond); + natsMutex_Destroy(gc->lock); +} + + +void +nats_garbageCollectorThreadf(void *closure) +{ + natsLib *lib = (natsLib*) closure; + natsGCList *gc = &(lib->gc); + natsGCItem *item; + natsGCItem *list; + + WAIT_LIB_INITIALIZED(lib); + + natsMutex_Lock(gc->lock); + + // Process all elements in the list, even on shutdown + while (true) + { + // Go into wait until we are notified to shutdown + // or there is something to garbage collect + gc->inWait = true; + + while (!(gc->shutdown) && (gc->head == NULL)) + { + natsCondition_Wait(gc->cond, gc->lock); + } + + // Out of the wait. Setting this boolean avoids unnecessary + // signaling when an item is added to the collector. + gc->inWait = false; + + // Do not break out on shutdown here, we want to clear the list, + // even on exit so that valgrind and the like are happy. + + // Under the lock, we will switch to a local list and reset the + // GC's list (so that others can add to the list without contention + // (at least from the GC itself). + do + { + list = gc->head; + gc->head = NULL; + + natsMutex_Unlock(gc->lock); + + // Now that we are outside of the lock, we can empty the list. + while ((item = list) != NULL) + { + // Pops item from the beginning of the list. + list = item->next; + item->next = NULL; + + // Invoke the freeCb associated with this object + (*(item->freeCb))((void*) item); + } + + natsMutex_Lock(gc->lock); + } + while (gc->head != NULL); + + // If we were ask to shutdown and since the list is now empty, exit + if (gc->shutdown) + break; + } + + natsMutex_Unlock(gc->lock); + + natsLib_Release(); +} + +bool +natsGC_collect(natsGCItem *item) +{ + natsGCList *gc; + bool signal; + + // If the object was not setup for garbage collection, return false + // so the caller frees the object. + if (item->freeCb == NULL) + return false; + + gc = &(nats_lib()->gc); + + natsMutex_Lock(gc->lock); + + // We will signal only if the GC is in the condition wait. + signal = gc->inWait; + + // Add to the front of the list. + item->next = gc->head; + + // Update head. + gc->head = item; + + if (signal) + natsCondition_Signal(gc->cond); + + natsMutex_Unlock(gc->lock); + + return true; +} + diff --git a/src/glib/glib_last_error.c b/src/glib/glib_last_error.c new file mode 100644 index 000000000..6ff9bddb7 --- /dev/null +++ b/src/glib/glib_last_error.c @@ -0,0 +1,349 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +static natsTLError* +_getThreadError(void) +{ + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + bool needFree = false; + + // The library should already be initialized, but let's protect against + // situations where foo() invokes bar(), which invokes baz(), which + // invokes nats_Open(). If that last call fails, when we un-wind down + // to foo(), it may be difficult to know that nats_Open() failed and + // that we should not try to invoke natsLib_setError. So we check again + // here that the library has been initialized properly, and if not, we + // simply don't set the error. + if (nats_Open(-1) != NATS_OK) + return NULL; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if (errTL == NULL) + { + errTL = (natsTLError*) NATS_CALLOC(1, sizeof(natsTLError)); + if (errTL != NULL) + errTL->framesCount = -1; + needFree = (errTL != NULL); + + } + + if ((errTL != NULL) + && (natsThreadLocal_SetEx(lib->errTLKey, + (const void*) errTL, false) != NATS_OK)) + { + if (needFree) + NATS_FREE(errTL); + + errTL = NULL; + } + + return errTL; +} + +static char* +_getErrorShortFileName(const char* fileName) +{ + char *file = strstr(fileName, "src"); + + if (file != NULL) + file = (file + 4); + else + file = (char*) fileName; + + return file; +} + +static void +_updateStack(natsTLError *errTL, const char *funcName, natsStatus errSts, + bool calledFromSetError) +{ + int idx; + + idx = errTL->framesCount; + if ((idx >= 0) + && (idx < MAX_FRAMES) + && (strcmp(errTL->func[idx], funcName) == 0)) + { + return; + } + + // In case no error was already set... + if ((errTL->framesCount == -1) && !calledFromSetError) + errTL->sts = errSts; + + idx = ++(errTL->framesCount); + + if (idx >= MAX_FRAMES) + return; + + errTL->func[idx] = funcName; +} + +#if !defined(_WIN32) +__attribute__ ((format (printf, 5, 6))) +#endif +natsStatus +nats_setErrorReal(const char *fileName, const char *funcName, int line, natsStatus errSts, const char *errTxtFmt, ...) +{ + natsTLError *errTL = _getThreadError(); + char tmp[256]; + va_list ap; + int n; + + if ((errTL == NULL) || errTL->skipUpdate) + return errSts; + + errTL->sts = errSts; + errTL->framesCount = -1; + + tmp[0] = '\0'; + + va_start(ap, errTxtFmt); + nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); + va_end(ap); + + if (strlen(tmp) > 0) + { + n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", + _getErrorShortFileName(fileName), line, tmp); + if ((n < 0) || (n >= (int) sizeof(errTL->text))) + { + int pos = ((int) strlen(errTL->text)) - 1; + int i; + + for (i=0; i<3; i++) + errTL->text[pos--] = '.'; + } + } + + _updateStack(errTL, funcName, errSts, true); + + return errSts; +} + +#if !defined(_WIN32) +__attribute__ ((format (printf, 4, 5))) +#endif +void +nats_updateErrTxt(const char *fileName, const char *funcName, int line, const char *errTxtFmt, ...) +{ + natsTLError *errTL = _getThreadError(); + char tmp[256]; + va_list ap; + int n; + + if ((errTL == NULL) || errTL->skipUpdate) + return; + + tmp[0] = '\0'; + + va_start(ap, errTxtFmt); + nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); + va_end(ap); + + if (strlen(tmp) > 0) + { + n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", + _getErrorShortFileName(fileName), line, tmp); + if ((n < 0) || (n >= (int) sizeof(errTL->text))) + { + int pos = ((int) strlen(errTL->text)) - 1; + int i; + + for (i=0; i<3; i++) + errTL->text[pos--] = '.'; + } + } +} + +void +nats_setErrStatusAndTxt(natsStatus err, const char *errTxt) +{ + natsTLError *errTL = _getThreadError(); + + if ((errTL == NULL) || errTL->skipUpdate) + return; + + errTL->sts = err; + snprintf(errTL->text, sizeof(errTL->text), "%s", errTxt); + errTL->framesCount = -1; +} + +natsStatus +nats_updateErrStack(natsStatus err, const char *func) +{ + natsTLError *errTL = _getThreadError(); + + if ((errTL == NULL) || errTL->skipUpdate) + return err; + + _updateStack(errTL, func, err, false); + + return err; +} + +void +nats_clearLastError(void) +{ + natsTLError *errTL = _getThreadError(); + + if ((errTL == NULL) || errTL->skipUpdate) + return; + + errTL->sts = NATS_OK; + errTL->text[0] = '\0'; + errTL->framesCount = -1; +} + +void +nats_doNotUpdateErrStack(bool skipStackUpdate) +{ + natsTLError *errTL = _getThreadError(); + + if (errTL == NULL) + return; + + if (skipStackUpdate) + { + errTL->skipUpdate++; + } + else + { + errTL->skipUpdate--; + assert(errTL->skipUpdate >= 0); + } +} + +const char* +nats_GetLastError(natsStatus *status) +{ + natsStatus s; + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + + if (status != NULL) + *status = NATS_OK; + + // Ensure the library is loaded + s = nats_Open(-1); + if (s != NATS_OK) + return NULL; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if ((errTL == NULL) || (errTL->sts == NATS_OK)) + return NULL; + + if (status != NULL) + *status = errTL->sts; + + return errTL->text; +} + +natsStatus +nats_GetLastErrorStack(char *buffer, size_t bufLen) +{ + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + int offset = 0; + int i, max, n, len; + + if ((buffer == NULL) || (bufLen == 0)) + return NATS_INVALID_ARG; + + buffer[0] = '\0'; + len = (int) bufLen; + + // Ensure the library is loaded + if (nats_Open(-1) != NATS_OK) + return NATS_FAILED_TO_INITIALIZE; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) + return NATS_OK; + + max = errTL->framesCount; + if (max >= MAX_FRAMES) + max = MAX_FRAMES - 1; + + for (i=0; (i<=max) && (len > 0); i++) + { + n = snprintf(buffer + offset, len, "%s%s", + errTL->func[i], + (i < max ? "\n" : "")); + // On Windows, n will be < 0 if len is not big enough. + if (n < 0) + { + len = 0; + } + else + { + offset += n; + len -= n; + } + } + + if ((max != errTL->framesCount) && (len > 0)) + { + n = snprintf(buffer + offset, len, "\n%d more...", + errTL->framesCount - max); + // On Windows, n will be < 0 if len is not big enough. + if (n < 0) + len = 0; + else + len -= n; + } + + if (len <= 0) + return NATS_INSUFFICIENT_BUFFER; + + return NATS_OK; +} + +void +nats_PrintLastErrorStack(FILE *file) +{ + natsLib *lib = nats_lib(); + natsTLError *errTL = NULL; + int i, max; + + // Ensure the library is loaded + if (nats_Open(-1) != NATS_OK) + return; + + errTL = natsThreadLocal_Get(lib->errTLKey); + if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) + return; + + fprintf(file, "Error: %u - %s", + errTL->sts, natsStatus_GetText(errTL->sts)); + if (errTL->text[0] != '\0') + fprintf(file, " - %s", errTL->text); + fprintf(file, "\n"); + fprintf(file, "Stack: (library version: %s)\n", nats_GetVersion()); + + max = errTL->framesCount; + if (max >= MAX_FRAMES) + max = MAX_FRAMES - 1; + + for (i=0; i<=max; i++) + fprintf(file, " %02d - %s\n", (i+1), errTL->func[i]); + + if (max != errTL->framesCount) + fprintf(file, " %d more...\n", errTL->framesCount - max); + + fflush(file); +} + diff --git a/src/glib/glib_ssl.c b/src/glib/glib_ssl.c new file mode 100644 index 000000000..e7470aae8 --- /dev/null +++ b/src/glib/glib_ssl.c @@ -0,0 +1,69 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + + +void +nats_cleanupThreadSSL(void *localStorage) +{ +#if defined(NATS_HAS_TLS) && !defined(NATS_USE_OPENSSL_1_1) + ERR_remove_thread_state(0); +#endif +} + +void nats_sslRegisterThreadForCleanup(void) +{ +#if defined(NATS_HAS_TLS) + natsLib *lib = nats_lib(); + // Set anything. The goal is that at thread exit, the thread local key + // will have something non NULL associated, which will trigger the + // destructor that we have registered. + (void)natsThreadLocal_Set(lib->sslTLKey, (void *)1); +#endif +} + +natsStatus +nats_initSSL(void) +{ + natsLib *lib = nats_lib(); + natsStatus s = NATS_OK; + + // Ensure the library is loaded + s = nats_openLib(NULL); + if (s != NATS_OK) + return s; + + natsMutex_Lock(lib->lock); + + if (!lib->sslInitialized) + { + // Regardless of success, mark as initialized so that we + // can do cleanup on exit. + lib->sslInitialized = true; + +#if defined(NATS_HAS_TLS) +#if !defined(NATS_USE_OPENSSL_1_1) + // Initialize SSL. + SSL_library_init(); + SSL_load_error_strings(); +#endif +#endif + s = natsThreadLocal_CreateKey(&(lib->sslTLKey), nats_cleanupThreadSSL); + } + + natsMutex_Unlock(lib->lock); + + return NATS_UPDATE_ERR_STACK(s); +} + diff --git a/src/glib/glib_timer.c b/src/glib/glib_timer.c new file mode 100644 index 000000000..ba99480a6 --- /dev/null +++ b/src/glib/glib_timer.c @@ -0,0 +1,332 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glibp.h" + +void nats_freeTimers(natsLib *lib) +{ + natsLibTimers *timers = &(lib->timers); + + natsThread_Destroy(timers->thread); + natsCondition_Destroy(timers->cond); + natsMutex_Destroy(timers->lock); +} + +static void +_insertTimer(natsTimer *t) +{ + natsTimer *cur = nats_lib()->timers.timers; + natsTimer *prev = NULL; + + while ((cur != NULL) && (cur->absoluteTime <= t->absoluteTime)) + { + prev = cur; + cur = cur->next; + } + + if (cur != NULL) + { + t->prev = prev; + t->next = cur; + cur->prev = t; + + if (prev != NULL) + prev->next = t; + } + else if (prev != NULL) + { + prev->next = t; + t->prev = prev; + t->next = NULL; + } + + if (prev == NULL) + nats_lib()->timers.timers = t; +} + +// Locks must be held before entering this function +static inline void +_removeTimer(natsLib *lib, natsTimer *t) +{ + // Switch flag + t->stopped = true; + + // It the timer was in the callback, it has already been removed from the + // list, so skip that. + if (!(t->inCallback)) + { + if (t->prev != NULL) + t->prev->next = t->next; + if (t->next != NULL) + t->next->prev = t->prev; + + if (t == lib->timers.timers) + lib->timers.timers = t->next; + + t->prev = NULL; + t->next = NULL; + } + + // Decrease the global count of timers + lib->timers.count--; +} + +void +nats_resetTimer(natsTimer *t, int64_t newInterval) +{ + natsLib *lib = nats_lib(); + natsLibTimers *timers = &(lib->timers); + + natsMutex_Lock(timers->lock); + natsMutex_Lock(t->mu); + + // If timer is active, we need first to remove it. This call does the + // right thing if the timer is in the callback. + if (!(t->stopped)) + _removeTimer(lib, t); + + // Bump the timer's global count (it as decreased in the _removeTimers call + timers->count++; + + // Switch stopped flag + t->stopped = false; + + // Set the new interval (may be same than it was before, but that's ok) + t->interval = newInterval; + + // If the timer is in the callback, the insertion and setting of the + // absolute time will be done by the timer thread when returning from + // the timer's callback. + if (!(t->inCallback)) + { + t->absoluteTime = nats_setTargetTime(t->interval); + _insertTimer(t); + } + + natsMutex_Unlock(t->mu); + + if (!(timers->changed)) + natsCondition_Signal(timers->cond); + + timers->changed = true; + + natsMutex_Unlock(timers->lock); +} + +void +nats_stopTimer(natsTimer *t) +{ + natsLib *lib = nats_lib(); + natsLibTimers *timers = &(lib->timers); + bool doCb = false; + + natsMutex_Lock(timers->lock); + natsMutex_Lock(t->mu); + + // If the timer was already stopped, nothing to do. + if (t->stopped) + { + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + return; + } + + _removeTimer(lib, t); + + doCb = (!(t->inCallback) && (t->stopCb != NULL)); + + natsMutex_Unlock(t->mu); + + if (!(timers->changed)) + natsCondition_Signal(timers->cond); + + timers->changed = true; + + natsMutex_Unlock(timers->lock); + + if (doCb) + (*(t->stopCb))(t, t->closure); +} + +int +nats_getTimersCount(void) +{ + natsLib *lib = nats_lib(); + int count = 0; + + natsMutex_Lock(lib->timers.lock); + + count = lib->timers.count; + + natsMutex_Unlock(lib->timers.lock); + + return count; +} + +int +nats_getTimersCountInList(void) +{ + natsLib *lib = nats_lib(); + int count = 0; + natsTimer *t; + + natsMutex_Lock(lib->timers.lock); + + t = lib->timers.timers; + while (t != NULL) + { + count++; + t = t->next; + } + + natsMutex_Unlock(lib->timers.lock); + + return count; +} + +void nats_timerThreadf(void *arg) +{ + natsLib *lib = (natsLib *)arg; + natsLibTimers *timers = &lib->timers; + natsTimer *t = NULL; + natsStatus s = NATS_OK; + bool doStopCb; + int64_t target; + + WAIT_LIB_INITIALIZED(lib); + + natsMutex_Lock(timers->lock); + + while (!(timers->shutdown)) + { + // Take the first timer that needs to fire. + t = timers->timers; + + if (t == NULL) + { + // No timer, fire in an hour... + target = nats_setTargetTime(3600 * 1000); + } + else + { + target = t->absoluteTime; + } + + timers->changed = false; + + s = NATS_OK; + + while (!(timers->shutdown) + && (s != NATS_TIMEOUT) + && !(timers->changed)) + { + s = natsCondition_AbsoluteTimedWait(timers->cond, timers->lock, + target); + } + + if (timers->shutdown) + break; + + if ((t == NULL) || timers->changed) + continue; + + natsMutex_Lock(t->mu); + + // Remove timer from the list: + timers->timers = t->next; + if (t->next != NULL) + t->next->prev = NULL; + + t->prev = NULL; + t->next = NULL; + + t->inCallback = true; + + // Retain the timer, since we are going to release the locks for the + // callback. The user may "destroy" the timer from there, so we need + // to be protected with reference counting. + t->refs++; + + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + (*(t->cb))(t, t->closure); + + natsMutex_Lock(timers->lock); + natsMutex_Lock(t->mu); + + t->inCallback = false; + + // Timer may have been stopped from within the callback, or during + // the window the locks were released. + doStopCb = (t->stopped && (t->stopCb != NULL)); + + // If not stopped, we need to put it back in our list + if (!doStopCb) + { + // Reset our view of what is the time this timer should fire + // because: + // 1- the callback may have taken longer than it should + // 2- the user may have called Reset() with a new interval + t->absoluteTime = nats_setTargetTime(t->interval); + _insertTimer(t); + } + + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + if (doStopCb) + (*(t->stopCb))(t, t->closure); + + // Compensate for the retain that we made before invoking the timer's + // callback + natsTimer_Release(t); + + natsMutex_Lock(timers->lock); + } + + // Process the timers that were left in the list (not stopped) when the + // library is shutdown. + while ((t = timers->timers) != NULL) + { + natsMutex_Lock(t->mu); + + // Check if we should invoke the callback. Note that although we are + // releasing the locks below, a timer present in the list here is + // guaranteed not to have been stopped (because it would not be in + // the list otherwise, since there is no chance that it is in the + // timer's callback). So just check if there is a stopCb to invoke. + doStopCb = (t->stopCb != NULL); + + // Remove the timer from the list. + _removeTimer(lib, t); + + natsMutex_Unlock(t->mu); + natsMutex_Unlock(timers->lock); + + // Invoke the callback now. + if (doStopCb) + (*(t->stopCb))(t, t->closure); + + // No release of the timer here. The user is still responsible to call + // natsTimer_Destroy(). + + natsMutex_Lock(timers->lock); + } + + natsMutex_Unlock(timers->lock); + + natsLib_Release(); +} + diff --git a/src/glib/glibp.h b/src/glib/glibp.h new file mode 100644 index 000000000..c6c4a8bb2 --- /dev/null +++ b/src/glib/glibp.h @@ -0,0 +1,145 @@ +// Copyright 2015-2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GLIBP_H_ +#define GLIBP_H_ + +#include "../natsp.h" +#include "../mem.h" +#if defined(NATS_HAS_STREAMING) +#include "../stan/stanp.h" +#endif + +#include +#include + +#include "glib.h" + +#define WAIT_LIB_INITIALIZED(_l) \ + natsMutex_Lock((_l)->lock); \ + while (!((_l)->initialized) && !((_l)->initAborted)) \ + natsCondition_Wait((_l)->cond, (_l)->lock); \ + natsMutex_Unlock((_l)->lock) + +typedef struct natsTLError +{ + natsStatus sts; + char text[256]; + const char *func[MAX_FRAMES]; + int framesCount; + int skipUpdate; + +} natsTLError; + +typedef struct __natsLibTimers +{ + natsMutex *lock; + natsCondition *cond; + natsThread *thread; + natsTimer *timers; + int count; + bool changed; + bool shutdown; + +} natsLibTimers; + +typedef struct __natsLibAsyncCbs +{ + natsMutex *lock; + natsCondition *cond; + natsThread *thread; + natsAsyncCbInfo *head; + natsAsyncCbInfo *tail; + bool shutdown; + +} natsLibAsyncCbs; + +typedef struct __natsGCList +{ + natsMutex *lock; + natsCondition *cond; + natsThread *thread; + natsGCItem *head; + bool shutdown; + bool inWait; + +} natsGCList; + +struct __natsDispatcherPool +{ + natsMutex *lock; + int useNext; // index of next dispatcher to use + int cap; // maximum number of concurrent dispatchers allowed + + natsDispatcher *dispatchers; + +}; + +typedef struct __natsLib +{ + // Leave these fields before 'refs' + natsMutex *lock; + volatile bool wasOpenedOnce; + bool sslInitialized; + natsThreadLocal errTLKey; + natsThreadLocal sslTLKey; + natsThreadLocal natsThreadKey; + bool initialized; + bool closed; + natsCondition *closeCompleteCond; + bool *closeCompleteBool; + bool closeCompleteSignal; + bool finalCleanup; + // Do not move 'refs' without checking _freeLib() + int refs; + + bool initializing; + bool initAborted; + + natsClientConfig config; + natsDispatcherPool messageDispatchers; + natsDispatcherPool replyDispatchers; + + natsLibTimers timers; + natsLibAsyncCbs asyncCbs; + + natsCondition *cond; + + natsGCList gc; + + // For micro services code + natsMutex *service_callback_mu; + // uses `microService*` as the key and the value. + natsHash *all_services_to_callback; + +} natsLib; + +natsLib *nats_lib(void); + +void nats_freeTimers(natsLib *lib); +void nats_timerThreadf(void *arg); // arg is &gLib + +void nats_freeAsyncCbs(natsLib *lib); +void nats_asyncCbsThreadf(void *arg); // arg is &gLib + +void nats_freeGC(natsLib *lib); +void nats_garbageCollectorThreadf(void *arg); // arg is &gLib + +natsStatus nats_initDispatcherPool(natsDispatcherPool *pool, int cap); +void nats_freeDispatcherPool(natsDispatcherPool *pool); +void nats_signalDispatcherPoolToShutdown(natsDispatcherPool *pool); +void nats_waitForDispatcherPoolShutdown(natsDispatcherPool *pool); + +void nats_cleanupThreadSSL(void *localStorage); + +#endif // GLIBP_H_ diff --git a/src/js.c b/src/js.c index e31dd2f0a..42a404275 100644 --- a/src/js.c +++ b/src/js.c @@ -1,4 +1,4 @@ -// Copyright 2021-2022 The NATS Authors +// Copyright 2021-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,6 +19,8 @@ #include "util.h" #include "opts.h" #include "sub.h" +#include "dispatch.h" +#include "glib/glib.h" #ifdef DEV_MODE // For type safety @@ -497,7 +499,7 @@ js_PublishMsg(jsPubAck **new_puback,jsCtx *js, natsMsg *msg, int64_t ttl = 0; nats_JSON *json = NULL; natsMsg *resp = NULL; - jsApiResponse ar; + jsApiResponse ar = JS_EMPTY_API_RESPONSE; if (errCode != NULL) *errCode = 0; @@ -837,23 +839,13 @@ _timeoutPubAsync(natsTimer *t, void *closure) if (natsMsg_Create(&m, pm->subject, NULL, NULL, 0) != NATS_OK) break; - m->sub = js->rsub; natsMsg_setTimeout(m); - natsSub_Lock(js->rsub); - if (js->rsub->msgList.tail != NULL) - { - js->rsub->msgList.tail->next = m; - } - else - { - js->rsub->msgList.head = m; - natsCondition_Signal(js->rsub->cond); - } - js->rsub->msgList.tail = m; - js->rsub->msgList.msgs++; - js->rsub->msgList.bytes += natsMsg_dataAndHdrLen(m); - natsSub_Unlock(js->rsub); + // Best attempt, ignore NATS_SLOW_CONSUMER errors which may be returned + // here. + nats_lockSubAndDispatcher(js->rsub); + natsSub_enqueueUserMessage(js->rsub, m); + nats_unlockSubAndDispatcher(js->rsub); js->pmHead = pm->next; _destroyPMInfo(pm); @@ -1236,11 +1228,7 @@ jsSub_free(jsSub *jsi) js = jsi->js; natsTimer_Destroy(jsi->hbTimer); - if (jsi->mhMsg != NULL) - { - natsMsg_clearNoDestroy(jsi->mhMsg); - natsMsg_Destroy(jsi->mhMsg); - } + NATS_FREE(jsi->stream); NATS_FREE(jsi->consumer); NATS_FREE(jsi->nxtMsgSubj); @@ -1644,24 +1632,24 @@ natsSubscription_GetSequenceMismatch(jsConsumerSequenceMismatch *csm, natsSubscr if ((csm == NULL) || (sub == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->jsi == NULL) { - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setError(NATS_INVALID_SUBSCRIPTION, "%s", jsErrNotAJetStreamSubscription); } jsi = sub->jsi; m = &jsi->mismatch; if (m->dseq == m->ldseq) { - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_NOT_FOUND; } memset(csm, 0, sizeof(jsConsumerSequenceMismatch)); csm->Stream = m->sseq; csm->ConsumerClient = m->dseq; csm->ConsumerServer = m->ldseq; - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1807,7 +1795,7 @@ _fetch(natsMsgList *list, natsSubscription *sub, jsFetchRequest *req, bool simpl jsSub *jsi = NULL; natsMsg *mhMsg = NULL; char *reqSubj = NULL; - bool noWait; + bool noWait = false; if (list == NULL) return nats_setDefaultError(NATS_INVALID_ARG); @@ -1841,7 +1829,7 @@ _fetch(natsMsgList *list, natsSubscription *sub, jsFetchRequest *req, bool simpl natsBuf_InitWithBackend(&buf, buffer, 0, sizeof(buffer)); nc = sub->conn; subj = jsi->nxtMsgSubj; - pmc = (sub->msgList.msgs > 0); + pmc = (sub->ownDispatcher.queue.msgs > 0); jsi->inFetch = true; jsi->fetchID++; if (nats_asprintf(&reqSubj, "%.*s%" PRIu64, (int) strlen(sub->subject)-1, sub->subject, jsi->fetchID) < 0) @@ -1850,23 +1838,22 @@ _fetch(natsMsgList *list, natsSubscription *sub, jsFetchRequest *req, bool simpl rply = (const char*) reqSubj; if ((s == NATS_OK) && req->Heartbeat) { - int64_t hbi = req->Heartbeat / 1000000; - sub->refs++; - if (jsi->hbTimer == NULL) + s = nats_createControlMessages(sub); + if (s == NATS_OK) { - s = natsMsg_create(&jsi->mhMsg, NULL, 0, NULL, 0, NULL, 0, -1); - if (s == NATS_OK) + int64_t hbi = req->Heartbeat / 1000000; + sub->refs++; + if (jsi->hbTimer == NULL) { - natsMsg_setNoDestroy(jsi->mhMsg); - s = natsTimer_Create(&jsi->hbTimer, _hbTimerFired, _hbTimerStopped, hbi*2, (void*) sub); + s = natsTimer_Create(&jsi->hbTimer, _hbTimerFired, _hbTimerStopped, hbi * 2, (void *)sub); + if (s != NATS_OK) + sub->refs--; } - if (s != NATS_OK) - sub->refs--; - } - else - natsTimer_Reset(jsi->hbTimer, hbi); + else + natsTimer_Reset(jsi->hbTimer, hbi); - mhMsg = jsi->mhMsg; + mhMsg = sub->control->batch.missedHeartbeat; + } } natsSub_Unlock(sub); @@ -2044,7 +2031,7 @@ _hbTimerFired(natsTimer *timer, void* closure) bool oc = false; natsStatus s = NATS_OK; - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); alert = !jsi->active; oc = jsi->ordered; jsi->active = false; @@ -2053,20 +2040,16 @@ _hbTimerFired(natsTimer *timer, void* closure) // If there are messages pending then we can't really consider // that we missed hearbeats. Wait for those to be processed and // we will check missed HBs again. - if (sub->msgList.msgs == 0) + if (sub->ownDispatcher.queue.msgs == 0) { - sub->msgList.msgs++; - sub->msgList.head = jsi->mhMsg; - sub->msgList.tail = jsi->mhMsg; - sub->msgList.bytes = natsMsg_dataAndHdrLen(jsi->mhMsg); - natsCondition_Signal(sub->cond); + natsSub_enqueueMessage(sub, sub->control->batch.missedHeartbeat); natsTimer_Stop(timer); } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return; } nc = sub->conn; - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); if (!alert) return; @@ -2074,14 +2057,14 @@ _hbTimerFired(natsTimer *timer, void* closure) // For ordered consumers, we will need to reset if (oc) { - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (!sub->closed) { // If we fail in that call, we will report to async err callback // (if one is specified). s = jsSub_resetOrderedConsumer(sub, sub->jsi->sseq+1); } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } natsConn_Lock(nc); @@ -2688,7 +2671,7 @@ _subscribeMulti(natsSubscription **new_sub, jsCtx *js, const char **subjects, in else { maxap = info->Config->MaxAckPending; - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); jsi->dc = true; jsi->pending = info->NumPending + info->Delivered.Consumer; // There may be a race in the case of an ordered consumer where by this @@ -2705,7 +2688,7 @@ _subscribeMulti(natsSubscription **new_sub, jsCtx *js, const char **subjects, in s = nats_setDefaultError(NATS_NO_MEMORY); } } - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } } diff --git a/src/js.h b/src/js.h index bc5ec2abe..0503849a7 100644 --- a/src/js.h +++ b/src/js.h @@ -205,6 +205,8 @@ typedef struct __jsApiResponse } jsApiResponse; +#define JS_EMPTY_API_RESPONSE { NULL, { 0, 0, NULL } } + // Sets the options in `resOpts` based on the given `opts` and defaults to the context // own options when some options are not specified. // Returns also the NATS connection to be used to send the request. diff --git a/src/jsm.c b/src/jsm.c index 3c8321491..7c2201231 100644 --- a/src/jsm.c +++ b/src/jsm.c @@ -1517,7 +1517,7 @@ js_GetStreamInfo(jsStreamInfo **new_si, jsCtx *js, const char *stream, jsOptions bool done = false; jsStreamInfo *si = NULL; jsStreamStateSubjects *subjects = NULL; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -1879,7 +1879,7 @@ _getMsg(natsMsg **msg, jsCtx *js, const char *stream, uint64_t seq, const char * bool freePfx = false; jsOptions o; char buffer[64]; - natsBuffer buf; + natsBuffer buf = NATS_EMPTY_BUFFER; if ((msg == NULL) || (js == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); @@ -2029,7 +2029,7 @@ js_DirectGetMsg(natsMsg **msg, jsCtx *js, const char *stream, jsOptions *opts, j bool doLBS = false; jsOptions o; char buffer[64]; - natsBuffer buf; + natsBuffer buf = NATS_EMPTY_BUFFER; if ((msg == NULL) || (js == NULL) || (dgOpts == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); @@ -2108,7 +2108,7 @@ _deleteMsg(jsCtx *js, bool noErase, const char *stream, uint64_t seq, jsOptions bool success = false; jsOptions o; char buffer[64]; - natsBuffer buf; + natsBuffer buf = NATS_EMPTY_BUFFER; if (errCode != NULL) *errCode = 0; @@ -2239,7 +2239,7 @@ js_Streams(jsStreamInfoList **new_list, jsCtx *js, jsOptions *opts, jsErrCode *e natsStrHash *streams= NULL; jsStreamInfoList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -2441,7 +2441,7 @@ js_StreamNames(jsStreamNamesList **new_list, jsCtx *js, jsOptions *opts, jsErrCo natsStrHash *names = NULL; jsStreamNamesList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -3693,7 +3693,7 @@ js_Consumers(jsConsumerInfoList **new_list, jsCtx *js, const char *stream, jsOpt natsStrHash *cons = NULL; jsConsumerInfoList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; @@ -3844,7 +3844,7 @@ js_ConsumerNames(jsConsumerNamesList **new_list, jsCtx *js, const char *stream, natsStrHash *names = NULL; jsConsumerNamesList *list = NULL; jsOptions o; - apiPaged page; + apiPaged page = {0}; if (errCode != NULL) *errCode = 0; diff --git a/src/kv.c b/src/kv.c index 81de4ce56..2b079680c 100644 --- a/src/kv.c +++ b/src/kv.c @@ -1065,7 +1065,7 @@ kvStore_Watch(kvWatcher **new_watcher, kvStore *kv, const char *key, kvWatchOpti natsStatus kvStore_WatchMulti(kvWatcher **new_watcher, kvStore *kv, const char **keys, int numKeys, kvWatchOptions *opts) { - natsStatus s; + natsStatus s = NATS_OK; kvWatcher *w = NULL; jsSubOptions so; char *singleSubject[1]; diff --git a/src/msg.h b/src/msg.h index 2cbc9433c..4f56611d5 100644 --- a/src/msg.h +++ b/src/msg.h @@ -1,4 +1,4 @@ -// Copyright 2015-2022 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -40,6 +40,7 @@ #define natsMsg_setNoDestroy(m) ((m)->flags |= (1 << 2)) #define natsMsg_isNoDestroy(m) (((m)->flags & (1 << 2)) != 0) #define natsMsg_clearNoDestroy(m) ((m)->flags &= ~(1 << 2)) +#define natsMsg_noDestroyFlag (1 << 2) #define natsMsg_setTimeout(m) ((m)->flags |= (1 << 3)) #define natsMsg_isTimeout(m) (((m)->flags & (1 << 3)) != 0) diff --git a/src/nats.c b/src/nats.c index 1c2973d63..06e46c5b9 100644 --- a/src/nats.c +++ b/src/nats.c @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,1680 +16,222 @@ #include "stan/stanp.h" #endif -#include -#include -#include -#include -#include -#include - #include "mem.h" -#include "timer.h" -#include "util.h" -#include "asynccb.h" -#include "conn.h" +#include "glib/glib.h" #include "sub.h" -#include "nkeys.h" -#include "crypto.h" - -#define WAIT_LIB_INITIALIZED \ - natsMutex_Lock(gLib.lock); \ - while (!(gLib.initialized) && !(gLib.initAborted)) \ - natsCondition_Wait(gLib.cond, gLib.lock); \ - natsMutex_Unlock(gLib.lock) - -typedef struct natsTLError -{ - natsStatus sts; - char text[256]; - const char *func[MAX_FRAMES]; - int framesCount; - int skipUpdate; - -} natsTLError; - -typedef struct __natsLibTimers -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - natsTimer *timers; - int count; - bool changed; - bool shutdown; - -} natsLibTimers; - -typedef struct __natsLibAsyncCbs -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - natsAsyncCbInfo *head; - natsAsyncCbInfo *tail; - bool shutdown; - -} natsLibAsyncCbs; - -typedef struct __natsGCList -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - natsGCItem *head; - bool shutdown; - bool inWait; - -} natsGCList; - -typedef struct __natsLibDlvWorkers -{ - natsMutex *lock; - int idx; - int size; - int maxSize; - natsMsgDlvWorker **workers; - -} natsLibDlvWorkers; - -typedef struct __natsLib -{ - // Leave these fields before 'refs' - natsMutex *lock; - volatile bool wasOpenedOnce; - bool sslInitialized; - natsThreadLocal errTLKey; - natsThreadLocal sslTLKey; - natsThreadLocal natsThreadKey; - bool initialized; - bool closed; - natsCondition *closeCompleteCond; - bool *closeCompleteBool; - bool closeCompleteSignal; - bool finalCleanup; - // Do not move 'refs' without checking _freeLib() - int refs; - - bool initializing; - bool initAborted; - bool libHandlingMsgDeliveryByDefault; - int64_t libDefaultWriteDeadline; - - natsLibTimers timers; - natsLibAsyncCbs asyncCbs; - natsLibDlvWorkers dlvWorkers; - - natsCondition *cond; - - natsGCList gc; - - // For micro services code - natsMutex *service_callback_mu; - // uses `microService*` as the key and the value. - natsHash *all_services_to_callback; - -} natsLib; - -int64_t gLockSpinCount = 2000; - -static natsInitOnceType gInitOnce = NATS_ONCE_STATIC_INIT; -static natsLib gLib; - -static void -_destroyErrTL(void *localStorage) -{ - natsTLError *err = (natsTLError*) localStorage; - - NATS_FREE(err); -} - -static void -_cleanupThreadSSL(void *localStorage) -{ -#if defined(NATS_HAS_TLS) && !defined(NATS_USE_OPENSSL_1_1) - ERR_remove_thread_state(0); -#endif -} - -static void -_finalCleanup(void) -{ - if (gLib.sslInitialized) - { -#if defined(NATS_HAS_TLS) -#if !defined(NATS_USE_OPENSSL_1_1) - ERR_free_strings(); - EVP_cleanup(); - CRYPTO_cleanup_all_ex_data(); - ERR_remove_thread_state(0); -#endif - sk_SSL_COMP_free(SSL_COMP_get_compression_methods()); -#endif - natsThreadLocal_DestroyKey(gLib.sslTLKey); - } - - natsThreadLocal_DestroyKey(gLib.errTLKey); - natsThreadLocal_DestroyKey(gLib.natsThreadKey); - natsMutex_Destroy(gLib.lock); - gLib.lock = NULL; -} - -void -nats_setNATSThreadKey(void) -{ - natsThreadLocal_Set(gLib.natsThreadKey, (const void*)1); -} - -void -nats_ReleaseThreadMemory(void) -{ - void *tl = NULL; - - if (!(gLib.wasOpenedOnce)) - return; - - tl = natsThreadLocal_Get(gLib.errTLKey); - if (tl != NULL) - { - _destroyErrTL(tl); - natsThreadLocal_SetEx(gLib.errTLKey, NULL, false); - } - - tl = NULL; - - natsMutex_Lock(gLib.lock); - if (gLib.sslInitialized) - { - tl = natsThreadLocal_Get(gLib.sslTLKey); - if (tl != NULL) - { - _cleanupThreadSSL(tl); - natsThreadLocal_SetEx(gLib.sslTLKey, NULL, false); - } - } - natsMutex_Unlock(gLib.lock); -} - -#if defined(_WIN32) && _WIN32 -#ifndef NATS_STATIC -BOOL WINAPI DllMain(HINSTANCE hinstDLL, // DLL module handle - DWORD fdwReason, // reason called - LPVOID lpvReserved) // reserved -{ - switch (fdwReason) - { - // For applications linking dynamically NATS library, - // release thread-local memory for user-created threads. - // For portable applications, the user should manually call - // nats_ReleaseThreadMemory() before the thread returns so - // that no memory is leaked regardless if they link statically - // or dynamically. It is safe to call nats_ReleaseThreadMemory() - // twice for the same threads. - case DLL_THREAD_DETACH: - { - nats_ReleaseThreadMemory(); - break; - } - default: - break; - } - - return TRUE; - UNREFERENCED_PARAMETER(hinstDLL); - UNREFERENCED_PARAMETER(lpvReserved); -} -#endif -#endif - -static void -natsLib_Destructor(void) -{ - int refs = 0; - - if (!(gLib.wasOpenedOnce)) - return; - - // Destroy thread locals for the current thread. - nats_ReleaseThreadMemory(); - - // Do the final cleanup if possible - natsMutex_Lock(gLib.lock); - refs = gLib.refs; - if (refs > 0) - { - // If some thread is still around when the process exits and has a - // reference to the library, then don't do the final cleanup now. - // If the process has not fully exited when the lib's last reference - // is decremented, the final cleanup will be executed from that thread. - gLib.finalCleanup = true; - } - natsMutex_Unlock(gLib.lock); - - if (refs != 0) - return; - - _finalCleanup(); -} - -static void -_freeTimers(void) -{ - natsLibTimers *timers = &(gLib.timers); - - natsThread_Destroy(timers->thread); - natsCondition_Destroy(timers->cond); - natsMutex_Destroy(timers->lock); -} - -static void -_freeAsyncCbs(void) -{ - natsLibAsyncCbs *cbs = &(gLib.asyncCbs); - - natsThread_Destroy(cbs->thread); - natsCondition_Destroy(cbs->cond); - natsMutex_Destroy(cbs->lock); -} - -static void -_freeGC(void) -{ - natsGCList *gc = &(gLib.gc); - - natsThread_Destroy(gc->thread); - natsCondition_Destroy(gc->cond); - natsMutex_Destroy(gc->lock); -} - -static void -_freeDlvWorker(natsMsgDlvWorker *worker) -{ - natsThread_Destroy(worker->thread); - natsCondition_Destroy(worker->cond); - natsMutex_Destroy(worker->lock); - NATS_FREE(worker); -} - -static void -_freeDlvWorkers(void) -{ - int i; - natsLibDlvWorkers *workers = &(gLib.dlvWorkers); - - for (i=0; isize; i++) - _freeDlvWorker(workers->workers[i]); - - NATS_FREE(workers->workers); - natsMutex_Destroy(workers->lock); - workers->idx = 0; - workers->size = 0; - workers->workers = NULL; -} - -static void -_freeLib(void) -{ - const unsigned int offset = (unsigned int) offsetof(natsLib, refs); - bool callFinalCleanup = false; - - _freeTimers(); - _freeAsyncCbs(); - _freeGC(); - _freeDlvWorkers(); - natsNUID_free(); - natsMutex_Destroy(gLib.service_callback_mu); - natsHash_Destroy(gLib.all_services_to_callback); - - natsCondition_Destroy(gLib.cond); - - memset((void*) (offset + (char*)&gLib), 0, sizeof(natsLib) - offset); - - natsMutex_Lock(gLib.lock); - callFinalCleanup = gLib.finalCleanup; - if (gLib.closeCompleteCond != NULL) - { - if (gLib.closeCompleteSignal) - { - *gLib.closeCompleteBool = true; - natsCondition_Signal(gLib.closeCompleteCond); - } - gLib.closeCompleteCond = NULL; - gLib.closeCompleteBool = NULL; - gLib.closeCompleteSignal = false; - } - gLib.closed = false; - gLib.initialized = false; - gLib.finalCleanup= false; - natsMutex_Unlock(gLib.lock); - - if (callFinalCleanup) - _finalCleanup(); -} - -void -natsLib_Retain(void) -{ - natsMutex_Lock(gLib.lock); - - gLib.refs++; - - natsMutex_Unlock(gLib.lock); -} - -void -natsLib_Release(void) -{ - int refs = 0; - - natsMutex_Lock(gLib.lock); - - refs = --(gLib.refs); - - natsMutex_Unlock(gLib.lock); - - if (refs == 0) - _freeLib(); -} - -static void -_doInitOnce(void) -{ - natsStatus s; - - memset(&gLib, 0, sizeof(natsLib)); - - s = natsMutex_Create(&(gLib.lock)); - if (s == NATS_OK) - s = natsThreadLocal_CreateKey(&(gLib.errTLKey), _destroyErrTL); - if (s == NATS_OK) - s = natsThreadLocal_CreateKey(&(gLib.natsThreadKey), NULL); - if (s != NATS_OK) - { - fprintf(stderr, "FATAL ERROR: Unable to initialize library!\n"); - fflush(stderr); - abort(); - } - - natsSys_Init(); - - // Setup a hook for when the process exits. - atexit(natsLib_Destructor); -} - -static void -_insertTimer(natsTimer *t) -{ - natsTimer *cur = gLib.timers.timers; - natsTimer *prev = NULL; - - while ((cur != NULL) && (cur->absoluteTime <= t->absoluteTime)) - { - prev = cur; - cur = cur->next; - } - - if (cur != NULL) - { - t->prev = prev; - t->next = cur; - cur->prev = t; - - if (prev != NULL) - prev->next = t; - } - else if (prev != NULL) - { - prev->next = t; - t->prev = prev; - t->next = NULL; - } - - if (prev == NULL) - gLib.timers.timers = t; -} - -// Locks must be held before entering this function -static void -_removeTimer(natsLibTimers *timers, natsTimer *t) -{ - // Switch flag - t->stopped = true; - - // It the timer was in the callback, it has already been removed from the - // list, so skip that. - if (!(t->inCallback)) - { - if (t->prev != NULL) - t->prev->next = t->next; - if (t->next != NULL) - t->next->prev = t->prev; - - if (t == gLib.timers.timers) - gLib.timers.timers = t->next; - - t->prev = NULL; - t->next = NULL; - } - - // Decrease the global count of timers - timers->count--; -} - -int64_t -nats_setTargetTime(int64_t timeout) -{ - int64_t target = nats_Now() + timeout; - if (target < 0) - target = 0x7FFFFFFFFFFFFFFF; - return target; -} - -void -nats_resetTimer(natsTimer *t, int64_t newInterval) -{ - natsLibTimers *timers = &(gLib.timers); - - natsMutex_Lock(timers->lock); - natsMutex_Lock(t->mu); - - // If timer is active, we need first to remove it. This call does the - // right thing if the timer is in the callback. - if (!(t->stopped)) - _removeTimer(timers, t); - - // Bump the timer's global count (it as decreased in the _removeTimers call - timers->count++; - - // Switch stopped flag - t->stopped = false; - - // Set the new interval (may be same than it was before, but that's ok) - t->interval = newInterval; - - // If the timer is in the callback, the insertion and setting of the - // absolute time will be done by the timer thread when returning from - // the timer's callback. - if (!(t->inCallback)) - { - t->absoluteTime = nats_setTargetTime(t->interval); - _insertTimer(t); - } - - natsMutex_Unlock(t->mu); - - if (!(timers->changed)) - natsCondition_Signal(timers->cond); - - timers->changed = true; - - natsMutex_Unlock(timers->lock); -} - -void -nats_stopTimer(natsTimer *t) -{ - natsLibTimers *timers = &(gLib.timers); - bool doCb = false; - - natsMutex_Lock(timers->lock); - natsMutex_Lock(t->mu); - - // If the timer was already stopped, nothing to do. - if (t->stopped) - { - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - return; - } - - _removeTimer(timers, t); - - doCb = (!(t->inCallback) && (t->stopCb != NULL)); - - natsMutex_Unlock(t->mu); - - if (!(timers->changed)) - natsCondition_Signal(timers->cond); - - timers->changed = true; - - natsMutex_Unlock(timers->lock); - - if (doCb) - (*(t->stopCb))(t, t->closure); -} - -int -nats_getTimersCount(void) -{ - int count = 0; - - natsMutex_Lock(gLib.timers.lock); - - count = gLib.timers.count; - - natsMutex_Unlock(gLib.timers.lock); - - return count; -} - -int -nats_getTimersCountInList(void) -{ - int count = 0; - natsTimer *t; - - natsMutex_Lock(gLib.timers.lock); - - t = gLib.timers.timers; - while (t != NULL) - { - count++; - t = t->next; - } - - natsMutex_Unlock(gLib.timers.lock); - - return count; -} - - -static void -_timerThread(void *arg) -{ - natsLibTimers *timers = &(gLib.timers); - natsTimer *t = NULL; - natsStatus s = NATS_OK; - bool doStopCb; - int64_t target; - - WAIT_LIB_INITIALIZED; - - natsMutex_Lock(timers->lock); - - while (!(timers->shutdown)) - { - // Take the first timer that needs to fire. - t = timers->timers; - - if (t == NULL) - { - // No timer, fire in an hour... - target = nats_setTargetTime(3600 * 1000); - } - else - { - target = t->absoluteTime; - } - - timers->changed = false; - - s = NATS_OK; - - while (!(timers->shutdown) - && (s != NATS_TIMEOUT) - && !(timers->changed)) - { - s = natsCondition_AbsoluteTimedWait(timers->cond, timers->lock, - target); - } - - if (timers->shutdown) - break; - - if ((t == NULL) || timers->changed) - continue; - - natsMutex_Lock(t->mu); - - // Remove timer from the list: - timers->timers = t->next; - if (t->next != NULL) - t->next->prev = NULL; - - t->prev = NULL; - t->next = NULL; - - t->inCallback = true; - - // Retain the timer, since we are going to release the locks for the - // callback. The user may "destroy" the timer from there, so we need - // to be protected with reference counting. - t->refs++; - - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - (*(t->cb))(t, t->closure); - - natsMutex_Lock(timers->lock); - natsMutex_Lock(t->mu); - - t->inCallback = false; - - // Timer may have been stopped from within the callback, or during - // the window the locks were released. - doStopCb = (t->stopped && (t->stopCb != NULL)); - - // If not stopped, we need to put it back in our list - if (!doStopCb) - { - // Reset our view of what is the time this timer should fire - // because: - // 1- the callback may have taken longer than it should - // 2- the user may have called Reset() with a new interval - t->absoluteTime = nats_setTargetTime(t->interval); - _insertTimer(t); - } - - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - if (doStopCb) - (*(t->stopCb))(t, t->closure); - - // Compensate for the retain that we made before invoking the timer's - // callback - natsTimer_Release(t); - - natsMutex_Lock(timers->lock); - } - - // Process the timers that were left in the list (not stopped) when the - // library is shutdown. - while ((t = timers->timers) != NULL) - { - natsMutex_Lock(t->mu); - - // Check if we should invoke the callback. Note that although we are - // releasing the locks below, a timer present in the list here is - // guaranteed not to have been stopped (because it would not be in - // the list otherwise, since there is no chance that it is in the - // timer's callback). So just check if there is a stopCb to invoke. - doStopCb = (t->stopCb != NULL); - - // Remove the timer from the list. - _removeTimer(timers, t); - - natsMutex_Unlock(t->mu); - natsMutex_Unlock(timers->lock); - - // Invoke the callback now. - if (doStopCb) - (*(t->stopCb))(t, t->closure); - - // No release of the timer here. The user is still responsible to call - // natsTimer_Destroy(). - - natsMutex_Lock(timers->lock); - } - - natsMutex_Unlock(timers->lock); - - natsLib_Release(); -} - -static void -_asyncCbsThread(void *arg) -{ - natsLibAsyncCbs *asyncCbs = &(gLib.asyncCbs); - natsAsyncCbInfo *cb = NULL; - natsConnection *nc = NULL; -#if defined(NATS_HAS_STREAMING) - stanConnection *sc = NULL; -#endif - - WAIT_LIB_INITIALIZED; - - natsMutex_Lock(asyncCbs->lock); - - // We want all callbacks to be invoked even on shutdown - while (true) - { - while (((cb = asyncCbs->head) == NULL) && !asyncCbs->shutdown) - natsCondition_Wait(asyncCbs->cond, asyncCbs->lock); - - if ((cb == NULL) && asyncCbs->shutdown) - break; - - asyncCbs->head = cb->next; - - if (asyncCbs->tail == cb) - asyncCbs->tail = NULL; - - cb->next = NULL; - - natsMutex_Unlock(asyncCbs->lock); - - nc = cb->nc; -#if defined(NATS_HAS_STREAMING) - sc = cb->sc; -#endif - - switch (cb->type) - { - case ASYNC_CLOSED: - { - (*(nc->opts->closedCb))(nc, nc->opts->closedCbClosure); - if (nc->opts->microClosedCb != NULL) - (*(nc->opts->microClosedCb))(nc, NULL); - break; - } - - case ASYNC_DISCONNECTED: - (*(nc->opts->disconnectedCb))(nc, nc->opts->disconnectedCbClosure); - break; - case ASYNC_RECONNECTED: - (*(nc->opts->reconnectedCb))(nc, nc->opts->reconnectedCbClosure); - break; - case ASYNC_CONNECTED: - (*(nc->opts->connectedCb))(nc, nc->opts->connectedCbClosure); - break; - case ASYNC_DISCOVERED_SERVERS: - (*(nc->opts->discoveredServersCb))(nc, nc->opts->discoveredServersClosure); - break; - case ASYNC_LAME_DUCK_MODE: - (*(nc->opts->lameDuckCb))(nc, nc->opts->lameDuckClosure); - break; - case ASYNC_ERROR: - { - if (cb->errTxt != NULL) - nats_setErrStatusAndTxt(cb->err, cb->errTxt); - - (*(nc->opts->asyncErrCb))(nc, cb->sub, cb->err, nc->opts->asyncErrCbClosure); - if (nc->opts->microAsyncErrCb != NULL) - (*(nc->opts->microAsyncErrCb))(nc, cb->sub, cb->err, NULL); - break; - } -#if defined(NATS_HAS_STREAMING) - case ASYNC_STAN_CONN_LOST: - (*(sc->opts->connectionLostCB))(sc, sc->connLostErrTxt, sc->opts->connectionLostCBClosure); - break; -#endif - default: - break; - } - - natsAsyncCb_Destroy(cb); - nats_clearLastError(); - - natsMutex_Lock(asyncCbs->lock); - } - - natsMutex_Unlock(asyncCbs->lock); - - natsLib_Release(); -} - -natsStatus -nats_postAsyncCbInfo(natsAsyncCbInfo *info) -{ - natsMutex_Lock(gLib.asyncCbs.lock); - - if (gLib.asyncCbs.shutdown) - { - natsMutex_Unlock(gLib.asyncCbs.lock); - return NATS_NOT_INITIALIZED; - } - - info->next = NULL; - - if (gLib.asyncCbs.head == NULL) - gLib.asyncCbs.head = info; - - if (gLib.asyncCbs.tail != NULL) - gLib.asyncCbs.tail->next = info; - - gLib.asyncCbs.tail = info; - - natsCondition_Signal(gLib.asyncCbs.cond); - - natsMutex_Unlock(gLib.asyncCbs.lock); - - return NATS_OK; -} - -static void -_garbageCollector(void *closure) -{ - natsGCList *gc = &(gLib.gc); - natsGCItem *item; - natsGCItem *list; - - WAIT_LIB_INITIALIZED; - - natsMutex_Lock(gc->lock); - - // Process all elements in the list, even on shutdown - while (true) - { - // Go into wait until we are notified to shutdown - // or there is something to garbage collect - gc->inWait = true; - - while (!(gc->shutdown) && (gc->head == NULL)) - { - natsCondition_Wait(gc->cond, gc->lock); - } - - // Out of the wait. Setting this boolean avoids unnecessary - // signaling when an item is added to the collector. - gc->inWait = false; - - // Do not break out on shutdown here, we want to clear the list, - // even on exit so that valgrind and the like are happy. - - // Under the lock, we will switch to a local list and reset the - // GC's list (so that others can add to the list without contention - // (at least from the GC itself). - do - { - list = gc->head; - gc->head = NULL; - - natsMutex_Unlock(gc->lock); - - // Now that we are outside of the lock, we can empty the list. - while ((item = list) != NULL) - { - // Pops item from the beginning of the list. - list = item->next; - item->next = NULL; - - // Invoke the freeCb associated with this object - (*(item->freeCb))((void*) item); - } - - natsMutex_Lock(gc->lock); - } - while (gc->head != NULL); - - // If we were ask to shutdown and since the list is now empty, exit - if (gc->shutdown) - break; - } - - natsMutex_Unlock(gc->lock); - - natsLib_Release(); -} - -bool -natsGC_collect(natsGCItem *item) -{ - natsGCList *gc; - bool signal; - - // If the object was not setup for garbage collection, return false - // so the caller frees the object. - if (item->freeCb == NULL) - return false; - - gc = &(gLib.gc); - - natsMutex_Lock(gc->lock); - - // We will signal only if the GC is in the condition wait. - signal = gc->inWait; - - // Add to the front of the list. - item->next = gc->head; - - // Update head. - gc->head = item; - - if (signal) - natsCondition_Signal(gc->cond); - - natsMutex_Unlock(gc->lock); - - return true; -} - -static void -_libTearDown(void) -{ - int i; - - for (i=0; ithread != NULL) - natsThread_Join(worker->thread); - } - - if (gLib.timers.thread != NULL) - natsThread_Join(gLib.timers.thread); - - if (gLib.asyncCbs.thread != NULL) - natsThread_Join(gLib.asyncCbs.thread); - - if (gLib.gc.thread != NULL) - natsThread_Join(gLib.gc.thread); - - natsLib_Release(); -} - -natsStatus -nats_Open(int64_t lockSpinCount) -{ - natsStatus s = NATS_OK; - - if (!nats_InitOnce(&gInitOnce, _doInitOnce)) - return NATS_FAILED_TO_INITIALIZE; - - natsMutex_Lock(gLib.lock); - - if (gLib.closed || gLib.initialized || gLib.initializing) - { - if (gLib.closed) - s = NATS_FAILED_TO_INITIALIZE; - else if (gLib.initializing) - s = NATS_ILLEGAL_STATE; - - natsMutex_Unlock(gLib.lock); - return s; - } - - gLib.initializing = true; - gLib.initAborted = false; - -#if !defined(_WIN32) - signal(SIGPIPE, SIG_IGN); -#endif - - srand((unsigned int) nats_NowInNanoSeconds()); - - gLib.refs = 1; - - // If the caller specifies negative value, then we use the default - if (lockSpinCount >= 0) - gLockSpinCount = lockSpinCount; - - nats_Base32_Init(); - - s = natsCondition_Create(&(gLib.cond)); - - if (s == NATS_OK) - s = natsCrypto_Init(); - - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.timers.lock)); - if (s == NATS_OK) - s = natsCondition_Create(&(gLib.timers.cond)); - if (s == NATS_OK) - { - s = natsThread_Create(&(gLib.timers.thread), _timerThread, NULL); - if (s == NATS_OK) - gLib.refs++; - } - - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.asyncCbs.lock)); - if (s == NATS_OK) - s = natsCondition_Create(&(gLib.asyncCbs.cond)); - if (s == NATS_OK) - { - s = natsThread_Create(&(gLib.asyncCbs.thread), _asyncCbsThread, NULL); - if (s == NATS_OK) - gLib.refs++; - } - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.gc.lock)); - if (s == NATS_OK) - s = natsCondition_Create(&(gLib.gc.cond)); - if (s == NATS_OK) - { - s = natsThread_Create(&(gLib.gc.thread), _garbageCollector, NULL); - if (s == NATS_OK) - gLib.refs++; - } - if (s == NATS_OK) - s = natsNUID_init(); - - if (s == NATS_OK) - s = natsMutex_Create(&(gLib.dlvWorkers.lock)); - if (s == NATS_OK) - { - char *defaultWriteDeadlineStr = getenv("NATS_DEFAULT_LIB_WRITE_DEADLINE"); - - if (defaultWriteDeadlineStr != NULL) - gLib.libDefaultWriteDeadline = (int64_t) atol(defaultWriteDeadlineStr); - - gLib.libHandlingMsgDeliveryByDefault = (getenv("NATS_DEFAULT_TO_LIB_MSG_DELIVERY") != NULL ? true : false); - gLib.dlvWorkers.maxSize = 1; - gLib.dlvWorkers.workers = NATS_CALLOC(gLib.dlvWorkers.maxSize, sizeof(natsMsgDlvWorker*)); - if (gLib.dlvWorkers.workers == NULL) - s = NATS_NO_MEMORY; - } - if (s == NATS_OK) - s = natsMutex_Create(&gLib.service_callback_mu); - if (s == NATS_OK) - s = natsHash_Create(&gLib.all_services_to_callback, 8); - - if (s == NATS_OK) - gLib.initialized = true; - - // In case of success or error, broadcast so that lib's threads - // can proceed. - if (gLib.cond != NULL) - { - if (s != NATS_OK) - { - gLib.initAborted = true; - gLib.timers.shutdown = true; - gLib.asyncCbs.shutdown = true; - gLib.gc.shutdown = true; - } - natsCondition_Broadcast(gLib.cond); - } - - gLib.initializing = false; - gLib.wasOpenedOnce = true; - - natsMutex_Unlock(gLib.lock); - - if (s != NATS_OK) - _libTearDown(); - - return s; -} - -natsStatus -natsInbox_Create(natsInbox **newInbox) -{ - natsStatus s; - char *inbox = NULL; - const int size = NATS_DEFAULT_INBOX_PRE_LEN + NUID_BUFFER_LEN + 1; - - s = nats_Open(-1); - if (s != NATS_OK) - return s; - - inbox = NATS_MALLOC(size); - if (inbox == NULL) - return nats_setDefaultError(NATS_NO_MEMORY); - - memcpy(inbox, NATS_DEFAULT_INBOX_PRE, NATS_DEFAULT_INBOX_PRE_LEN); - s = natsNUID_Next(inbox + NATS_DEFAULT_INBOX_PRE_LEN, NUID_BUFFER_LEN + 1); - if (s == NATS_OK) - { - inbox[size-1] = '\0'; - *newInbox = (natsInbox*) inbox; - } - else - NATS_FREE(inbox); - return NATS_UPDATE_ERR_STACK(s); -} - -void -natsInbox_Destroy(natsInbox *inbox) -{ - if (inbox == NULL) - return; - - NATS_FREE(inbox); -} - - -static natsStatus -_close(bool wait, int64_t timeout) -{ - natsStatus s = NATS_OK; - natsCondition *cond = NULL; - bool complete = false; - int i; - - // This is to protect against a call to nats_Close() while there - // was no prior call to nats_Open(), either directly or indirectly. - if (!nats_InitOnce(&gInitOnce, _doInitOnce)) - return NATS_ERR; - - natsMutex_Lock(gLib.lock); - - if (gLib.closed || !gLib.initialized) - { - bool closed = gLib.closed; - - natsMutex_Unlock(gLib.lock); - - if (closed) - return NATS_ILLEGAL_STATE; - return NATS_NOT_INITIALIZED; - } - if (wait) - { - if (natsThreadLocal_Get(gLib.natsThreadKey) != NULL) - s = NATS_ILLEGAL_STATE; - if (s == NATS_OK) - s = natsCondition_Create(&cond); - if (s != NATS_OK) - { - natsMutex_Unlock(gLib.lock); - return s; - } - gLib.closeCompleteCond = cond; - gLib.closeCompleteBool = &complete; - gLib.closeCompleteSignal = true; - } - - gLib.closed = true; - - natsMutex_Lock(gLib.timers.lock); - gLib.timers.shutdown = true; - natsCondition_Signal(gLib.timers.cond); - natsMutex_Unlock(gLib.timers.lock); - - natsMutex_Lock(gLib.asyncCbs.lock); - gLib.asyncCbs.shutdown = true; - natsCondition_Signal(gLib.asyncCbs.cond); - natsMutex_Unlock(gLib.asyncCbs.lock); - - natsMutex_Lock(gLib.gc.lock); - gLib.gc.shutdown = true; - natsCondition_Signal(gLib.gc.cond); - natsMutex_Unlock(gLib.gc.lock); - - natsMutex_Lock(gLib.dlvWorkers.lock); - for (i=0; ilock); - worker->shutdown = true; - natsCondition_Signal(worker->cond); - natsMutex_Unlock(worker->lock); - } - natsMutex_Unlock(gLib.dlvWorkers.lock); - - natsMutex_Unlock(gLib.lock); - - nats_ReleaseThreadMemory(); - _libTearDown(); - - if (wait) - { - natsMutex_Lock(gLib.lock); - while ((s != NATS_TIMEOUT) && !complete) - { - if (timeout <= 0) - natsCondition_Wait(cond, gLib.lock); - else - s = natsCondition_TimedWait(cond, gLib.lock, timeout); - } - if (s != NATS_OK) - gLib.closeCompleteSignal = false; - natsMutex_Unlock(gLib.lock); - - natsCondition_Destroy(cond); - } - - return s; -} - -void -nats_Close(void) -{ - _close(false, true); -} - -natsStatus -nats_CloseAndWait(int64_t timeout) -{ - return _close(true, timeout); -} - -const char* -nats_GetVersion(void) -{ - return LIB_NATS_VERSION_STRING; -} - -uint32_t -nats_GetVersionNumber(void) -{ - return LIB_NATS_VERSION_NUMBER; -} - -static void -_versionGetString(char *buffer, size_t bufLen, uint32_t verNumber) -{ - snprintf(buffer, bufLen, "%u.%u.%u", - ((verNumber >> 16) & 0xF), - ((verNumber >> 8) & 0xF), - (verNumber & 0xF)); -} - -bool -nats_CheckCompatibilityImpl(uint32_t headerReqVerNumber, uint32_t headerVerNumber, - const char *headerVerString) -{ - if ((headerVerNumber < LIB_NATS_VERSION_REQUIRED_NUMBER) - || (headerReqVerNumber > LIB_NATS_VERSION_NUMBER)) - { - char reqVerString[10]; - char libReqVerString[10]; - - _versionGetString(reqVerString, sizeof(reqVerString), headerReqVerNumber); - _versionGetString(libReqVerString, sizeof(libReqVerString), NATS_VERSION_REQUIRED_NUMBER); - - printf("Incompatible versions:\n" \ - "Header : %s (requires %s)\n" \ - "Library: %s (requires %s)\n", - headerVerString, reqVerString, - NATS_VERSION_STRING, libReqVerString); - exit(1); - } - - return true; -} - -static natsTLError* -_getTLError(void) -{ - natsTLError *errTL = NULL; - bool needFree = false; - - // The library should already be initialized, but let's protect against - // situations where foo() invokes bar(), which invokes baz(), which - // invokes nats_Open(). If that last call fails, when we un-wind down - // to foo(), it may be difficult to know that nats_Open() failed and - // that we should not try to invoke natsLib_setError. So we check again - // here that the library has been initialized properly, and if not, we - // simply don't set the error. - if (nats_Open(-1) != NATS_OK) - return NULL; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if (errTL == NULL) - { - errTL = (natsTLError*) NATS_CALLOC(1, sizeof(natsTLError)); - if (errTL != NULL) - errTL->framesCount = -1; - needFree = (errTL != NULL); - - } - - if ((errTL != NULL) - && (natsThreadLocal_SetEx(gLib.errTLKey, - (const void*) errTL, false) != NATS_OK)) - { - if (needFree) - NATS_FREE(errTL); - - errTL = NULL; - } - - return errTL; -} - -static char* -_getErrorShortFileName(const char* fileName) -{ - char *file = strstr(fileName, "src"); - - if (file != NULL) - file = (file + 4); - else - file = (char*) fileName; - - return file; -} - -static void -_updateStack(natsTLError *errTL, const char *funcName, natsStatus errSts, - bool calledFromSetError) -{ - int idx; - - idx = errTL->framesCount; - if ((idx >= 0) - && (idx < MAX_FRAMES) - && (strcmp(errTL->func[idx], funcName) == 0)) - { - return; - } - - // In case no error was already set... - if ((errTL->framesCount == -1) && !calledFromSetError) - errTL->sts = errSts; - - idx = ++(errTL->framesCount); - - if (idx >= MAX_FRAMES) - return; - - errTL->func[idx] = funcName; -} - -#if !defined(_WIN32) -__attribute__ ((format (printf, 5, 6))) -#endif -natsStatus -nats_setErrorReal(const char *fileName, const char *funcName, int line, natsStatus errSts, const char *errTxtFmt, ...) -{ - natsTLError *errTL = _getTLError(); - char tmp[256]; - va_list ap; - int n; - - if ((errTL == NULL) || errTL->skipUpdate) - return errSts; - - errTL->sts = errSts; - errTL->framesCount = -1; - - tmp[0] = '\0'; - - va_start(ap, errTxtFmt); - nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); - va_end(ap); +#include "conn.h" - if (strlen(tmp) > 0) +#if defined(_WIN32) && _WIN32 +#ifndef NATS_STATIC +BOOL WINAPI DllMain(HINSTANCE hinstDLL, // DLL module handle + DWORD fdwReason, // reason called + LPVOID lpvReserved) // reserved +{ + switch (fdwReason) { - n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", - _getErrorShortFileName(fileName), line, tmp); - if ((n < 0) || (n >= (int) sizeof(errTL->text))) + // For applications linking dynamically NATS library, + // release thread-local memory for user-created threads. + // For portable applications, the user should manually call + // nats_ReleaseThreadMemory() before the thread returns so + // that no memory is leaked regardless if they link statically + // or dynamically. It is safe to call nats_ReleaseThreadMemory() + // twice for the same threads. + case DLL_THREAD_DETACH: { - int pos = ((int) strlen(errTL->text)) - 1; - int i; - - for (i=0; i<3; i++) - errTL->text[pos--] = '.'; + nats_ReleaseThreadMemory(); + break; } + default: + break; } - _updateStack(errTL, funcName, errSts, true); - - return errSts; + return TRUE; + UNREFERENCED_PARAMETER(hinstDLL); + UNREFERENCED_PARAMETER(lpvReserved); } - -#if !defined(_WIN32) -__attribute__ ((format (printf, 4, 5))) #endif -void -nats_updateErrTxt(const char *fileName, const char *funcName, int line, const char *errTxtFmt, ...) -{ - natsTLError *errTL = _getTLError(); - char tmp[256]; - va_list ap; - int n; - - if ((errTL == NULL) || errTL->skipUpdate) - return; - - tmp[0] = '\0'; - - va_start(ap, errTxtFmt); - nats_vsnprintf(tmp, sizeof(tmp), errTxtFmt, ap); - va_end(ap); - - if (strlen(tmp) > 0) - { - n = snprintf(errTL->text, sizeof(errTL->text), "(%s:%d): %s", - _getErrorShortFileName(fileName), line, tmp); - if ((n < 0) || (n >= (int) sizeof(errTL->text))) - { - int pos = ((int) strlen(errTL->text)) - 1; - int i; +#endif - for (i=0; i<3; i++) - errTL->text[pos--] = '.'; - } - } +static void _overwriteInt64(const char *envVar, int64_t *val) +{ + char *str = getenv(envVar); + if (str != NULL) + *val = atoll(str); } -void -nats_setErrStatusAndTxt(natsStatus err, const char *errTxt) +static void _overwriteInt(const char *envVar, int *val) { - natsTLError *errTL = _getTLError(); - - if ((errTL == NULL) || errTL->skipUpdate) - return; - - errTL->sts = err; - snprintf(errTL->text, sizeof(errTL->text), "%s", errTxt); - errTL->framesCount = -1; + char *str = getenv(envVar); + if (str != NULL) + *val = atoi(str); } -natsStatus -nats_updateErrStack(natsStatus err, const char *func) +static void _overwriteBool(const char *envVar, bool *val) { - natsTLError *errTL = _getTLError(); - - if ((errTL == NULL) || errTL->skipUpdate) - return err; - - _updateStack(errTL, func, err, false); - - return err; + char *str = getenv(envVar); + if (str != NULL) + *val = (strcasecmp(str, "true") == 0) || + (strcasecmp(str, "on") == 0) || + (atoi(str) != 0); } -void -nats_clearLastError(void) +static void _overrideWithEnv(natsClientConfig *config) { - natsTLError *errTL = _getTLError(); - - if ((errTL == NULL) || errTL->skipUpdate) - return; - - errTL->sts = NATS_OK; - errTL->text[0] = '\0'; - errTL->framesCount = -1; + _overwriteInt64("NATS_DEFAULT_LIB_WRITE_DEADLINE", &config->DefaultWriteDeadline); + _overwriteBool("NATS_USE_THREAD_POOL", &config->DefaultToThreadPool); + _overwriteInt("NATS_THREAD_POOL_MAX", &config->ThreadPoolMax); + _overwriteBool("NATS_USE_THREAD_POOL_FOR_REPLIES", &config->DefaultRepliesToThreadPool); + _overwriteInt("NATS_REPLY_THREAD_POOL_MAX", &config->ReplyThreadPoolMax); } -void -nats_doNotUpdateErrStack(bool skipStackUpdate) +// environment variables will override the default options. +natsStatus +nats_OpenWithConfig(natsClientConfig *config) { - natsTLError *errTL = _getTLError(); - - if (errTL == NULL) - return; - - if (skipStackUpdate) - { - errTL->skipUpdate++; - } - else - { - errTL->skipUpdate--; - assert(errTL->skipUpdate >= 0); - } + return nats_openLib(config); } -const char* -nats_GetLastError(natsStatus *status) +natsStatus +nats_Open(int64_t lockSpinCount) { - natsStatus s; - natsTLError *errTL = NULL; - - if (status != NULL) - *status = NATS_OK; - - // Ensure the library is loaded - s = nats_Open(-1); - if (s != NATS_OK) - return NULL; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if ((errTL == NULL) || (errTL->sts == NATS_OK)) - return NULL; - - if (status != NULL) - *status = errTL->sts; + bool defaultToSharedDispatchers = (getenv("NATS_DEFAULT_TO_LIB_MSG_DELIVERY") != NULL ? true : false); + + natsClientConfig config = { + .LockSpinCount = lockSpinCount, + .DefaultToThreadPool = defaultToSharedDispatchers, + .ThreadPoolMax = 1, + .DefaultRepliesToThreadPool = false, + .ReplyThreadPoolMax = 0, + }; - return errTL->text; + return nats_openLib(&config); } natsStatus -nats_GetLastErrorStack(char *buffer, size_t bufLen) +natsInbox_Create(natsInbox **newInbox) { - natsTLError *errTL = NULL; - int offset = 0; - int i, max, n, len; - - if ((buffer == NULL) || (bufLen == 0)) - return NATS_INVALID_ARG; - - buffer[0] = '\0'; - len = (int) bufLen; - - // Ensure the library is loaded - if (nats_Open(-1) != NATS_OK) - return NATS_FAILED_TO_INITIALIZE; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) - return NATS_OK; + natsStatus s; + char *inbox = NULL; + const int size = NATS_DEFAULT_INBOX_PRE_LEN + NUID_BUFFER_LEN + 1; - max = errTL->framesCount; - if (max >= MAX_FRAMES) - max = MAX_FRAMES - 1; + s = nats_Open(-1); + if (s != NATS_OK) + return s; - for (i=0; (i<=max) && (len > 0); i++) - { - n = snprintf(buffer + offset, len, "%s%s", - errTL->func[i], - (i < max ? "\n" : "")); - // On Windows, n will be < 0 if len is not big enough. - if (n < 0) - { - len = 0; - } - else - { - offset += n; - len -= n; - } - } + inbox = NATS_MALLOC(size); + if (inbox == NULL) + return nats_setDefaultError(NATS_NO_MEMORY); - if ((max != errTL->framesCount) && (len > 0)) + memcpy(inbox, NATS_DEFAULT_INBOX_PRE, NATS_DEFAULT_INBOX_PRE_LEN); + s = natsNUID_Next(inbox + NATS_DEFAULT_INBOX_PRE_LEN, NUID_BUFFER_LEN + 1); + if (s == NATS_OK) { - n = snprintf(buffer + offset, len, "\n%d more...", - errTL->framesCount - max); - // On Windows, n will be < 0 if len is not big enough. - if (n < 0) - len = 0; - else - len -= n; + inbox[size-1] = '\0'; + *newInbox = (natsInbox*) inbox; } - - if (len <= 0) - return NATS_INSUFFICIENT_BUFFER; - - return NATS_OK; + else + NATS_FREE(inbox); + return NATS_UPDATE_ERR_STACK(s); } void -nats_PrintLastErrorStack(FILE *file) +natsInbox_Destroy(natsInbox *inbox) { - natsTLError *errTL = NULL; - int i, max; - - // Ensure the library is loaded - if (nats_Open(-1) != NATS_OK) - return; - - errTL = natsThreadLocal_Get(gLib.errTLKey); - if ((errTL == NULL) || (errTL->sts == NATS_OK) || (errTL->framesCount == -1)) + if (inbox == NULL) return; - fprintf(file, "Error: %u - %s", - errTL->sts, natsStatus_GetText(errTL->sts)); - if (errTL->text[0] != '\0') - fprintf(file, " - %s", errTL->text); - fprintf(file, "\n"); - fprintf(file, "Stack: (library version: %s)\n", nats_GetVersion()); - - max = errTL->framesCount; - if (max >= MAX_FRAMES) - max = MAX_FRAMES - 1; - - for (i=0; i<=max; i++) - fprintf(file, " %02d - %s\n", (i+1), errTL->func[i]); - - if (max != errTL->framesCount) - fprintf(file, " %d more...\n", errTL->framesCount - max); - - fflush(file); + NATS_FREE(inbox); } void -nats_sslRegisterThreadForCleanup(void) +nats_Close(void) { -#if defined(NATS_HAS_TLS) - // Set anything. The goal is that at thread exit, the thread local key - // will have something non NULL associated, which will trigger the - // destructor that we have registered. - (void) natsThreadLocal_Set(gLib.sslTLKey, (void*) 1); -#endif + nats_closeLib(false, true); } natsStatus -nats_sslInit(void) +nats_CloseAndWait(int64_t timeout) { - natsStatus s = NATS_OK; + return nats_closeLib(true, timeout); +} - // Ensure the library is loaded - s = nats_Open(-1); - if (s != NATS_OK) - return s; +const char* +nats_GetVersion(void) +{ + return LIB_NATS_VERSION_STRING; +} + +uint32_t +nats_GetVersionNumber(void) +{ + return LIB_NATS_VERSION_NUMBER; +} - natsMutex_Lock(gLib.lock); +static void +_versionGetString(char *buffer, size_t bufLen, uint32_t verNumber) +{ + snprintf(buffer, bufLen, "%u.%u.%u", + ((verNumber >> 16) & 0xF), + ((verNumber >> 8) & 0xF), + (verNumber & 0xF)); +} - if (!(gLib.sslInitialized)) +bool +nats_CheckCompatibilityImpl(uint32_t headerReqVerNumber, uint32_t headerVerNumber, + const char *headerVerString) +{ + if ((headerVerNumber < LIB_NATS_VERSION_REQUIRED_NUMBER) + || (headerReqVerNumber > LIB_NATS_VERSION_NUMBER)) { - // Regardless of success, mark as initialized so that we - // can do cleanup on exit. - gLib.sslInitialized = true; + char reqVerString[10]; + char libReqVerString[10]; -#if defined(NATS_HAS_TLS) -#if !defined(NATS_USE_OPENSSL_1_1) - // Initialize SSL. - SSL_library_init(); - SSL_load_error_strings(); -#endif -#endif - s = natsThreadLocal_CreateKey(&(gLib.sslTLKey), _cleanupThreadSSL); - } + _versionGetString(reqVerString, sizeof(reqVerString), headerReqVerNumber); + _versionGetString(libReqVerString, sizeof(libReqVerString), NATS_VERSION_REQUIRED_NUMBER); - natsMutex_Unlock(gLib.lock); + printf("Incompatible versions:\n" \ + "Header : %s (requires %s)\n" \ + "Library: %s (requires %s)\n", + headerVerString, reqVerString, + NATS_VERSION_STRING, libReqVerString); + exit(1); + } - return NATS_UPDATE_ERR_STACK(s); + return true; } -static void -_deliverMsgs(void *arg) -{ - natsMsgDlvWorker *dlv = (natsMsgDlvWorker*) arg; - natsConnection *nc; - natsSubscription *sub; - natsMsgHandler mcb; - void *mcbClosure; - uint64_t delivered; - uint64_t max; - natsMsg *msg; - bool timerNeedReset = false; - jsSub *jsi; - char *fcReply; - - natsMutex_Lock(dlv->lock); +void +nats_deliverMsgsPoolf(void *arg) +{ + natsDispatcher *d = (natsDispatcher *)arg; + natsConnection *nc; + natsSubscription *sub; + natsMsgHandler mcb; + void *mcbClosure; + uint64_t delivered; + uint64_t max; + natsMsg *msg; + bool timerNeedReset = false; + jsSub *jsi; + char *fcReply; + + natsMutex_Lock(d->mu); while (true) { - while (((msg = dlv->msgList.head) == NULL) && !dlv->shutdown) - natsCondition_Wait(dlv->cond, dlv->lock); + while (((msg = d->queue.head) == NULL) && !d->shutdown) + natsCondition_Wait(d->cond, d->mu); // Break out only when list is empty - if ((msg == NULL) && dlv->shutdown) + if ((msg == NULL) && d->shutdown) { break; } // Remove message from list now... - dlv->msgList.head = msg->next; - if (dlv->msgList.tail == msg) - dlv->msgList.tail = NULL; + d->queue.head = msg->next; + if (d->queue.tail == msg) + d->queue.tail = NULL; msg->next = NULL; // Get subscription reference from message @@ -1704,16 +246,12 @@ _deliverMsgs(void *arg) // Is this a control message? if (msg->subject[0] == '\0') { - bool closed = sub->closed; - bool timedOut = sub->timedOut; - bool draining = sub->libDlvDraining; - - // Switch off this flag. - if (draining) - sub->libDlvDraining = false; + bool closed = (msg == sub->control->sub.close); + bool timedOut = (msg == sub->control->sub.timeout); + bool draining = (msg == sub->control->sub.drain); // We need to release this lock... - natsMutex_Unlock(dlv->lock); + natsMutex_Unlock(d->mu); // Release the message natsMsg_Destroy(msg); @@ -1728,15 +266,15 @@ _deliverMsgs(void *arg) } else if (closed) { - natsOnCompleteCB cb = NULL; - void *closure = NULL; + natsOnCompleteCB cb = NULL; + void *closure = NULL; // Call this in case the subscription was draining. natsSub_setDrainCompleteState(sub); // Check for completion callback natsSub_Lock(sub); - cb = sub->onCompleteCB; + cb = sub->onCompleteCB; closure = sub->onCompleteCBClosure; natsSub_Unlock(sub); @@ -1753,7 +291,7 @@ _deliverMsgs(void *arg) } // Grab the lock, we go back to beginning of loop. - natsMutex_Lock(dlv->lock); + natsMutex_Lock(d->mu); if (!draining && !closed && timedOut) { @@ -1768,9 +306,11 @@ _deliverMsgs(void *arg) continue; } - // Update before checking closed state. - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_dataAndHdrLen(msg); + // Update the sub's stats before checking closed state. (We now post + // control messages to the sub's queue, because hbTimer processing is + // expecting it, so need to clear the stats for them, too) + sub->ownDispatcher.queue.msgs--; + sub->ownDispatcher.queue.bytes -= natsMsg_dataAndHdrLen(msg); // Need to check for closed subscription again here. // The subscription could have been unsubscribed from a callback @@ -1796,15 +336,15 @@ _deliverMsgs(void *arg) // If we are dealing with the last pending message for this sub, // we will reset the timer after the user callback returns. - if (sub->msgList.msgs == 0) + if (sub->ownDispatcher.queue.msgs == 0) timerNeedReset = true; } - natsMutex_Unlock(dlv->lock); + natsMutex_Unlock(d->mu); if ((max == 0) || (delivered <= max)) { - (*mcb)(nc, sub, msg, mcbClosure); + (*mcb)(nc, sub, msg, mcbClosure); } else { @@ -1828,7 +368,7 @@ _deliverMsgs(void *arg) natsConn_removeSubscription(nc, sub); } - natsMutex_Lock(dlv->lock); + natsMutex_Lock(d->mu); // Check if timer need to be reset for subscriptions that can timeout. if (!sub->closed && (sub->timeout != 0) && timerNeedReset) @@ -1847,7 +387,7 @@ _deliverMsgs(void *arg) } } - natsMutex_Unlock(dlv->lock); + natsMutex_Unlock(d->mu); natsLib_Release(); } @@ -1856,194 +396,12 @@ natsStatus nats_SetMessageDeliveryPoolSize(int max) { natsStatus s = NATS_OK; - natsLibDlvWorkers *workers; // Ensure the library is loaded s = nats_Open(-1); if (s != NATS_OK) return s; - workers = &gLib.dlvWorkers; - - natsMutex_Lock(workers->lock); - - if (max <= 0) - { - natsMutex_Unlock(workers->lock); - return nats_setError(NATS_ERR, "%s", "Pool size cannot be negative or zero"); - } - - // Do not error on max < workers->maxSize in case we allow shrinking - // the pool in the future. - if (max > workers->maxSize) - { - natsMsgDlvWorker **newArray = NATS_CALLOC(max, sizeof(natsMsgDlvWorker*)); - if (newArray == NULL) - s = nats_setDefaultError(NATS_NO_MEMORY); - if (s == NATS_OK) - { - int i; - for (i=0; isize; i++) - newArray[i] = workers->workers[i]; - - NATS_FREE(workers->workers); - workers->workers = newArray; - workers->maxSize = max; - } - } - - natsMutex_Unlock(workers->lock); - - return NATS_UPDATE_ERR_STACK(s); -} - -// Post a control message to the worker's queue. -natsStatus -natsLib_msgDeliveryPostControlMsg(natsSubscription *sub) -{ - natsStatus s; - natsMsg *controlMsg = NULL; - natsMsgDlvWorker *worker = (sub->libDlvWorker); - - // Create a "end" message and post it to the delivery worker - s = natsMsg_create(&controlMsg, NULL, 0, NULL, 0, NULL, 0, -1); - if (s == NATS_OK) - { - nats_MsgList *l; - bool signal = false; - - natsMutex_Lock(worker->lock); - - controlMsg->sub = sub; - - l = &(worker->msgList); - if (l->head == NULL) - { - l->head = controlMsg; - signal = true; - } - else - l->tail->next = controlMsg; - l->tail = controlMsg; - - if (signal) - natsCondition_Signal(worker->cond); - - natsMutex_Unlock(worker->lock); - } - return NATS_UPDATE_ERR_STACK(s); -} - -natsStatus -natsLib_msgDeliveryAssignWorker(natsSubscription *sub) -{ - natsStatus s = NATS_OK; - natsLibDlvWorkers *workers = &(gLib.dlvWorkers); - natsMsgDlvWorker *worker = NULL; - - natsMutex_Lock(workers->lock); - - if (workers->maxSize == 0) - { - natsMutex_Unlock(workers->lock); - return nats_setError(NATS_FAILED_TO_INITIALIZE, "%s", "Message delivery thread pool size is 0!"); - } - - worker = workers->workers[workers->idx]; - if (worker == NULL) - { - worker = NATS_CALLOC(1, sizeof(natsMsgDlvWorker)); - if (worker == NULL) - s = nats_setDefaultError(NATS_NO_MEMORY); - if (s == NATS_OK) - s = natsMutex_Create(&worker->lock); - if (s == NATS_OK) - s = natsCondition_Create(&worker->cond); - if (s == NATS_OK) - { - natsLib_Retain(); - s = natsThread_Create(&worker->thread, _deliverMsgs, (void*) worker); - if (s != NATS_OK) - natsLib_Release(); - } - if (s == NATS_OK) - { - workers->workers[workers->idx] = worker; - workers->size++; - } - else - { - _freeDlvWorker(worker); - } - } - if (s == NATS_OK) - { - sub->libDlvWorker = worker; - if (++(workers->idx) == workers->maxSize) - workers->idx = 0; - } - - natsMutex_Unlock(workers->lock); - - return NATS_UPDATE_ERR_STACK(s); -} - -bool -natsLib_isLibHandlingMsgDeliveryByDefault(void) -{ - return gLib.libHandlingMsgDeliveryByDefault; -} - -int64_t -natsLib_defaultWriteDeadline(void) -{ - return gLib.libDefaultWriteDeadline; -} - -void -natsLib_getMsgDeliveryPoolInfo(int *maxSize, int *size, int *idx, natsMsgDlvWorker ***workersArray) -{ - natsLibDlvWorkers *workers = &gLib.dlvWorkers; - - natsMutex_Lock(workers->lock); - *maxSize = workers->maxSize; - *size = workers->size; - *idx = workers->idx; - *workersArray = workers->workers; - natsMutex_Unlock(workers->lock); -} - -natsStatus -natsLib_startServiceCallbacks(microService *m) -{ - natsStatus s; - - natsMutex_Lock(gLib.service_callback_mu); - s = natsHash_Set(gLib.all_services_to_callback, (int64_t)m, (void *)m, NULL); - natsMutex_Unlock(gLib.service_callback_mu); - + s = nats_setMessageDispatcherPoolCap(max); return NATS_UPDATE_ERR_STACK(s); } - -void -natsLib_stopServiceCallbacks(microService *m) -{ - if (m == NULL) - return; - - natsMutex_Lock(gLib.service_callback_mu); - natsHash_Remove(gLib.all_services_to_callback, (int64_t)m); - natsMutex_Unlock(gLib.service_callback_mu); -} - -natsMutex* -natsLib_getServiceCallbackMutex(void) -{ - return gLib.service_callback_mu; -} - -natsHash* -natsLib_getAllServicesToCallback(void) -{ - return gLib.all_services_to_callback; -} diff --git a/src/nats.h b/src/nats.h index dc331e08f..24e88c190 100644 --- a/src/nats.h +++ b/src/nats.h @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -186,6 +186,28 @@ typedef struct __natsOptions natsOptions; */ typedef char natsInbox; + +/** \brief An initial configuration for NATS client. Provides control over the + * threading model, and sets many default option values. + * + * @see natsOptions_Create + */ +typedef struct __natsClientConfig +{ + int64_t DefaultWriteDeadline; + + int64_t LockSpinCount; + + // Subscription message delivery thread control + bool DefaultToThreadPool; + int ThreadPoolMax; + + // Reply message delivery thread control + bool DefaultRepliesToThreadPool; + bool UseSeparatePoolForReplies; + int ReplyThreadPoolMax; +} natsClientConfig; + /** \brief A list of NATS messages. * * Used by some APIs which return a list of #natsMsg objects. @@ -1043,7 +1065,7 @@ typedef struct jsConsumerNamesList */ typedef struct jsConsumerPauseResponse { - bool Paused; + bool Paused; int64_t PauseUntil; ///< UTC time expressed as number of nanoseconds since epoch. int64_t PauseRemaining; ///< Remaining time in nanoseconds. } jsConsumerPauseResponse; @@ -1773,6 +1795,23 @@ typedef void (*stanConnectionLostHandler)( * @{ */ +/** \brief Initializes the library. + * + * This initializes the library, with more control over how threads and/or + * thread pools are used to deliver messages to the application. + * + * It is invoked automatically when creating a connection, with the default + * settings (same as nats_Open(-1)). + * + * \warning You must not call #nats_Open[WithConfig] and #nats_Close + * concurrently. + * + * @param config points to a natsClientConfig. A copy of the settings is made, + * so the config can be freed after initializing the NATS client. + */ +natsStatus +nats_OpenWithConfig(natsClientConfig *config); + /** \brief Initializes the library. * * This initializes the library. @@ -2327,6 +2366,23 @@ natsOptions_SetName(natsOptions *opts, const char *name); NATS_EXTERN natsStatus natsOptions_SetSecure(natsOptions *opts, bool secure); +/** \brief Performs TLS handshake first. + * + * If the server is not configured to require the client to perform + * the TLS handshake first, the server sends an INFO protocol first. + * When receiving it, the client and server are then initiate the + * TLS handshake. + * + * If the server is configured to require the client to perform + * the TLS handshake first, the client will fail to connect if + * not setting this option. Conversely, if the client is configured + * with this option but the server is not, the connection will fail. + * + * @param opts the pointer to the #natsOptions object. + */ +NATS_EXTERN natsStatus +natsOptions_TLSHandshakeFirst(natsOptions *opts); + /** \brief Loads the trusted CA certificates from a file. * * Loads the trusted CA certificates from a file. @@ -4039,7 +4095,7 @@ natsConnection_Connect(natsConnection **nc, natsOptions *options); * This means that all subscriptions and consumers should be resubscribed and * their work resumed after successful reconnect where all reconnect options are * respected. - * + * * @param nc the pointer to the #natsConnection object. */ natsStatus diff --git a/src/natsp.h b/src/natsp.h index c749b84bb..84bf00612 100644 --- a/src/natsp.h +++ b/src/natsp.h @@ -1,4 +1,4 @@ -// Copyright 2015-2022 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -41,6 +41,7 @@ #include "url.h" #include "srvpool.h" #include "msg.h" +#include "dispatch.h" #include "asynccb.h" #include "hash.h" #include "stats.h" @@ -224,6 +225,7 @@ struct __natsOptions bool pedantic; bool allowReconnect; bool secure; + bool tlsHandshakeFirst; int ioBufSize; int maxReconnect; int64_t reconnectWait; @@ -272,7 +274,16 @@ struct __natsOptions void *evLoop; natsEvLoopCallbacks evCbs; - bool libMsgDelivery; + // If set to false, the client will start a per-subscription dedicated + // thread to deliver messages to the user callbacks. If true, a shared + // thread out of a thread pool is used. natsClientConfig controls the pool + // size. + bool useSharedDispatcher; + + // If set to false, the client will start a per-connection dedicated thread + // to deliver reply messages to the user callbacks. If true, a shared thread + // out of a thread pool is used. natsClientConfig controls the pool size. + bool useSharedReplyDispatcher; int orderIP; // possible values: 0,4,6,46,64 @@ -333,26 +344,6 @@ struct __natsOptions // Custom message payload padding size int payloadPaddingSize; }; - -typedef struct __nats_MsgList -{ - natsMsg *head; - natsMsg *tail; - int msgs; - int bytes; - -} nats_MsgList; - -typedef struct __natsMsgDlvWorker -{ - natsMutex *lock; - natsCondition *cond; - natsThread *thread; - bool shutdown; - nats_MsgList msgList; - -} natsMsgDlvWorker; - typedef struct __pmInfo { char *subject; @@ -406,7 +397,6 @@ typedef struct __jsSub int64_t hbi; bool active; natsTimer *hbTimer; - natsMsg *mhMsg; char *cmeta; uint64_t sseq; @@ -496,6 +486,21 @@ struct __kvWatcher }; +typedef struct __natsSubscriptionControlMessages +{ + struct + { + natsMsg *timeout; + natsMsg *close; + natsMsg *drain; + } sub; + struct + { + natsMsg *expired; + natsMsg *missedHeartbeat; + } batch; +} natsSubscriptionControlMessages; + struct __natsSubscription { natsMutex *mu; @@ -505,22 +510,28 @@ struct __natsSubscription // This is non-zero when auto-unsubscribe is used. uint64_t max; - // This is updated in the delivery thread (or NextMsg) and indicates - // how many message have been presented to the callback (or returned - // from NextMsg). Like 'msgs', this is also used to determine if we - // have reached the max number of messages. + // We always have a dispatcher to keep track of things, even if the + // subscription is sync. The dispatcher is set up at the subscription + // creation time, and may point to a dedicated thread using sub's own + // dispatchQueue, or a shared worker using its own dispatch queue, which + // dispatcher->queue then points to. + natsDispatcher *dispatcher; + natsDispatcher ownDispatcher; + + // These are a signals to the sub's async dispatcher thread that something + // happened - draining or closing the subscription, or some sort of a + // timeout. Since these are optional, we only allocate them when starting an + // async dispatcher. + natsSubscriptionControlMessages *control; + + // This is updated in the delivery thread (or NextMsg) and indicates how + // many message have been presented to the callback (or returned from + // NextMsg). Together with the messages pending dispatch in + // dispatch->queue, this is also used to determine if we have reached the + // max number of messages. uint64_t delivered; - - // The list of messages waiting to be delivered to the callback (or - // returned from NextMsg). - nats_MsgList msgList; - - // True if msgList.count is over pendingMax + // True if ownDispatcher.queue.msgs is over pendingMax bool slowConsumer; - - // Condition variable used to wait for message delivery. - natsCondition *cond; - // The subscriber is closed (or closing). bool closed; @@ -536,11 +547,7 @@ struct __natsSubscription int64_t drainTimeout; // This is set if the flush failed and will prevent the connection for pushing further messages. bool drainSkip; - - // Same than draining but for the global delivery situation. - // This boolean will be switched off when processed, as opposed - // to draining that once set does not get reset. - bool libDlvDraining; + natsCondition *drainCond; // If true, the subscription is closed, but because the connection // was closed, not because of subscription (auto-)unsubscribe. @@ -561,13 +568,6 @@ struct __natsSubscription // Reference to the connection that created this subscription. struct __natsConnection *conn; - // Delivery thread (for async subscription). - natsThread *deliverMsgsThread; - - // If message delivery is done by the library instead, this is the - // reference to the worker handling this subscription. - natsMsgDlvWorker *libDlvWorker; - // Message callback and closure (for async subscription). natsMsgHandler msgCb; void *msgCbClosure; @@ -590,7 +590,6 @@ struct __natsSubscription // For JetStream jsSub *jsi; - }; typedef struct __natsPong @@ -714,7 +713,7 @@ struct __natsConnection // New Request style char respId[NATS_MAX_REQ_ID_LEN+1]; int respIdPos; - int respIdVal; + char respIdVal; char *respSub; // The wildcard subject natsSubscription *respMux; // A single response subscription natsStrHash *respMap; // Request map for the response msg @@ -750,62 +749,9 @@ struct __natsConnection } srvVersion; }; -// -// Library -// - -void -natsSys_Init(void); - -void -natsLib_Retain(void); - -void -natsLib_Release(void); - -int64_t -nats_setTargetTime(int64_t timeout); - -void -nats_resetTimer(natsTimer *t, int64_t newInterval); - -void -nats_stopTimer(natsTimer *t); - -// Returns the number of timers that have been created and not stopped. -int -nats_getTimersCount(void); - -// Returns the number of timers actually in the list. This should be -// equal to nats_getTimersCount() or nats_getTimersCount() - 1 when a -// timer thread is invoking a timer's callback. -int -nats_getTimersCountInList(void); - -natsStatus -nats_postAsyncCbInfo(natsAsyncCbInfo *info); - void nats_sslRegisterThreadForCleanup(void); -natsStatus -nats_sslInit(void); - -natsStatus -natsLib_msgDeliveryPostControlMsg(natsSubscription *sub); - -natsStatus -natsLib_msgDeliveryAssignWorker(natsSubscription *sub); - -bool -natsLib_isLibHandlingMsgDeliveryByDefault(void); - -int64_t -natsLib_defaultWriteDeadline(void); - -void -natsLib_getMsgDeliveryPoolInfo(int *maxSize, int *size, int *idx, natsMsgDlvWorker ***workersArray); - void nats_setNATSThreadKey(void); @@ -938,4 +884,18 @@ jsSub_resetOrderedConsumer(natsSubscription *sub, uint64_t sseq); bool natsMsg_isJSCtrl(natsMsg *msg, int *ctrlType); +static inline void nats_lockDispatcher(natsDispatcher *d) +{ + if (d->mu != NULL) + natsMutex_Lock(d->mu); +} + +static inline void nats_unlockDispatcher(natsDispatcher *d) +{ + if (d->mu != NULL) + natsMutex_Unlock(d->mu); +} + +void nats_deliverMsgsPoolf(void *arg); + #endif /* NATSP_H_ */ diff --git a/src/natstime.c b/src/natstime.c index 8e87d7568..9712d9c8f 100644 --- a/src/natstime.c +++ b/src/natstime.c @@ -85,3 +85,12 @@ natsDeadline_GetTimeout(natsDeadline *deadline) return timeout; } + +int64_t +nats_setTargetTime(int64_t timeout) +{ + int64_t target = nats_Now() + timeout; + if (target < 0) + target = 0x7FFFFFFFFFFFFFFF; + return target; +} diff --git a/src/natstime.h b/src/natstime.h index b7041d2ea..66da73b93 100644 --- a/src/natstime.h +++ b/src/natstime.h @@ -33,5 +33,7 @@ natsDeadline_GetTimeout(natsDeadline *deadline); void natsDeadline_Clear(natsDeadline *deadline); +int64_t +nats_setTargetTime(int64_t timeout); #endif /* NATSTIME_H_ */ diff --git a/src/opts.c b/src/opts.c index 1c7864635..32776afb8 100644 --- a/src/opts.c +++ b/src/opts.c @@ -1,4 +1,4 @@ -// Copyright 2015-2021 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -19,6 +19,7 @@ #include "opts.h" #include "util.h" #include "conn.h" +#include "glib/glib.h" natsStatus natsOptions_SetURL(natsOptions *opts, const char* url) @@ -306,7 +307,7 @@ _getSSLCtx(natsOptions *opts) { natsStatus s; - s = nats_sslInit(); + s = nats_initSSL(); if ((s == NATS_OK) && (opts->sslCtx != NULL)) { bool createNew = false; @@ -363,6 +364,24 @@ natsOptions_SetSecure(natsOptions *opts, bool secure) return NATS_UPDATE_ERR_STACK(s); } +natsStatus +natsOptions_TLSHandshakeFirst(natsOptions *opts) +{ + natsStatus s = NATS_OK; + + LOCK_AND_CHECK_OPTIONS(opts, 0); + + s = natsOptions_SetSecure(opts, true); + if (s == NATS_OK) + { + opts->tlsHandshakeFirst = true; + } + + UNLOCK_OPTS(opts); + + return NATS_UPDATE_ERR_STACK(s); +} + natsStatus natsOptions_LoadCATrustedCertificates(natsOptions *opts, const char *fileName) { @@ -689,6 +708,12 @@ natsOptions_SetSecure(natsOptions *opts, bool secure) return nats_setError(NATS_ILLEGAL_STATE, "%s", NO_SSL_ERR); } +natsStatus +natsOptions_TLSHandshakeFirst(natsOptions *opts) +{ + return nats_setError(NATS_ILLEGAL_STATE, "%s", NO_SSL_ERR); +} + natsStatus natsOptions_LoadCATrustedCertificates(natsOptions *opts, const char *fileName) { @@ -1044,7 +1069,7 @@ natsOptions_UseGlobalMessageDelivery(natsOptions *opts, bool global) // Sets if the subscriptions created from the connection will // create their own delivery thread or use the one(s) from // the library. - opts->libMsgDelivery = global; + opts->useSharedDispatcher = global; UNLOCK_OPTS(opts); @@ -1515,7 +1540,8 @@ natsOptions_Create(natsOptions **newOpts) if (opts == NULL) return nats_setDefaultError(NATS_NO_MEMORY); - if (natsMutex_Create(&(opts->mu)) != NATS_OK) + IFOK(s, natsMutex_Create(&opts->mu)); + if(s != NATS_OK) { NATS_FREE(opts); return NATS_UPDATE_ERR_STACK(NATS_NO_MEMORY); @@ -1531,13 +1557,14 @@ natsOptions_Create(natsOptions **newOpts) opts->maxPendingMsgs = NATS_OPTS_DEFAULT_MAX_PENDING_MSGS; opts->maxPendingBytes = -1; opts->timeout = NATS_OPTS_DEFAULT_TIMEOUT; - opts->libMsgDelivery = natsLib_isLibHandlingMsgDeliveryByDefault(); - opts->writeDeadline = natsLib_defaultWriteDeadline(); opts->reconnectBufSize = NATS_OPTS_DEFAULT_RECONNECT_BUF_SIZE; opts->reconnectJitter = NATS_OPTS_DEFAULT_RECONNECT_JITTER; opts->reconnectJitterTLS = NATS_OPTS_DEFAULT_RECONNECT_JITTER_TLS; opts->asyncErrCb = natsConn_defaultErrHandler; + // Override with values from the config (or from environment variables) + nats_overrideDefaultOptionsWithConfig(opts); + *newOpts = opts; return NATS_OK; diff --git a/src/stan/conn.c b/src/stan/conn.c index 50aaec982..4bfe90eb6 100644 --- a/src/stan/conn.c +++ b/src/stan/conn.c @@ -18,6 +18,7 @@ #include "../asynccb.h" #include "../conn.h" #include "../sub.h" +#include "../glib/glib.h" // Client send connID in ConnectRequest and PubMsg, and server // listens and responds to client PINGs. The validity of the diff --git a/src/sub.c b/src/sub.c index 499909797..561867b2b 100644 --- a/src/sub.c +++ b/src/sub.c @@ -1,4 +1,4 @@ -// Copyright 2015-2021 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,54 +23,99 @@ #include "util.h" #include "js.h" #include "opts.h" +#include "glib/glib.h" #ifdef DEV_MODE -static void _retain(natsSubscription *sub) { sub->refs++; } -static void _release(natsSubscription *sub) { sub->refs--; } - -void natsSub_Lock(natsSubscription *sub) { natsMutex_Lock(sub->mu); } -void natsSub_Unlock(natsSubscription *sub) { natsMutex_Unlock(sub->mu); } +static inline int _retain(natsSubscription *sub) { return ++(sub->refs); } +static inline int _release(natsSubscription *sub) { return --(sub->refs); } #else -#define _retain(s) ((s)->refs++) -#define _release(s) ((s)->refs--) +#define _retain(s) (++((s)->refs)) +#define _release(s) (--((s)->refs)) #endif // DEV_MODE -#define SUB_DLV_WORKER_LOCK(s) if ((s)->libDlvWorker != NULL) \ - natsMutex_Lock((s)->libDlvWorker->lock) - -#define SUB_DLV_WORKER_UNLOCK(s) if ((s)->libDlvWorker != NULL) \ - natsMutex_Unlock((s)->libDlvWorker->lock) - bool testDrainAutoUnsubRace = false; -static void -_freeSubscription(natsSubscription *sub) +static inline void _destroyControlMessage(natsMsg *msg) { - natsMsg *m; + if (msg != NULL) + { + natsMsg_clearNoDestroy(msg); + natsMsg_Destroy(msg); + } +} - if (sub == NULL) +static inline void _freeControlMessages(natsSubscription *sub) +{ + if (sub->control == NULL) return; - while ((m = sub->msgList.head) != NULL) + _destroyControlMessage(sub->control->sub.timeout); + _destroyControlMessage(sub->control->sub.close); + _destroyControlMessage(sub->control->sub.drain); + _destroyControlMessage(sub->control->batch.expired); + _destroyControlMessage(sub->control->batch.missedHeartbeat); + NATS_FREE(sub->control); +} + +static inline natsStatus _createControlMessage(natsMsg **msg, natsSubscription *sub) +{ + natsStatus s = natsMsg_create(msg, NULL, 0, NULL, 0, NULL, 0, -1); + if (s == NATS_OK) { - sub->msgList.head = m->next; - natsMsg_Destroy(m); + natsMsg_setNoDestroy(*msg); + (*msg)->sub = sub; } + return s; +} - NATS_FREE(sub->subject); - NATS_FREE(sub->queue); +// Sets up the default (user-thread NextMsg-only) dispatcher for a sub. Avoid +// the use of term "pull" since it's not to be confused with JetStream consumer +// pull requests. +static natsStatus +_initOwnDispatcher(natsSubscription *sub) +{ + natsStatus s = NATS_OK; + + if (sub->ownDispatcher.dedicatedTo != NULL) + return nats_setDefaultError(NATS_ILLEGAL_STATE); + + sub->ownDispatcher.dedicatedTo = sub; + sub->ownDispatcher.mu = sub->mu; + s = natsCondition_Create(&sub->ownDispatcher.cond); + return NATS_UPDATE_ERR_STACK(s); +} + +static inline void _cleanupOwnDispatcher(natsSubscription *sub) +{ + nats_destroyQueuedMessages(&sub->ownDispatcher.queue); - if (sub->deliverMsgsThread != NULL) + if (sub->ownDispatcher.thread != NULL) { - natsThread_Detach(sub->deliverMsgsThread); - natsThread_Destroy(sub->deliverMsgsThread); + natsThread_Join(sub->ownDispatcher.thread); + natsThread_Destroy(sub->ownDispatcher.thread); + sub->ownDispatcher.thread = NULL; } + + natsCondition_Destroy(sub->ownDispatcher.cond); +} + +void _freeSub(natsSubscription *sub) +{ + if (sub == NULL) + return; + + _freeControlMessages(sub); + _cleanupOwnDispatcher(sub); + + NATS_FREE(sub->subject); + NATS_FREE(sub->queue); + + natsCondition_Destroy(sub->drainCond); natsTimer_Destroy(sub->timeoutTimer); - natsCondition_Destroy(sub->cond); natsMutex_Destroy(sub->mu); jsSub_free(sub->jsi); @@ -79,18 +124,7 @@ _freeSubscription(natsSubscription *sub) NATS_FREE(sub); } -void -natsSub_retain(natsSubscription *sub) -{ - natsSub_Lock(sub); - - sub->refs++; - - natsSub_Unlock(sub); -} - -void -natsSub_release(natsSubscription *sub) +void natsSub_release(natsSubscription *sub) { int refs = 0; @@ -99,53 +133,29 @@ natsSub_release(natsSubscription *sub) natsSub_Lock(sub); - refs = --(sub->refs); + refs = _release(sub); natsSub_Unlock(sub); if (refs == 0) - _freeSubscription(sub); -} - -void -natsSubAndLdw_Lock(natsSubscription *sub) -{ - natsMutex_Lock(sub->mu); - SUB_DLV_WORKER_LOCK(sub); -} - -void -natsSubAndLdw_LockAndRetain(natsSubscription *sub) -{ - natsMutex_Lock(sub->mu); - sub->refs++; - SUB_DLV_WORKER_LOCK(sub); -} - -void -natsSubAndLdw_Unlock(natsSubscription *sub) -{ - SUB_DLV_WORKER_UNLOCK(sub); - natsMutex_Unlock(sub->mu); + _freeSub(sub); } -void -natsSubAndLdw_UnlockAndRelease(natsSubscription *sub) +void natsSub_unlockRelease(natsSubscription *sub) { int refs = 0; - SUB_DLV_WORKER_UNLOCK(sub); + refs = _release(sub); - refs = --(sub->refs); - natsMutex_Unlock(sub->mu); + natsSub_Unlock(sub); if (refs == 0) - _freeSubscription(sub); + _freeSub(sub); } // Runs under the subscription lock but will release it for a JS subscription // if the JS consumer needs to be deleted. -static void +static inline void _setDrainCompleteState(natsSubscription *sub) { // It is possible that we are here without being in "drain in progress" @@ -178,12 +188,12 @@ _setDrainCompleteState(natsSubscription *sub) sub->drainStatus = NATS_INVALID_SUBSCRIPTION; } sub->drainState |= SUB_DRAIN_COMPLETE; - natsCondition_Broadcast(sub->cond); + + natsCondition_Broadcast(sub->drainCond); } } -void -natsSub_setDrainCompleteState(natsSubscription *sub) +void natsSub_setDrainCompleteState(natsSubscription *sub) { natsSub_Lock(sub); _setDrainCompleteState(sub); @@ -191,24 +201,23 @@ natsSub_setDrainCompleteState(natsSubscription *sub) } // _deliverMsgs is used to deliver messages to asynchronous subscribers. -void -natsSub_deliverMsgs(void *arg) +void natsSub_deliverMsgs(void *arg) { - natsSubscription *sub = (natsSubscription*) arg; - natsConnection *nc = sub->conn; - natsMsgHandler mcb = sub->msgCb; - void *mcbClosure = sub->msgCbClosure; - uint64_t delivered; - uint64_t max; - natsMsg *msg; - int64_t timeout; - natsStatus s = NATS_OK; - bool draining = false; - bool rmSub = false; - natsOnCompleteCB onCompleteCB = NULL; - void *onCompleteCBClosure = NULL; - char *fcReply = NULL; - jsSub *jsi = NULL; + natsSubscription *sub = (natsSubscription *)arg; + natsConnection *nc = sub->conn; + natsMsgHandler mcb = sub->msgCb; + void *mcbClosure = sub->msgCbClosure; + uint64_t delivered; + uint64_t max; + natsMsg *msg; + int64_t timeout; + natsStatus s = NATS_OK; + bool draining = false; + bool rmSub = false; + natsOnCompleteCB onCompleteCB = NULL; + void *onCompleteCBClosure = NULL; + char *fcReply = NULL; + jsSub *jsi = NULL; // This just serves as a barrier for the creation of this thread. natsConn_Lock(nc); @@ -224,12 +233,12 @@ natsSub_deliverMsgs(void *arg) natsSub_Lock(sub); s = NATS_OK; - while (((msg = sub->msgList.head) == NULL) && !(sub->closed) && !(sub->draining) && (s != NATS_TIMEOUT)) + while (((msg = sub->ownDispatcher.queue.head) == NULL) && !(sub->closed) && !(sub->draining) && (s != NATS_TIMEOUT)) { if (timeout != 0) - s = natsCondition_TimedWait(sub->cond, sub->mu, timeout); + s = natsCondition_TimedWait(sub->ownDispatcher.cond, sub->mu, timeout); else - natsCondition_Wait(sub->cond, sub->mu); + natsCondition_Wait(sub->ownDispatcher.cond, sub->mu); } if (sub->closed) @@ -256,13 +265,13 @@ natsSub_deliverMsgs(void *arg) delivered = ++(sub->delivered); - sub->msgList.head = msg->next; + sub->ownDispatcher.queue.head = msg->next; - if (sub->msgList.tail == msg) - sub->msgList.tail = NULL; + if (sub->ownDispatcher.queue.tail == msg) + sub->ownDispatcher.queue.tail = NULL; - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_dataAndHdrLen(msg); + sub->ownDispatcher.queue.msgs--; + sub->ownDispatcher.queue.bytes -= natsMsg_dataAndHdrLen(msg); msg->next = NULL; @@ -276,7 +285,7 @@ natsSub_deliverMsgs(void *arg) if ((max == 0) || (delivered <= max)) { - (*mcb)(nc, sub, msg, mcbClosure); + (*mcb)(nc, sub, msg, mcbClosure); } else { @@ -301,7 +310,7 @@ natsSub_deliverMsgs(void *arg) } natsSub_Lock(sub); - onCompleteCB = sub->onCompleteCB; + onCompleteCB = sub->onCompleteCB; onCompleteCBClosure = sub->onCompleteCBClosure; _setDrainCompleteState(sub); natsSub_Unlock(sub); @@ -315,17 +324,27 @@ natsSub_deliverMsgs(void *arg) natsSub_release(sub); } -bool -natsSub_setMax(natsSubscription *sub, uint64_t max) +// Should be called only during the subscription creation process, no need to lock +static inline natsStatus +_runOwnDispatcher(natsSubscription *sub, bool forReplies) +{ + natsStatus s = NATS_OK; + if (sub->ownDispatcher.thread != NULL) + return nats_setDefaultError(NATS_ILLEGAL_STATE); // already running + + sub->dispatcher = &sub->ownDispatcher; + s = natsThread_Create(&sub->ownDispatcher.thread, natsSub_deliverMsgs, (void *) sub); + return NATS_UPDATE_ERR_STACK(s); +} + +bool natsSub_setMax(natsSubscription *sub, uint64_t max) { bool accepted = false; - natsSub_Lock(sub); - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); sub->max = (max <= sub->delivered ? 0 : max); accepted = sub->max != 0; - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return accepted; } @@ -350,12 +369,9 @@ natsSubscription_SetOnCompleteCB(natsSubscription *sub, natsOnCompleteCB cb, voi return s; } -void -natsSub_close(natsSubscription *sub, bool connectionClosed) +void natsSub_close(natsSubscription *sub, bool connectionClosed) { - natsSub_Lock(sub); - - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); if (!(sub->closed)) { @@ -365,40 +381,40 @@ natsSub_close(natsSubscription *sub, bool connectionClosed) if ((sub->jsi != NULL) && (sub->jsi->hbTimer != NULL)) natsTimer_Stop(sub->jsi->hbTimer); - if (sub->libDlvWorker != NULL) + // If this is a subscription with timeout, stop the timer. + if (sub->timeout != 0) + natsTimer_Stop(sub->timeoutTimer); + + if (sub->dispatcher != &sub->ownDispatcher) { - // If this is a subscription with timeout, stop the timer. - if (sub->timeout != 0) - natsTimer_Stop(sub->timeoutTimer); - - // Post a control message to wake-up the worker which will - // ensure that all pending messages for this subscription - // are removed and the subscription will ultimately be - // released in the worker thread. - natsLib_msgDeliveryPostControlMsg(sub); + // Post a control message to wake-up the worker which will ensure + // that all pending messages for this subscription are removed, + // release the subscription and self-destroy. + natsSub_enqueueMessage(sub, sub->control->sub.close); } else - natsCondition_Broadcast(sub->cond); + { + // Notify any pending natsSub_NextMsg() that the subscription is + // closed. + natsCondition_Broadcast(sub->ownDispatcher.cond); + } } - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } static void -_asyncTimeoutCb(natsTimer *timer, void* closure) +_asyncTimeoutCb(natsTimer *timer, void *closure) { - natsSubscription *sub = (natsSubscription*) closure; + natsSubscription *sub = (natsSubscription *)closure; // Should not happen, but in case - if (sub->libDlvWorker == NULL) + if (sub->dispatcher == NULL) return; - SUB_DLV_WORKER_LOCK(sub); - - // If the subscription is closed, or if we are prevented from posting - // a "timeout" control message, do nothing. + nats_lockSubAndDispatcher(sub); + // If the subscription has already timed out and has not reset, is closed or + // draining - do nothing. if (!sub->closed && !sub->timedOut && !sub->timeoutSuspended) { // Prevent from scheduling another control message while we are not @@ -407,32 +423,50 @@ _asyncTimeoutCb(natsTimer *timer, void* closure) // Set the timer to a very high value, it will be reset from the // worker thread. - natsTimer_Reset(sub->timeoutTimer, 60*60*1000); + natsTimer_Reset(sub->timeoutTimer, 60 * 60 * 1000); // Post a control message to the worker thread. - natsLib_msgDeliveryPostControlMsg(sub); + natsSub_enqueueMessage(sub, sub->control->sub.timeout); } - - SUB_DLV_WORKER_UNLOCK(sub); + nats_unlockSubAndDispatcher(sub); } static void -_asyncTimeoutStopCb(natsTimer *timer, void* closure) +_asyncTimeoutStopCb(natsTimer *timer, void *closure) { - natsSubscription *sub = (natsSubscription*) closure; + natsSubscription *sub = (natsSubscription *)closure; natsSub_release(sub); } +natsStatus nats_createControlMessages(natsSubscription *sub) +{ + natsStatus s = NATS_OK; + + if (sub->control != NULL) + return NATS_OK; + sub->control = NATS_CALLOC(1, sizeof(natsSubscriptionControlMessages)); + if (sub->control == NULL) + return nats_setDefaultError(NATS_NO_MEMORY); + IFOK(s, _createControlMessage(&(sub->control->sub.timeout), sub)); + IFOK(s, _createControlMessage(&sub->control->sub.close, sub)); + IFOK(s, _createControlMessage(&sub->control->sub.drain, sub)); + IFOK(s, _createControlMessage(&sub->control->batch.expired, sub)); + IFOK(s, _createControlMessage(&sub->control->batch.missedHeartbeat, sub)); + + // no need to free on failure, sub's free will clean it up. + return NATS_UPDATE_ERR_STACK(s); +} + natsStatus natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, const char *queueGroup, int64_t timeout, natsMsgHandler cb, void *cbClosure, - bool preventUseOfLibDlvPool, jsSub *jsi) + bool forReplies, jsSub *jsi) { - natsStatus s = NATS_OK; - natsSubscription *sub = NULL; + natsStatus s = NATS_OK; + natsSubscription *sub = NULL; - sub = (natsSubscription*) NATS_CALLOC(1, sizeof(natsSubscription)); + sub = (natsSubscription *)NATS_CALLOC(1, sizeof(natsSubscription)); if (sub == NULL) return nats_setDefaultError(NATS_NO_MEMORY); @@ -451,7 +485,7 @@ natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, sub->msgCb = cb; sub->msgCbClosure = cbClosure; sub->msgsLimit = nc->opts->maxPendingMsgs; - sub->bytesLimit = nc->opts->maxPendingBytes == -1 ? nc->opts->maxPendingMsgs * 1024 : nc->opts->maxPendingBytes;; + sub->bytesLimit = nc->opts->maxPendingBytes == -1 ? nc->opts->maxPendingMsgs * 1024 : (int)nc->opts->maxPendingBytes;; sub->jsi = jsi; sub->subject = NATS_STRDUP(subj); @@ -464,38 +498,45 @@ natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, if (sub->queue == NULL) s = nats_setDefaultError(NATS_NO_MEMORY); } + if (s == NATS_OK) - s = natsCondition_Create(&(sub->cond)); - if ((s == NATS_OK) && (cb != NULL)) - { - if (!(nc->opts->libMsgDelivery) || preventUseOfLibDlvPool) - { - // Let's not rely on the created thread acquiring the lock that - // would make it safe to retain only on success. - _retain(sub); + s = natsCondition_Create(&sub->drainCond); + if (s == NATS_OK) + s = _initOwnDispatcher(sub); - // If we have an async callback, start up a sub specific - // thread to deliver the messages. - s = natsThread_Create(&(sub->deliverMsgsThread), natsSub_deliverMsgs, - (void*) sub); - if (s != NATS_OK) - _release(sub); + bool useShared = (forReplies ? nc->opts->useSharedReplyDispatcher : nc->opts->useSharedDispatcher); + bool useAsyncThread = (cb != NULL); + if ((s == NATS_OK) && useAsyncThread) + s = nats_createControlMessages(sub); + if (s == NATS_OK) + { + _retain(sub); + if (!useAsyncThread) + { + sub->dispatcher = &sub->ownDispatcher; + _release(sub); } - else + else if (useShared && !forReplies) { - _retain(sub); - s = natsLib_msgDeliveryAssignWorker(sub); + s = nats_assignSubToDispatch(sub); + + // If we are using a shared dispatcher, we need to start the + // timeout timer. Own dispatcher uses a timed wait on the + // condition, and does not need the timer. if ((s == NATS_OK) && (timeout > 0)) { _retain(sub); - s = natsTimer_Create(&sub->timeoutTimer, _asyncTimeoutCb, - _asyncTimeoutStopCb, timeout, (void*) sub); + s = natsTimer_Create(&sub->timeoutTimer, _asyncTimeoutCb, _asyncTimeoutStopCb, timeout, sub); if (s != NATS_OK) _release(sub); } - if (s != NATS_OK) - _release(sub); } + else + { + s = _runOwnDispatcher(sub, forReplies); + } + if (s != NATS_OK) + _release(sub); } if (s == NATS_OK) @@ -547,7 +588,6 @@ natsConnection_SubscribeTimeout(natsSubscription **sub, natsConnection *nc, cons return NATS_UPDATE_ERR_STACK(s); } - /* * natsSubscribeSync is syntactic sugar for natsSubscribe(&sub, nc, subject, NULL). */ @@ -569,8 +609,8 @@ natsConnection_SubscribeSync(natsSubscription **sub, natsConnection *nc, const c */ natsStatus natsConnection_QueueSubscribe(natsSubscription **sub, natsConnection *nc, - const char *subject, const char *queueGroup, - natsMsgHandler cb, void *cbClosure) + const char *subject, const char *queueGroup, + natsMsgHandler cb, void *cbClosure) { natsStatus s; @@ -591,13 +631,12 @@ natsConnection_QueueSubscribe(natsSubscription **sub, natsConnection *nc, */ natsStatus natsConnection_QueueSubscribeTimeout(natsSubscription **sub, natsConnection *nc, - const char *subject, const char *queueGroup, - int64_t timeout, natsMsgHandler cb, void *cbClosure) + const char *subject, const char *queueGroup, + int64_t timeout, natsMsgHandler cb, void *cbClosure) { natsStatus s; - if ((queueGroup == NULL) || (strlen(queueGroup) == 0) || (cb == NULL) - || (timeout <= 0)) + if ((queueGroup == NULL) || (strlen(queueGroup) == 0) || (cb == NULL) || (timeout <= 0)) { return nats_setDefaultError(NATS_INVALID_ARG); } @@ -612,7 +651,7 @@ natsConnection_QueueSubscribeTimeout(natsSubscription **sub, natsConnection *nc, */ natsStatus natsConnection_QueueSubscribeSync(natsSubscription **sub, natsConnection *nc, - const char *subject, const char *queueGroup) + const char *subject, const char *queueGroup) { natsStatus s; @@ -645,13 +684,13 @@ natsSubscription_NoDeliveryDelay(natsSubscription *sub) natsStatus natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool pullSubInternal) { - natsStatus s = NATS_OK; - natsConnection *nc = NULL; - natsMsg *msg = NULL; - bool removeSub = false; - int64_t target = 0; - jsSub *jsi = NULL; - char *fcReply = NULL; + natsStatus s = NATS_OK; + natsConnection *nc = NULL; + natsMsg *msg = NULL; + bool removeSub = false; + int64_t target = 0; + jsSub *jsi = NULL; + char *fcReply = NULL; if ((sub == NULL) || (nextMsg == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); @@ -705,19 +744,16 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool } nc = sub->conn; - jsi= sub->jsi; + jsi = sub->jsi; if (timeout > 0) { - while ((sub->msgList.msgs == 0) - && (s != NATS_TIMEOUT) - && !(sub->closed) - && !(sub->draining)) + while ((sub->ownDispatcher.queue.msgs == 0) && (s != NATS_TIMEOUT) && !(sub->closed) && !(sub->draining)) { if (target == 0) target = nats_setTargetTime(timeout); - s = natsCondition_AbsoluteTimedWait(sub->cond, sub->mu, target); + s = natsCondition_AbsoluteTimedWait(sub->ownDispatcher.cond, sub->mu, target); if (s != NATS_OK) s = nats_setDefaultError(s); } @@ -729,14 +765,14 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool } else { - s = (sub->msgList.msgs == 0 ? NATS_TIMEOUT : NATS_OK); + s = (sub->ownDispatcher.queue.msgs == 0 ? NATS_TIMEOUT : NATS_OK); if ((s != NATS_OK) && !pullSubInternal) s = nats_setDefaultError(s); } if (s == NATS_OK) { - msg = sub->msgList.head; + msg = sub->ownDispatcher.queue.head; if ((msg == NULL) && sub->draining) { removeSub = true; @@ -744,13 +780,13 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool } else { - sub->msgList.head = msg->next; + sub->ownDispatcher.queue.head = msg->next; - if (sub->msgList.tail == msg) - sub->msgList.tail = NULL; + if (sub->ownDispatcher.queue.tail == msg) + sub->ownDispatcher.queue.tail = NULL; - sub->msgList.msgs--; - sub->msgList.bytes -= natsMsg_dataAndHdrLen(msg); + sub->ownDispatcher.queue.msgs--; + sub->ownDispatcher.queue.bytes -= natsMsg_dataAndHdrLen(msg); msg->next = NULL; @@ -765,7 +801,7 @@ natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool removeSub = true; } - if (sub->draining && (sub->msgList.msgs == 0)) + if (sub->draining && (sub->ownDispatcher.queue.msgs == 0)) removeSub = true; } if (removeSub) @@ -815,10 +851,10 @@ natsSubscription_NextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeo static natsStatus _unsubscribe(natsSubscription *sub, int max, bool drainMode, int64_t timeout) { - natsStatus s = NATS_OK; - natsConnection *nc = NULL; - bool dc = false; - jsSub *jsi; + natsStatus s = NATS_OK; + natsConnection *nc = NULL; + bool dc = false; + jsSub *jsi; if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); @@ -853,6 +889,7 @@ _unsubscribe(natsSubscription *sub, int max, bool drainMode, int64_t timeout) natsStatus natsSubscription_Unsubscribe(natsSubscription *sub) { + natsStatus s = _unsubscribe(sub, 0, false, 0); return NATS_UPDATE_ERR_STACK(s); } @@ -864,43 +901,34 @@ natsSubscription_AutoUnsubscribe(natsSubscription *sub, int max) return NATS_UPDATE_ERR_STACK(s); } -void -natsSub_drain(natsSubscription *sub) +void natsSub_drain(natsSubscription *sub) { - natsSub_Lock(sub); - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return; } sub->draining = true; - if (sub->libDlvWorker != NULL) - { - // If this is a subscription with timeout, stop the timer. - if (sub->timeout != 0) - { - natsTimer_Stop(sub->timeoutTimer); - // Prevent code to reset this timer - sub->timeoutSuspended = true; - } - // Set this to true. It will be set to false in the - // worker delivery thread when the control message is - // processed. - sub->libDlvDraining = true; + // If this is a subscription with timeout, stop the timer. + if (sub->timeout != 0) + { + natsTimer_Stop(sub->timeoutTimer); + // Prevent code to reset this timer + sub->timeoutSuspended = true; + } - // Post a control message to wake-up the worker which will - // ensure that all pending messages for this subscription - // are removed and the subscription will ultimately be - // released in the worker thread. - natsLib_msgDeliveryPostControlMsg(sub); + if (sub->dispatcher != &sub->ownDispatcher) + { + natsSub_enqueueMessage(sub, sub->control->sub.drain); } else - natsCondition_Broadcast(sub->cond); - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + { + natsCondition_Broadcast(sub->ownDispatcher.cond); + } + + nats_unlockSubAndDispatcher(sub); } static void @@ -911,8 +939,7 @@ _updateDrainStatus(natsSubscription *sub, natsStatus s) sub->drainStatus = s; } -void -natsSub_updateDrainStatus(natsSubscription *sub, natsStatus s) +void natsSub_updateDrainStatus(natsSubscription *sub, natsStatus s) { natsSub_Lock(sub); _updateDrainStatus(sub, s); @@ -920,33 +947,30 @@ natsSub_updateDrainStatus(natsSubscription *sub, natsStatus s) } // Mark the subscription such that connection stops to try to push messages into its list. -void -natsSub_setDrainSkip(natsSubscription *sub, natsStatus s) +void natsSub_setDrainSkip(natsSubscription *sub, natsStatus s) { - natsSub_Lock(sub); - SUB_DLV_WORKER_LOCK(sub); + nats_lockSubAndDispatcher(sub); _updateDrainStatus(sub, s); sub->drainSkip = true; - SUB_DLV_WORKER_UNLOCK(sub); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); } static void _flushAndDrain(void *closure) { - natsSubscription *sub = (natsSubscription*) closure; - natsConnection *nc = NULL; - natsThread *t = NULL; - int64_t timeout = 0; - int64_t deadline = 0; - bool sync = false; - natsStatus s; + natsSubscription *sub = (natsSubscription *)closure; + natsConnection *nc = NULL; + natsThread *t = NULL; + int64_t timeout = 0; + int64_t deadline = 0; + bool sync = false; + natsStatus s; natsSub_Lock(sub); - nc = sub->conn; - t = sub->drainThread; + nc = sub->conn; + t = sub->drainThread; timeout = sub->drainTimeout; - sync = (sub->msgCb == NULL ? true : false); + sync = (sub->msgCb == NULL ? true : false); natsSub_Unlock(sub); // Make sure that negative value is considered no timeout. @@ -982,14 +1006,15 @@ _flushAndDrain(void *closure) // already called NextMsg() for all pending messages before the sub // was marked as "draining", so if we detect this situation, we need // to switch status to complete here. - if (sync && !natsSub_drainComplete(sub) && (sub->msgList.msgs == 0)) + + if (sync && !natsSub_drainComplete(sub) && (sub->ownDispatcher.queue.msgs == 0)) { _setDrainCompleteState(sub); } else { while ((s != NATS_TIMEOUT) && !natsSub_drainComplete(sub)) - s = natsCondition_AbsoluteTimedWait(sub->cond, sub->mu, deadline); + s = natsCondition_AbsoluteTimedWait(sub->drainCond, sub->mu, deadline); } natsSub_Unlock(sub); @@ -1006,8 +1031,7 @@ _flushAndDrain(void *closure) } // Switch subscription's drain state to "started". -void -natsSub_initDrain(natsSubscription *sub) +void natsSub_initDrain(natsSubscription *sub) { natsSub_Lock(sub); sub->drainState |= SUB_DRAIN_STARTED; @@ -1034,7 +1058,7 @@ natsSub_startDrain(natsSubscription *sub, int64_t timeout) // to make sure that this call will not block. s = natsConn_enqueueUnsubProto(sub->conn, sub->sid); if (s == NATS_OK) - s = natsThread_Create(&(sub->drainThread), _flushAndDrain, (void*) sub); + s = natsThread_Create(&(sub->drainThread), _flushAndDrain, (void *)sub); if (s == NATS_OK) { sub->drainTimeout = timeout; @@ -1067,9 +1091,9 @@ natsSubscription_DrainTimeout(natsSubscription *sub, int64_t timeout) natsStatus natsSubscription_WaitForDrainCompletion(natsSubscription *sub, int64_t timeout) { - natsStatus s = NATS_OK; - int64_t deadline = 0; - bool dc = false; + natsStatus s = NATS_OK; + int64_t deadline = 0; + bool dc = false; if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); @@ -1090,9 +1114,9 @@ natsSubscription_WaitForDrainCompletion(natsSubscription *sub, int64_t timeout) while ((s != NATS_TIMEOUT) && !natsSub_drainComplete(sub)) { if (timeout > 0) - s = natsCondition_AbsoluteTimedWait(sub->cond, sub->mu, deadline); + s = natsCondition_AbsoluteTimedWait(sub->drainCond, sub->mu, deadline); else - natsCondition_Wait(sub->cond, sub->mu); + natsCondition_Wait(sub->drainCond, sub->mu); } natsSub_Unlock(sub); @@ -1130,21 +1154,21 @@ natsSubscription_DrainCompletionStatus(natsSubscription *sub) natsStatus natsSubscription_QueuedMsgs(natsSubscription *sub, uint64_t *queuedMsgs) { - natsStatus s; - int msgs = 0; + natsStatus s; + int msgs = 0; if (queuedMsgs == NULL) return nats_setDefaultError(NATS_INVALID_ARG); s = natsSubscription_GetPending(sub, &msgs, NULL); if (s == NATS_OK) - *queuedMsgs = (uint64_t) msgs; + *queuedMsgs = (uint64_t)msgs; return s; } int64_t -natsSubscription_GetID(natsSubscription* sub) +natsSubscription_GetID(natsSubscription *sub) { int64_t id = 0; @@ -1166,10 +1190,10 @@ natsSubscription_GetID(natsSubscription* sub) return id; } -const char* -natsSubscription_GetSubject(natsSubscription* sub) +const char * +natsSubscription_GetSubject(natsSubscription *sub) { - const char* subject = NULL; + const char *subject = NULL; if (sub == NULL) return NULL; @@ -1182,38 +1206,36 @@ natsSubscription_GetSubject(natsSubscription* sub) return NULL; } - subject = (const char*)sub->subject; + subject = (const char *)sub->subject; natsSub_Unlock(sub); return subject; } +// This works for both shared and dedicated dispatchers since we maintain the +// per-sub stats. natsStatus natsSubscription_GetPending(natsSubscription *sub, int *msgs, int *bytes) { if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - if (msgs != NULL) - *msgs = sub->msgList.msgs; + *msgs = sub->ownDispatcher.queue.msgs; if (bytes != NULL) - *bytes = sub->msgList.bytes; + *bytes = sub->ownDispatcher.queue.bytes; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1226,24 +1248,20 @@ natsSubscription_SetPendingLimits(natsSubscription *sub, int msgLimit, int bytes if ((msgLimit == 0) || (bytesLimit == 0)) return nats_setError(NATS_INVALID_ARG, "%s", - "Limits must be either > 0 or negative to specify no limit"); + "Limits must be either > 0 or negative to specify no limit"); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - sub->msgsLimit = msgLimit; sub->bytesLimit = bytesLimit; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1254,25 +1272,21 @@ natsSubscription_GetPendingLimits(natsSubscription *sub, int *msgLimit, int *byt if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - if (msgLimit != NULL) *msgLimit = sub->msgsLimit; if (bytesLimit != NULL) *bytesLimit = sub->bytesLimit; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1283,21 +1297,17 @@ natsSubscription_GetDelivered(natsSubscription *sub, int64_t *msgs) if ((sub == NULL) || (msgs == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - - *msgs = (int64_t) sub->delivered; + *msgs = (int64_t)sub->delivered; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1308,21 +1318,17 @@ natsSubscription_GetDropped(natsSubscription *sub, int64_t *msgs) if ((sub == NULL) || (msgs == NULL)) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - *msgs = sub->dropped; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1333,25 +1339,21 @@ natsSubscription_GetMaxPending(natsSubscription *sub, int *msgs, int *bytes) if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - if (msgs != NULL) *msgs = sub->msgsMax; if (bytes != NULL) *bytes = sub->bytesMax; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1362,53 +1364,47 @@ natsSubscription_ClearMaxPending(natsSubscription *sub) if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); - + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - sub->msgsMax = 0; sub->bytesMax = 0; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } natsStatus natsSubscription_GetStats(natsSubscription *sub, - int *pendingMsgs, - int *pendingBytes, - int *maxPendingMsgs, - int *maxPendingBytes, - int64_t *deliveredMsgs, - int64_t *droppedMsgs) + int *pendingMsgs, + int *pendingBytes, + int *maxPendingMsgs, + int *maxPendingBytes, + int64_t *deliveredMsgs, + int64_t *droppedMsgs) { if (sub == NULL) return nats_setDefaultError(NATS_INVALID_ARG); - natsSub_Lock(sub); + nats_lockSubAndDispatcher(sub); if (sub->closed) { - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return nats_setDefaultError(NATS_INVALID_SUBSCRIPTION); } - SUB_DLV_WORKER_LOCK(sub); - + // messages and bytes are up to date even with a shared dispatcher. if (pendingMsgs != NULL) - *pendingMsgs = sub->msgList.msgs; + *pendingMsgs = sub->ownDispatcher.queue.msgs; if (pendingBytes != NULL) - *pendingBytes = sub->msgList.bytes; + *pendingBytes = sub->ownDispatcher.queue.bytes; if (maxPendingMsgs != NULL) *maxPendingMsgs = sub->msgsMax; @@ -1417,14 +1413,12 @@ natsSubscription_GetStats(natsSubscription *sub, *maxPendingBytes = sub->bytesMax; if (deliveredMsgs != NULL) - *deliveredMsgs = (int) sub->delivered; + *deliveredMsgs = (int)sub->delivered; if (droppedMsgs != NULL) *droppedMsgs = sub->dropped; - SUB_DLV_WORKER_UNLOCK(sub); - - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); return NATS_OK; } @@ -1434,8 +1428,7 @@ natsSubscription_GetStats(natsSubscription *sub, * This will return false if the subscription has already been closed, * or auto unsubscribed. */ -bool -natsSubscription_IsValid(natsSubscription *sub) +bool natsSubscription_IsValid(natsSubscription *sub) { bool valid = false; @@ -1455,8 +1448,7 @@ natsSubscription_IsValid(natsSubscription *sub) * Destroys the subscription object, freeing up memory. * If not already done, this call will removes interest on the subject. */ -void -natsSubscription_Destroy(natsSubscription *sub) +void natsSubscription_Destroy(natsSubscription *sub) { bool doUnsub = false; @@ -1480,7 +1472,7 @@ natsSubscription_Destroy(natsSubscription *sub) natsSub_Unlock(sub); if (doUnsub) - (void) natsSubscription_Unsubscribe(sub); + (void)natsSubscription_Unsubscribe(sub); natsSub_release(sub); } diff --git a/src/sub.h b/src/sub.h index 876fabe3b..52f89023f 100644 --- a/src/sub.h +++ b/src/sub.h @@ -1,4 +1,4 @@ -// Copyright 2015-2021 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,32 +16,79 @@ #include "natsp.h" -#ifdef DEV_MODE -// For type safety... +#define SUB_DRAIN_STARTED ((uint8_t)1) +#define SUB_DRAIN_COMPLETE ((uint8_t)2) -void natsSub_Lock(natsSubscription *sub); -void natsSub_Unlock(natsSubscription *sub); - -#else - -#define natsSub_Lock(s) natsMutex_Lock((s)->mu) -#define natsSub_Unlock(s) natsMutex_Unlock((s)->mu) - -#endif // DEV_MODE - -#define SUB_DRAIN_STARTED ((uint8_t) 1) -#define SUB_DRAIN_COMPLETE ((uint8_t) 2) - -#define natsSub_drainStarted(s) (((s)->drainState & SUB_DRAIN_STARTED) != 0) -#define natsSub_drainComplete(s) (((s)->drainState & SUB_DRAIN_COMPLETE) != 0) +#define natsSub_drainStarted(s) (((s)->drainState & SUB_DRAIN_STARTED) != 0) +#define natsSub_drainComplete(s) (((s)->drainState & SUB_DRAIN_COMPLETE) != 0) extern bool testDrainAutoUnsubRace; -void -natsSub_retain(natsSubscription *sub); +// lock/unlock sub +static inline void natsSub_Lock(natsSubscription *sub) +{ + natsMutex_Lock(sub->mu); +} + +static inline void natsSub_Unlock(natsSubscription *sub) +{ + natsMutex_Unlock(sub->mu); +} + +// lock/unlock with retain/release +static inline void natsSub_lockRetain(natsSubscription *sub) +{ + natsSub_Lock(sub); + + sub->refs++; +} +void natsSub_unlockRelease(natsSubscription *sub); + + +// retain/release, lock is obtained +static inline void natsSub_retain(natsSubscription *sub) +{ + natsSub_lockRetain(sub); + natsSub_Unlock(sub); +} +void natsSub_release(natsSubscription *sub); + +// lock/unlock sub's dispatcher ONLY +static inline void natsSub_lockDispatcher(natsSubscription *sub) +{ + if (sub->dispatcher != &sub->ownDispatcher) + nats_lockDispatcher(sub->dispatcher); +} +static inline void natsSub_unlockDispatcher(natsSubscription *sub) +{ + if (sub->dispatcher != &sub->ownDispatcher) + nats_unlockDispatcher(sub->dispatcher); +} + +// lock/unlock sub and its dispatcher, together +static inline void nats_lockSubAndDispatcher(natsSubscription *sub) +{ + natsSub_Lock(sub); + natsSub_lockDispatcher(sub); +} +static inline void nats_unlockSubAndDispatcher(natsSubscription *sub) +{ + natsSub_unlockDispatcher(sub); + natsSub_Unlock(sub); +} + +// lock/unlock sub and its dispatcher, and retain/release the sub +static inline void nats_lockRetainSubAndDispatcher(natsSubscription *sub) +{ + natsSub_lockRetain(sub); + natsSub_lockDispatcher(sub); +} +static inline void nats_unlockReleaseSubAndDispatcher(natsSubscription *sub) +{ + natsSub_unlockDispatcher(sub); + natsSub_unlockRelease(sub); +} -void -natsSub_release(natsSubscription *sub); natsStatus natsSub_create(natsSubscription **newSub, natsConnection *nc, const char *subj, @@ -72,19 +119,9 @@ natsSub_drain(natsSubscription *sub); natsStatus natsSub_nextMsg(natsMsg **nextMsg, natsSubscription *sub, int64_t timeout, bool pullSubInternal); -void -natsSubAndLdw_Lock(natsSubscription *sub); - -void -natsSubAndLdw_Unlock(natsSubscription *sub); - -void -natsSubAndLdw_LockAndRetain(natsSubscription *sub); - -void -natsSubAndLdw_UnlockAndRelease(natsSubscription *sub); - void natsSub_close(natsSubscription *sub, bool connectionClosed); +natsStatus nats_createControlMessages(natsSubscription *sub); + #endif /* SUB_H_ */ diff --git a/src/timer.c b/src/timer.c index 7a449456f..ba2548aa2 100644 --- a/src/timer.c +++ b/src/timer.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -14,6 +14,7 @@ #include "natsp.h" #include "mem.h" #include "util.h" +#include "glib/glib.h" static void _freeTimer(natsTimer *t) @@ -75,6 +76,9 @@ natsTimer_Create(natsTimer **timer, natsTimerCb timerCb, natsTimerStopCb stopCb, void natsTimer_Stop(natsTimer *timer) { + if (timer == NULL) + return; + // Proxy for this call: nats_stopTimer(timer); } @@ -82,6 +86,8 @@ natsTimer_Stop(natsTimer *timer) void natsTimer_Reset(natsTimer *timer, int64_t interval) { + if (timer == NULL) + return; // Proxy for this call: nats_resetTimer(timer, interval); } diff --git a/src/unix/sock.c b/src/unix/sock.c index 94e512c21..4bfcfa39c 100644 --- a/src/unix/sock.c +++ b/src/unix/sock.c @@ -16,7 +16,7 @@ #include "../comsock.h" void -natsSys_Init(void) +nats_initForOS(void) { // Would do anything that needs to be initialized when // the library loads, specific to unix. diff --git a/src/unix/thread.c b/src/unix/thread.c index 2f7773844..624f66080 100644 --- a/src/unix/thread.c +++ b/src/unix/thread.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -13,6 +13,7 @@ #include "../natsp.h" #include "../mem.h" +#include "../glib/glib.h" bool nats_InitOnce(natsInitOnceType *control, natsInitOnceCb cb) diff --git a/src/util.c b/src/util.c index 4510b4bfe..2d33b3d23 100644 --- a/src/util.c +++ b/src/util.c @@ -1705,7 +1705,7 @@ nats_Base32_Init(void) int i; for (i=0; i<(int)sizeof(base32DecodeMap); i++) - base32DecodeMap[i] = (char) 0xFF; + base32DecodeMap[i] = '\xFF'; for (i=0; i 0) len++; // For the ',' if (strings[i] == NULL) - len += strlen("(null)"); + len += (int)strlen("(null)"); else - len += strlen(strings[i]); + len += (int)strlen(strings[i]); } len++; // For the ']' len++; // For the '\0' diff --git a/src/win/sock.c b/src/win/sock.c index ea2845e07..74a808264 100644 --- a/src/win/sock.c +++ b/src/win/sock.c @@ -20,7 +20,7 @@ #include "../comsock.h" void -natsSys_Init(void) +nats_initForOS(void) { WSADATA wsaData; int errorno; diff --git a/src/win/thread.c b/src/win/thread.c index c85e828ae..b50de09a4 100644 --- a/src/win/thread.c +++ b/src/win/thread.c @@ -1,4 +1,4 @@ -// Copyright 2015-2018 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -16,6 +16,7 @@ #include #include "../mem.h" +#include "../glib/glib.h" static BOOL CALLBACK _initHandleFunction(PINIT_ONCE InitOnce, PVOID Parameter, PVOID *lpContext) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 58a4aa14f..72308f18e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,9 +1,14 @@ +cmake_minimum_required(VERSION 3.10) + +enable_testing() + if(NOT BUILD_TESTING) return() endif() + if(NOT NATS_BUILD_LIB_STATIC) - MESSAGE(FATAL_ERROR - "Building tests require static library, or run CMake with -DBUILD_TESTING=OFF") + MESSAGE(FATAL_ERROR + "Building tests require static library, or run CMake with -DBUILD_TESTING=OFF") return() endif() @@ -13,40 +18,62 @@ endif() # We need this to build the test program include_directories(${PROJECT_SOURCE_DIR}/src) + if(NATS_BUILD_WITH_TLS) include_directories(${OPENSSL_INCLUDE_DIR}) endif(NATS_BUILD_WITH_TLS) + if(NATS_BUILD_STREAMING) include_directories(${NATS_PROTOBUF_INCLUDE_DIRS}) include_directories(${PROJECT_SOURCE_DIR}/src/stan) endif(NATS_BUILD_STREAMING) # Build the test program -add_executable(testsuite test.c) +add_executable(testsuite test.c bench_sub_async.c) # Link statically with the library target_link_libraries(testsuite nats_static ${NATS_EXTRA_LIB}) -# Set the test index to 0 -set(testIndex 0) +set(BENCH_LIST ${PROJECT_SOURCE_DIR}/test/list_bench.txt) +set(STAN_LIST ${PROJECT_SOURCE_DIR}/test/list_stan.txt) -# Read the file 'list.txt' to get all the test names -file(STRINGS list.txt listOfTestNames) +list(APPEND ALL_LISTS ${PROJECT_SOURCE_DIR}/test/list_test.txt) +list(APPEND ALL_LISTS ${BENCH_LIST}) +if(NATS_BUILD_STREAMING) +list(APPEND ALL_LISTS ${STAN_LIST}) +message("Building Streaming tests" ${ALL_LISTS}) +endif() -# For each test name -foreach(name ${listOfTestNames}) +set(TEST_NAMES) - # Create a test and pass the index (start and end are the same) - # to the testsuite executable - add_test(NAME Test_${name} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMAND testsuite ${testIndex} ${testIndex}) +foreach(LIST_FILE ${ALL_LISTS}) + # Get all the test names + if(EXISTS ${LIST_FILE}) + file(STRINGS ${LIST_FILE} TEST_NAMES) + else() + set(TEST_NAMES) + endif() - # Make sure the test passes - set_tests_properties(Test_${name} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED") + foreach(name ${TEST_NAMES}) + # Remove the _test() prefix + string(REGEX REPLACE "_test\\(([^)]+)\\)" "\\1" TEST_NAME ${name}) + add_test(NAME ${TEST_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND testsuite ${TEST_NAME}) - # Bump the test index number - math(EXPR testIndex "${testIndex}+1") -endforeach() + if(${LIST_FILE} STREQUAL ${BENCH_LIST}) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "bench") + else() + # Make sure the test passes + set_tests_properties(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED") + # Set TSAN_OPTIONS for the test + if(NATS_SANITIZE) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT "TSAN_OPTIONS=detect_deadlocks=1:second_deadlock_stack=1:halt_on_error=1:report_signal_unsafe=1") + endif(NATS_SANITIZE) + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "test") + endif() + endforeach() +endforeach() diff --git a/test/bench_sub_async.c b/test/bench_sub_async.c new file mode 100644 index 000000000..47c58c7ae --- /dev/null +++ b/test/bench_sub_async.c @@ -0,0 +1,447 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test.h" +#include "sub.h" + +#define REPEAT 5 + +typedef struct __env ENV; + +typedef struct +{ + bool useGlobalDelivery; + int max; +} threadConfig; + +typedef struct +{ + natsSubscription *sub; + uint64_t sum; + uint64_t xor ; + uint64_t count; + int64_t closedTimestamp; + + ENV *env; +} subState; + +typedef natsStatus (*publishFunc)(natsConnection *nc, const char *subject, ENV *env); + +struct __env +{ + natsMutex *mu; + int numSubs; + threadConfig threads; + int numPubMessages; + + bool progressiveFlush; + publishFunc pubf; + int64_t delayNano; + + subState subs[1000]; // magic number is always enough. +}; + +static void _onMessage(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure); +static void _onError(natsConnection *nc, natsSubscription *sub, natsStatus err, void *closure); +static void _onComplete(void *closure); +static void _benchMatrix(threadConfig *threadsVector, int lent, int *subsVector, int lens, int numMessages, ENV *env); +static natsStatus _bench(ENV *env, int *best, int *avg, int *worst); +static natsStatus _publish(natsConnection *nc, const char *subject, ENV *env); +static natsStatus _inject(natsConnection *nc, const char *subject, ENV *env); +static natsStatus _enqueueToSub(natsSubscription *sub, natsMsg *m); +static uint64_t _expectedSum(int N); +static uint64_t _expectedXOR(int N); + +#define RUN_MATRIX(_threads, _subs, _messages, _env) _benchMatrix(_threads, sizeof(_threads) / sizeof(*_threads), _subs, sizeof(_subs) / sizeof(*_subs), _messages, _env) + +// This benchmark publishes messages ASAP (no rate limiting) and measures +// message delivery to a few subscribers. This approach does not work well for a +// large number of subscribers because the server will be overwhelmed having to +// send too many messages at once. +void test_BenchSubscribeAsync_Small(void) +{ + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 2}, + {true, 3}, + {true, 5}, + {true, 7}, + }; + + int subs[] = {1, 2, 3, 7, 8, 13}; + + ENV env = { + .pubf = _publish, + .progressiveFlush = false, + }; + RUN_MATRIX(threads, subs, 200 * 1000, &env); +} + +// This benchmark publishes messages, flushing the connection every now and then +// to ensure the server is not overwhelmed, and measures message delivery to a +// few subscribers. This approach works well for a large number of +// subscribers, but can be too slow for a few subscriptions. +void test_BenchSubscribeAsync_Large(void) +{ + threadConfig threads[] = { + {false, 1}, + {true, 5}, + {true, 11}, + {true, 23}, + {true, 47}, + {true, 91}, + }; + + int subs[] = {1, 2, 23, 47, 81, 120}; + + ENV env = { + .pubf = _publish, + .progressiveFlush = true, + }; + + RUN_MATRIX(threads, subs, 100 * 1000, &env); +} + +// This benchmark injects the messages directly into the relevant queue for +// delivery, bypassing the publish step. +void test_BenchSubscribeAsync_Inject(void) +{ + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 2}, + {true, 3}, + {true, 7}, + {true, 11}, + {true, 19}, + {true, 163}, + }; + + int subs[] = {1, 2, 3, 5, 10, 23, 83, 163, 499}; + + ENV env = { + .pubf = _inject, + }; + + RUN_MATRIX(threads, subs, 100 * 1000, &env); +} + +// This benchmark injects the messages directly into the relevant queue for +// delivery, bypassing the publish step. It uses a delay to simulate a slow-ish +// callback. +void test_BenchSubscribeAsync_InjectSlow(void) +{ +#ifdef _WIN32 + // This test relies on nanosleep, not sure what the Windows equivalent is. Skip fr now. + printf("Skipping BenchSubscribeAsync_InjectSlow on Windows\n"); + return; + +#else + + threadConfig threads[] = { + {false, 1}, // 1 is not used in this case, just to quiet nats_SetMessageDeliveryPoolSize + {true, 1}, + {true, 2}, + {true, 3}, + {true, 7}, + {true, 11}, + {true, 79}, + {true, 163}, + }; + + int subs[] = {1, 3, 7, 23, 83, 163, 499}; + + ENV env = { + .pubf = _inject, + .delayNano = 10 * 1000, // 10µs + }; + + RUN_MATRIX(threads, subs, 10000, &env); +#endif // _WIN32 +} + +static void _benchMatrix(threadConfig *threadsVector, int lent, int *subsVector, int lens, int NMessages, ENV *env) +{ + if (natsMutex_Create(&env->mu) != NATS_OK) + { + fprintf(stderr, "Error creating mutex\n"); + exit(1); + } + printf("[\n"); + for (int *sv = subsVector; sv < subsVector + lens; sv++) + { + int numSubs = *sv; + bool uselessFromHere = false; + int numPubMessages = NMessages / numSubs; + if (numPubMessages == 0) + numPubMessages = 1; + + for (threadConfig *tv = threadsVector; tv < threadsVector + lent; tv++) + { + natsStatus s = NATS_OK; + threadConfig threads = *tv; + int best = 0, average = 0, worst = 0; + + if (threads.useGlobalDelivery) + { + if (uselessFromHere) + continue; + if (threads.max > numSubs) + uselessFromHere = true; // execute this test, but a larger MaxThreads will not make a difference. + } + + env->numSubs = numSubs; + env->numPubMessages = numPubMessages; + env->threads = threads; + for (int i = 0; i < REPEAT; i++) + { + int b = 0, a = 0, w = 0; + s = _bench(env, &b, &a, &w); + if (s != NATS_OK) + { + fprintf(stderr, "Error: %s\n", natsStatus_GetText(s)); + nats_PrintLastErrorStack(stderr); + exit(1); + } + + if ((b < best) || (best == 0)) + best = b; + if (w > worst) + worst = w; + average += a; + } + average /= REPEAT; + + const char *comma = (sv == subsVector + lens - 1) && (tv == threadsVector + lent - 1) ? "" : ","; + printf("\t{\"subs\":%d, \"threads\":%d, \"messages\":%d, \"best\":%d, \"average\":%d, \"worst\":%d}%s\n", + numSubs, env->threads.useGlobalDelivery ? env->threads.max : 0, numPubMessages * numSubs, best, average, worst, comma); + fflush(stdout); + } + } + printf("]\n"); + natsMutex_Destroy(env->mu); +} + +static natsStatus _bench(ENV *env, int *best, int *avg, int *worst) +{ + natsConnection *nc = NULL; + natsOptions *opts = NULL; + uint64_t expectedSum = _expectedSum(env->numPubMessages); + uint64_t expectedXOR = _expectedXOR(env->numPubMessages); + char subject[256]; + int64_t start, b, w, a; + + if (env->numSubs > 1000) // magic number check. + return NATS_INVALID_ARG; + memset(env->subs, 0, sizeof(subState) * 1000); + for (int i = 0; i < env->numSubs; i++) + env->subs[i].env = env; // set the environment to access it in the callbacks. + + natsPid pid = _startServer("nats://127.0.0.1:4222", NULL, true); + if (pid == NATS_INVALID_PID) + return NATS_ERR; + + natsStatus s = nats_Open(-1); + IFOK(s, natsNUID_Next(subject, NUID_BUFFER_LEN + 1)); + IFOK(s, natsOptions_Create(&opts)); + IFOK(s, nats_SetMessageDeliveryPoolSize(env->threads.max)); + IFOK(s, natsOptions_SetErrorHandler(opts, _onError, NULL)); + IFOK(s, natsOptions_UseGlobalMessageDelivery(opts, env->threads.useGlobalDelivery)); + + IFOK(s, natsConnection_Connect(&nc, opts)); + + for (int i = 0; i < env->numSubs; i++) + { + IFOK(s, natsConnection_Subscribe(&(env->subs[i].sub), nc, subject, _onMessage, &env->subs[i])); + IFOK(s, natsSubscription_SetPendingLimits(env->subs[i].sub, -1, -1)); + IFOK(s, natsSubscription_AutoUnsubscribe(env->subs[i].sub, env->numPubMessages)); + IFOK(s, natsSubscription_SetOnCompleteCB(env->subs[i].sub, _onComplete, &env->subs[i])); + } + + start = nats_Now(); + + // Publish or inject the messages! + IFOK(s, env->pubf(nc, subject, env)); + + while (s == NATS_OK) + { + bool done = true; + for (int i = 0; i < env->numSubs; i++) + { + // threads don't touch this, should be safe + if (natsSubscription_IsValid(env->subs[i].sub)) + { + done = false; + break; + } + } + + nats_Sleep(10); + if (done) + break; + } + + b = w = a = 0; + natsMutex_Lock(env->mu); + if (s == NATS_OK) + { + for (int i = 0; i < env->numSubs; i++) + { + if (env->subs[i].sum != expectedSum) + { + s = NATS_ERR; + fprintf(stderr, "Error: sum is %" PRId64 " for sub %d, expected %" PRId64 "\n", env->subs[i].sum, i, expectedSum); + break; + } + if (env->subs[i].xor != expectedXOR) + { + fprintf(stderr, "Error: xor is %" PRId64 " for sub %d, expected %" PRId64 "\n", env->subs[i].xor, i, expectedXOR); + s = NATS_ERR; + break; + } + if ((int)(env->subs[i].count) != env->numPubMessages) + { + fprintf(stderr, "Error: count is %" PRId64 " for sub %d, expected %d\n", env->subs[i].count, i, env->numPubMessages); + s = NATS_ERR; + break; + } + + int64_t dur = env->subs[i].closedTimestamp - start; + if (dur > w) + w = dur; + if ((dur < b) || (b == 0)) + b = dur; + a += dur; + } + } + natsMutex_Unlock(env->mu); + + // cleanup + for (int i = 0; i < env->numSubs; i++) + natsSubscription_Destroy(env->subs[i].sub); + natsConnection_Destroy(nc); + natsOptions_Destroy(opts); + _stopServer(pid); + nats_CloseAndWait(0); + + *best = (int)b; + *avg = (int)(a / env->numSubs); + *worst = (int)w; + + return s; +} + +static void _onMessage(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure) +{ + subState *ss = (subState *)closure; + +#ifndef _WIN32 + if (ss->env->delayNano > 0) + { + struct timespec wait = {0, ss->env->delayNano}; + nanosleep(&wait, NULL); + } +#endif + + char buf[32]; + int len = natsMsg_GetDataLength(msg); + if (len > 31) + len = 31; + + strncpy(buf, natsMsg_GetData(msg), len); + buf[len] = '\0'; + int64_t val = atoi(buf); + + ss->sum += val; + ss->xor ^= val; + ss->count++; + + natsMsg_Destroy(msg); +} + +static void _onComplete(void *closure) +{ + subState *ss = (subState *)closure; + natsMutex_Lock(ss->env->mu); + ss->closedTimestamp = nats_Now(); + natsMutex_Unlock(ss->env->mu); +} + +static natsStatus _publish(natsConnection *nc, const char *subject, ENV *env) +{ + natsStatus s = NATS_OK; + char buf[16]; + + int flushAfter = env->progressiveFlush ? env->numPubMessages / (env->numSubs * 3) : // trigger + env->numPubMessages + 1; // do not trigger + for (int i = 0; i < env->numPubMessages; i++) + { + snprintf(buf, sizeof(buf), "%d", i); + IFOK(s, natsConnection_PublishString(nc, subject, buf)); + + if (((i != 0) && ((i % flushAfter) == 0)) || // progressive flush + (i == (env->numPubMessages - 1))) // last message in batch + { + IFOK(s, natsConnection_Flush(nc)); + } + } + + return s; +} + +static natsStatus _inject(natsConnection *nc, const char *subject, ENV *env) +{ + natsStatus s = NATS_OK; + natsMsg *m = NULL; + char buf[16]; + + for (int i = 0; i < env->numPubMessages; i++) + { + for (int n = 0; n < env->numSubs; n++) + { + snprintf(buf, sizeof(buf), "%d", i); + + s = natsMsg_Create(&m, subject, NULL, buf, (int)strlen(buf)); + natsSubscription *sub = env->subs[n].sub; + nats_lockSubAndDispatcher(sub); + IFOK(s, natsSub_enqueueUserMessage(sub, m)); + nats_unlockSubAndDispatcher(sub); + } + } + + return s; +} + +static uint64_t _expectedSum(int N) +{ + uint64_t sum = 0; + for (int64_t i = 0; i < N; i++) + sum += i; + return sum; +} + +static uint64_t _expectedXOR(int N) +{ + uint64_t xor = 0; + for (int64_t i = 0; i < N; i++) + xor ^= i; + return xor; +} + +static void _onError(natsConnection *nc, natsSubscription *sub, natsStatus err, void *closure) +{ + int64_t dropped = 0; + natsSubscription_GetDropped(sub, (int64_t *)&dropped); + printf("Async error: sid:%" PRId64 ", dropped:%" PRId64 ": %u - %s\n", sub->sid, dropped, err, natsStatus_GetText(err)); +} diff --git a/test/diffstat_sub_async.go b/test/diffstat_sub_async.go new file mode 100644 index 000000000..67e039668 --- /dev/null +++ b/test/diffstat_sub_async.go @@ -0,0 +1,193 @@ +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "regexp" + "sort" + "strings" +) + +const NOISE_THRESHOLD = 0.03 + +type Key struct { + Subs int `json:"subs"` + Threads int `json:"threads"` + Messages int `json:"messages"` +} + +type TestData struct { + Key + Best int `json:"best"` + Average int `json:"average"` + Worst int `json:"worst"` +} + +type Diff struct { + Key + BaseAverage int `json:"base"` + BranchAverage int `json:"branch"` + Diff float64 `json:"diff"` +} + +type DiffData struct { + Records []Diff `json:"records"` + + Total struct { + BestDiff float64 `json:"best"` + AverageDiff float64 `json:"average"` + WorstDiff float64 `json:"worst"` + } `json:"total"` +} + +func main() { + flag.Parse() + + if len(os.Args) != 3 { + log.Fatalf("usage: %s
", os.Args[0]) + } + + m, err := readFile(os.Args[1]) + if err != nil { + log.Fatal(err) + } + b, err := readFile(os.Args[2]) + if err != nil { + log.Fatal(err) + } + + diff := map[string]*DiffData{} + for benchName := range b { + if _, ok := m[benchName]; !ok { + log.Printf("missing bench %s in main data", benchName) + continue + } + diff[benchName], err = calculateDiff(m[benchName], b[benchName]) + if err != nil { + log.Fatal(err) + } + } + + // bb, err := json.MarshalIndent(diff, "", " ") + // if err != nil { + // log.Fatal(err) + // } + // fmt.Println(string(bb)) + + for key, d := range diff { + fmt.Printf("== %s ==\n", key) + fmt.Printf("Best: %.2f%%\n", d.Total.BestDiff*100) + fmt.Printf("Average: %.2f%%\n", d.Total.AverageDiff*100) + fmt.Printf("Worst: %.2f%%\n", d.Total.WorstDiff*100) + fmt.Println() + for _, r := range d.Records { + fmt.Printf("subs=%d threads=%d messages=%d base=%d branch=%d diff=%.2f%%\n", + r.Subs, r.Threads, r.Messages, r.BaseAverage, r.BranchAverage, r.Diff*100) + } + } +} + +func calculateDiff(main, bench map[Key]TestData) (*DiffData, error) { + diff := DiffData{} + mBestSum, mAverageSum, mWorstSum := 0, 0, 0 + bBestSum, bAverageSum, bWorstSum := 0, 0, 0 + + for key, b := range bench { + m, ok := main[key] + if !ok { + log.Printf("warning: missing key %+v in main data", key) + continue + } + + // Exclude records with less than .5% difference from the output + d := float64(b.Average-m.Average) / float64(m.Average) + if d >= NOISE_THRESHOLD || d <= -NOISE_THRESHOLD { + diff.Records = append(diff.Records, Diff{ + Key: Key{ + Subs: m.Subs, + Threads: m.Threads, + Messages: m.Messages, + }, + BaseAverage: m.Average, + BranchAverage: b.Average, + Diff: d, + }) + } + + mBestSum += m.Best + mAverageSum += m.Average + mWorstSum += m.Worst + bBestSum += b.Best + bAverageSum += b.Average + bWorstSum += b.Worst + } + + sort.Slice(diff.Records, func(i, j int) bool { + return diff.Records[i].Diff > diff.Records[j].Diff + }) + + diff.Total.WorstDiff = float64(bWorstSum-mWorstSum) / float64(mWorstSum) + diff.Total.AverageDiff = float64(bAverageSum-mAverageSum) / float64(mAverageSum) + diff.Total.BestDiff = float64(bBestSum-mBestSum) / float64(mBestSum) + return &diff, nil +} + +func readFile(path string) (map[string]map[Key]TestData, error) { + r, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + scanner := bufio.NewScanner(r) + result := make(map[string]map[Key]TestData) + var benchName string + re := regexp.MustCompile(`^\d+: (.*)`) + + for scanner.Scan() { + line := scanner.Text() + + // ignore irrelevant lines, strip the test # prefix + matches := re.FindStringSubmatch(line) + if matches == nil { + continue + } + line = matches[1] + + if strings.HasPrefix(line, "== ") && strings.HasSuffix(line, " ==") { + benchName = strings.TrimSpace(strings.TrimPrefix(strings.TrimSuffix(line, " =="), "== ")) + continue + } + + line = strings.TrimPrefix(line, "\x1b[0;0m") + if strings.HasPrefix(line, "[") { + var data []TestData + jsonData := strings.Join([]string{line}, "") + for scanner.Scan() { + line := scanner.Text() + if matches := re.FindStringSubmatch(line); matches != nil { + line = matches[1] + jsonData += line + if strings.HasSuffix(line, "]") { + break + } + } + } + if err := json.Unmarshal([]byte(jsonData), &data); err != nil { + return nil, fmt.Errorf("%s: failed to parse JSON data: %w", path, err) + } + + hash := make(map[Key]TestData) + for _, d := range data { + hash[d.Key] = d + } + if benchName != "" { + result[benchName] = hash + } + } + } + + return result, nil +} diff --git a/test/list.h b/test/list.h new file mode 100644 index 000000000..7b0e368b5 --- /dev/null +++ b/test/list.h @@ -0,0 +1,30 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef _TEST_PROTO +#undef _test +#define _test(name) void test_##name(void); +#endif + +#ifdef _TEST_LIST +#undef _test +#define _test(name) {#name, test_##name}, +#endif + +#include "list_test.txt" +#include "list_bench.txt" +#if defined(NATS_HAS_STREAMING) +#include "list_stan.txt" +#endif + +#undef _test diff --git a/test/list.txt b/test/list.txt deleted file mode 100644 index 60be59288..000000000 --- a/test/list.txt +++ /dev/null @@ -1,301 +0,0 @@ -Version -VersionMatchesTag -OpenCloseAndWait -natsNowAndSleep -natsAllocSprintf -natsStrCaseStr -natsSnprintf -natsBuffer -natsParseInt64 -natsParseControl -natsNormalizeErr -natsMutex -natsThread -natsCondition -natsTimer -natsUrl -natsCreateStringFromBuffer -natsHash -natsHashing -natsStrHash -natsInbox -natsOptions -natsSock_ConnectTcp -natsSock_ShuffleIPs -natsSock_IPOrder -natsSock_ReadLine -natsJSON -natsEncodeTimeUTC -natsErrWithLongText -natsErrStackMoreThanMaxFrames -natsMsg -natsBase32 -natsBase64 -natsCRC16 -natsKeys -natsReadFile -natsGetJWTOrSeed -natsHostIsIP -natsWaitReady -natsSign -HeadersLift -HeadersAPIs -MsgIsJSControl -SrvVersionAtLeast -FormatStringArray -ReconnectServerStats -ParseStateReconnectFunctionality -ServersRandomize -SelectNextServer -ParserPing -ParserErr -ParserOK -ParseINFO -ParserShouldFail -ParserSplitMsg -ProcessMsgArgs -LibMsgDelivery -AsyncINFO -RequestPool -NoFlusherIfSendAsapOption -HeadersAndSubPendingBytes -DefaultConnection -SimplifiedURLs -IPResolutionOrder -UseDefaultURLIfNoServerSpecified -ConnectToWithMultipleURLs -ConnectionWithNULLOptions -ConnectionToWithNullURLs -ConnectionStatus -ConnClosedCB -CloseDisconnectedCB -ServerStopDisconnectedCB -ClosedConnections -ConnectVerboseOption -ReconnectThreadLeak -ReconnectTotalTime -ReconnectDisallowedFlags -ReconnectAllowedFlags -ConnCloseBreaksReconnectLoop -BasicReconnectFunctionality -ExtendedReconnectFunctionality -QueueSubsOnReconnect -IsClosed -IsReconnectingAndStatus -ReconnectBufSize -RetryOnFailedConnect -NoPartialOnReconnect -ReconnectFailsPendingRequests -ForcedReconnect -ErrOnConnectAndDeadlock -ErrOnMaxPayloadLimit -Auth -AuthFailNoDisconnectCB -AuthToken -AuthTokenHandler -PermViolation -AuthViolation -AuthenticationExpired -AuthenticationExpiredReconnect -ConnectedServer -MultipleClose -SimplePublish -SimplePublishNoData -PublishMsg -InvalidSubsArgs -AsyncSubscribe -AsyncSubscribeTimeout -SyncSubscribe -PubSubWithReply -NoResponders -Flush -ConnCloseDoesFlush -QueueSubscriber -ReplyArg -SyncReplyArg -Unsubscribe -DoubleUnsubscribe -SubRemovedWhileProcessingMsg -RequestTimeout -Request -RequestNoBody -RequestMuxWithMappedSubject -OldRequest -SimultaneousRequests -RequestClose -CustomInbox -MessagePadding -FlushInCb -ReleaseFlush -FlushErrOnDisconnect -Inbox -Stats -BadSubject -SubBadSubjectAndQueueNames -ClientAsyncAutoUnsub -ClientSyncAutoUnsub -ClientAutoUnsubAndReconnect -AutoUnsubNoUnsubOnDestroy -NextMsgOnClosedSub -CloseSubRelease -IsValidSubscriber -SlowSubscriber -SlowAsyncSubscriber -SlowConsumerCb -PendingLimitsDeliveredAndDropped -PendingLimitsWithSyncSub -AsyncSubscriptionPending -AsyncSubscriptionPendingDrain -SyncSubscriptionPending -SyncSubscriptionPendingDrain -AsyncErrHandlerMaxPendingMsgs -AsyncErrHandlerMaxPendingBytes -AsyncErrHandlerSubDestroyed -AsyncSubscriberStarvation -AsyncSubscriberOnClose -NextMsgCallOnAsyncSub -SubOnComplete -GetLastError -StaleConnection -ServerErrorClosesConnection -NoEcho -NoEchoOldServer -DrainSub -DrainSubStops -DrainSubRaceOnAutoUnsub -DrainSubNotResentOnReconnect -DrainConn -NoDoubleCloseCbOnDrain -GetClientID -GetClientIP -GetRTT -GetLocalIPAndPort -UserCredsCallbacks -UserCredsFromFiles -UserCredsFromMemory -NKey -NKeyFromSeed -ConnSign -WriteDeadline -HeadersNotSupported -HeadersBasic -MsgsFilter -EventLoop -EventLoopRetryOnFailedConnect -EventLoopTLS -SSLBasic -SSLVerify -SSLCAFromMemory -SSLCertAndKeyFromMemory -SSLVerifyHostname -SSLSkipServerVerification -SSLCiphers -SSLMultithreads -SSLConnectVerboseOption -SSLSocketLeakEventLoop -SSLReconnectWithAuthError -SSLAvailable -ServersOption -AuthServers -AuthFailToReconnect -ReconnectWithTokenHandler -BasicClusterReconnect -HotSpotReconnect -ProperReconnectDelay -ProperFalloutAfterMaxAttempts -StopReconnectAfterTwoAuthErr -TimeoutOnNoServer -PingReconnect -GetServers -GetDiscoveredServers -DiscoveredServersCb -IgnoreDiscoveredServers -INFOAfterFirstPONGisProcessedOK -ServerPoolUpdatedOnClusterUpdate -ReconnectJitter -CustomReconnectDelay -LameDuckMode -ReconnectImplicitUserInfo -JetStreamUnmarshalAccInfo -JetStreamUnmarshalStreamState -JetStreamUnmarshalStreamCfg -JetStreamUnmarshalStreamInfo -JetStreamMarshalStreamCfg -JetStreamUnmarshalConsumerInfo -JetStreamContext -JetStreamDomain -JetStreamMgtStreams -JetStreamMgtConsumers -JetStreamPublish -JetStreamPublishAsync -JetStreamPublishAckHandler -JetStreamSubscribe -JetStreamSubscribeSync -JetStreamSubscribeConfigCheck -JetStreamSubscribeIdleHeartbeat -JetStreamSubscribeFlowControl -JetStreamSubscribePull -JetStreamSubscribeHeadersOnly -JetStreamOrderedCons -JetStreamOrderedConsWithErrors -JetStreamOrderedConsAutoUnsub -JetStreamOrderedConsSrvRestart -JetStreamSubscribeWithFWC -JetStreamStreamsSealAndRollup -JetStreamGetMsgAndLastMsg -JetStreamConvertDirectMsg -JetStreamDirectGetMsg -JetStreamNakWithDelay -JetStreamBackOffRedeliveries -JetStreamInfoWithSubjects -JetStreamInfoAlternates -KeyValueManager -KeyValueBasics -KeyValueWatch -KeyValueWatchMulti -KeyValueHistory -KeyValueKeys -KeyValueDeleteVsPurge -KeyValueDeleteTombstones -KeyValueDeleteMarkerThreshold -KeyValueCrossAccount -KeyValueDiscardOldToNew -KeyValueRePublish -KeyValueMirrorDirectGet -KeyValueMirrorCrossDomains -MicroMatchEndpointSubject -MicroAddService -MicroGroups -MicroBasics -MicroStartStop -MicroServiceStopsOnClosedConn -MicroServiceStopsWhenServerStops -MicroAsyncErrorHandlerMaxPendingMsgs -MicroAsyncErrorHandlerMaxPendingBytes -StanPBufAllocator -StanConnOptions -StanSubOptions -StanMsg -StanServerNotReachable -StanBasicConnect -StanConnectError -StanBasicPublish -StanBasicPublishAsync -StanPublishTimeout -StanPublishMaxAcksInflight -StanBasicSubscription -StanSubscriptionCloseAndUnsub -StanDurableSubscription -StanBasicQueueSubscription -StanDurableQueueSubscription -StanCheckReceivedMsg -StanSubscriptionAckMsg -StanPings -StanPingsNoResponder -StanConnectionLostHandlerNotSet -StanPingsUnblockPublishCalls -StanGetNATSConnection -StanNoRetryOnFailedConnect -StanInternalSubsNotPooled -StanSubOnComplete -StanSubTimeout diff --git a/test/list_bench.txt b/test/list_bench.txt new file mode 100644 index 000000000..2c51bb260 --- /dev/null +++ b/test/list_bench.txt @@ -0,0 +1,4 @@ +_test(BenchSubscribeAsync_Large) +_test(BenchSubscribeAsync_Small) +_test(BenchSubscribeAsync_Inject) +_test(BenchSubscribeAsync_InjectSlow) diff --git a/test/list_stan.txt b/test/list_stan.txt new file mode 100644 index 000000000..b4cd28666 --- /dev/null +++ b/test/list_stan.txt @@ -0,0 +1,27 @@ +_test(StanBasicConnect) +_test(StanBasicPublish) +_test(StanBasicPublishAsync) +_test(StanBasicQueueSubscription) +_test(StanBasicSubscription) +_test(StanCheckReceivedMsg) +_test(StanConnectError) +_test(StanConnectionLostHandlerNotSet) +_test(StanConnOptions) +_test(StanDurableQueueSubscription) +_test(StanDurableSubscription) +_test(StanGetNATSConnection) +_test(StanInternalSubsNotPooled) +_test(StanMsg) +_test(StanNoRetryOnFailedConnect) +_test(StanPBufAllocator) +_test(StanPings) +_test(StanPingsNoResponder) +_test(StanPingsUnblockPubCalls) +_test(StanPublishMaxAcksInflight) +_test(StanPublishTimeout) +_test(StanServerNotReachable) +_test(StanSubOnComplete) +_test(StanSubOptions) +_test(StanSubscriptionAckMsg) +_test(StanSubscriptionCloseAndUnsubscribe) +_test(StanSubTimeout) diff --git a/test/list_test.txt b/test/list_test.txt new file mode 100644 index 000000000..496d5b3f8 --- /dev/null +++ b/test/list_test.txt @@ -0,0 +1,276 @@ +_test(AssignSubToDispatch) +_test(AsyncErrHandlerMaxPendingBytes) +_test(AsyncErrHandlerMaxPendingMsgs) +_test(AsyncErrHandlerSubDestroyed) +_test(AsyncINFO) +_test(AsyncSubscribe) +_test(AsyncSubscriberOnClose) +_test(AsyncSubscriberStarvation) +_test(AsyncSubscribeTimeout) +_test(AsyncSubscriptionPending) +_test(AsyncSubscriptionPendingDrain) +_test(Auth) +_test(AuthenticationExpired) +_test(AuthenticationExpiredReconnect) +_test(AuthFailNoDisconnectCB) +_test(AuthFailToReconnect) +_test(AuthServers) +_test(AuthToken) +_test(AuthTokenHandler) +_test(AuthViolation) +_test(AutoUnsubNoUnsubOnDestroy) +_test(BadSubject) +_test(BasicClusterReconnect) +_test(BasicReconnectFunctionality) +_test(ClientAsyncAutoUnsub) +_test(ClientAutoUnsubAndReconnect) +_test(ClientSyncAutoUnsub) +_test(ClosedConnections) +_test(CloseDisconnectedCB) +_test(CloseSubRelease) +_test(ConnCloseBreaksReconnectLoop) +_test(ConnClosedCB) +_test(ConnCloseDoesFlush) +_test(ConnectedServer) +_test(ConnectionStatus) +_test(ConnectionToWithNullURLs) +_test(ConnectionWithNullOptions) +_test(ConnectToWithMultipleURLs) +_test(ConnectVerboseOption) +_test(ConnSign) +_test(CustomInbox) +_test(CustomReconnectDelay) +_test(DefaultConnection) +_test(DiscoveredServersCb) +_test(DoubleUnsubscribe) +_test(DrainConn) +_test(DrainSub) +_test(DrainSubNotResentOnReconnect) +_test(DrainSubRaceOnAutoUnsub) +_test(DrainSubStops) +_test(ErrOnConnectAndDeadlock) +_test(ErrOnMaxPayloadLimit) +_test(EventLoop) +_test(EventLoopRetryOnFailedConnect) +_test(EventLoopTLS) +_test(ExtendedReconnectFunctionality) +_test(Flush) +_test(FlushErrOnDisconnect) +_test(FlushInCb) +_test(ForcedReconnect) +_test(GetClientID) +_test(GetClientIP) +_test(GetDiscoveredServers) +_test(GetLastError) +_test(GetLocalIPAndPort) +_test(GetRTT) +_test(GetServers) +_test(HeadersAndSubPendingBytes) +_test(HeadersBasic) +_test(HeadersLift) +_test(HeadersNotSupported) +_test(HotSpotReconnect) +_test(IgnoreDiscoveredServers) +_test(Inbox) +_test(InvalidSubsArgs) +_test(IPResolutionOrder) +_test(IsClosed) +_test(IsReconnectingAndStatus) +_test(IsValidSubscriber) +_test(JetStreamBackOffRedeliveries) +_test(JetStreamContext) +_test(JetStreamContextDomain) +_test(JetStreamConvertDirectMsg) +_test(JetStreamDirectGetMsg) +_test(JetStreamGetMsgAndLastMsg) +_test(JetStreamInfoAlternates) +_test(JetStreamInfoWithSubjects) +_test(JetStreamMarshalStreamConfig) +_test(JetStreamMgtConsumers) +_test(JetStreamMgtStreams) +_test(JetStreamNakWithDelay) +_test(JetStreamOrderedConsSrvRestart) +_test(JetStreamOrderedConsumer) +_test(JetStreamOrderedConsumerWithAutoUnsub) +_test(JetStreamOrderedConsumerWithErrors) +_test(JetStreamPublish) +_test(JetStreamPublishAckHandler) +_test(JetStreamPublishAsync) +_test(JetStreamStreamsSealAndRollup) +_test(JetStreamSubscribe) +_test(JetStreamSubscribeConfigCheck) +_test(JetStreamSubscribeFlowControl) +_test(JetStreamSubscribeHeadersOnly) +_test(JetStreamSubscribeIdleHearbeat) +_test(JetStreamSubscribePull) +_test(JetStreamSubscribeSync) +_test(JetStreamSubscribeWithFWC) +_test(JetStreamUnmarshalAccountInfo) +_test(JetStreamUnmarshalConsumerInfo) +_test(JetStreamUnmarshalStreamConfig) +_test(JetStreamUnmarshalStreamInfo) +_test(JetStreamUnmarshalStreamState) +_test(KeyValueBasics) +_test(KeyValueCrossAccount) +_test(KeyValueDeleteTombstones) +_test(KeyValueDeleteVsPurge) +_test(KeyValueDiscardOldToNew) +_test(KeyValueHistory) +_test(KeyValueKeys) +_test(KeyValueManager) +_test(KeyValueMirrorCrossDomains) +_test(KeyValueMirrorDirectGet) +_test(KeyValuePurgeDeletesMarkerThreshold) +_test(KeyValueRePublish) +_test(KeyValueWatch) +_test(KeyValueWatchMulti) +_test(LameDuckMode) +_test(MessageBufferPadding) +_test(MicroAddService) +_test(MicroAsyncErrorHandlerMaxPendingBytes) +_test(MicroAsyncErrorHandlerMaxPendingMsgs) +_test(MicroBasics) +_test(MicroGroups) +_test(MicroMatchEndpointSubject) +_test(MicroServiceStopsOnClosedConn) +_test(MicroServiceStopsWhenServerStops) +_test(MicroStartStop) +_test(MultipleClose) +_test(natsAllocSprintf) +_test(natsBase32Decode) +_test(natsBase64Encode) +_test(natsBuffer) +_test(natsCondition) +_test(natsCRC16) +_test(natsCreateStringFromBuffer) +_test(natsEncodeTimeUTC) +_test(natsErrStackMoreThanMaxFrames) +_test(natsErrWithLongText) +_test(natsFormatStringArray) +_test(natsGetJWTOrSeed) +_test(natsHash) +_test(natsHashing) +_test(natsHostIsIP) +_test(natsInbox) +_test(natsJSON) +_test(natsKeys) +_test(natsMsg) +_test(natsMsgHeaderAPIs) +_test(natsMsgIsJSCtrl) +_test(natsMsgsFilter) +_test(natsMutex) +_test(natsNormalizeErr) +_test(natsNowAndSleep) +_test(natsOptions) +_test(natsParseControl) +_test(natsParseInt64) +_test(natsReadFile) +_test(natsSign) +_test(natsSnprintf) +_test(natsSock_ConnectTcp) +_test(natsSock_IPOrder) +_test(natsSock_ReadLine) +_test(natsSock_ShuffleIPs) +_test(natsSrvVersionAtLeast) +_test(natsStrCaseStr) +_test(natsStrHash) +_test(natsThread) +_test(natsTimer) +_test(natsUrl) +_test(natsWaitReady) +_test(NextMsgCallOnAsyncSub) +_test(NextMsgOnClosedSub) +_test(NKey) +_test(NKeyFromSeed) +_test(NoDoubleConnClosedOnDrain) +_test(NoEcho) +_test(NoEchoOldServer) +_test(NoFlusherIfSendAsap) +_test(NoPartialOnReconnect) +_test(NoResponders) +_test(OldRequest) +_test(OpenCloseAndWait) +_test(ParseINFO) +_test(ParserErr) +_test(ParserOK) +_test(ParserPing) +_test(ParserShouldFail) +_test(ParserSplitMsg) +_test(ParseStateReconnectFunctionality) +_test(PendingLimitsDeliveredAndDropped) +_test(PendingLimitsWithSyncSub) +_test(PermViolation) +_test(PingReconnect) +_test(ProcessMsgArgs) +_test(ProperFalloutAfterMaxAttempts) +_test(ProperReconnectDelay) +_test(PublishMsg) +_test(PubSubWithReply) +_test(QueueSubscriber) +_test(QueueSubsOnReconnect) +_test(ReceiveINFORightAfterFirstPONG) +_test(ReconnectAllowedFlags) +_test(ReconnectBufSize) +_test(ReconnectDisallowedFlags) +_test(ReconnectFailsPendingRequest) +_test(ReconnectImplicitUserInfo) +_test(ReconnectJitter) +_test(ReconnectServerStats) +_test(ReconnectThreadLeak) +_test(ReconnectTotalTime) +_test(ReconnectWithTokenHandler) +_test(ReleaseFlush) +_test(ReplyArg) +_test(Request) +_test(RequestClose) +_test(RequestMuxWithMappedSubject) +_test(RequestNoBody) +_test(RequestPool) +_test(RequestTimeout) +_test(RetryOnFailedConnect) +_test(SelectNextServer) +_test(ServerErrorClosesConnection) +_test(ServerPoolUpdatedOnClusterUpdate) +_test(ServersOption) +_test(ServersRandomize) +_test(ServerStopDisconnectedCB) +_test(SimplePublish) +_test(SimplePublishNoData) +_test(SimplifiedURLs) +_test(SimultaneousRequest) +_test(SlowAsyncSubscriber) +_test(SlowConsumerCB) +_test(SlowSubscriber) +_test(SSLAvailable) +_test(SSLBasic) +_test(SSLCertAndKeyFromMemory) +_test(SSLCiphers) +_test(SSLConnectVerboseOption) +_test(SSLHandshakeFirst) +_test(SSLServerNameIndication) +_test(SSLLoadCAFromMemory) +_test(SSLMultithreads) +_test(SSLReconnectWithAuthError) +_test(SSLSkipServerVerification) +_test(SSLSocketLeakWithEventLoop) +_test(SSLVerify) +_test(SSLVerifyHostname) +_test(StaleConnection) +_test(Stats) +_test(StopReconnectAfterTwoAuthErr) +_test(SubBadSubjectAndQueueName) +_test(SubOnComplete) +_test(SubRemovedWhileProcessingMsg) +_test(SyncReplyArg) +_test(SyncSubscribe) +_test(SyncSubscriptionPending) +_test(SyncSubscriptionPendingDrain) +_test(TimeoutOnNoServer) +_test(Unsubscribe) +_test(UseDefaultURLIfNoServerSpecified) +_test(UserCredsCallbacks) +_test(UserCredsFromFiles) +_test(UserCredsFromMemory) +_test(Version) +_test(VersionMatchesTag) +_test(WriteDeadline) diff --git a/test/test.c b/test/test.c index a2c898038..5c46669a4 100644 --- a/test/test.c +++ b/test/test.c @@ -1,4 +1,4 @@ -// Copyright 2015-2023 The NATS Authors +// Copyright 2015-2024 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "natsp.h" +#include "test.h" #include #include @@ -33,51 +33,69 @@ #include "sub.h" #include "msg.h" #include "stats.h" -#include "comsock.h" #include "crypto.h" #include "nkeys.h" #include "parser.h" #include "js.h" #include "kv.h" #include "microp.h" +#include "glib/glibp.h" + #if defined(NATS_HAS_STREAMING) + #include "stan/conn.h" #include "stan/pub.h" #include "stan/sub.h" #include "stan/copts.h" #include "stan/sopts.h" -#endif + +static const char *clientName = "client"; + +#endif // NATS_HAS_STREAMING + +typedef void (*testFunc)(void); + +typedef struct __testInfo +{ + const char *name; + testFunc func; +} testInfo; + +#define _TEST_PROTO +#include "list.h" +#undef _TEST_PROTO + +#define _TEST_LIST +testInfo allTests[] = +{ +#include "list.h" +}; +#undef _TEST_LIST static int tests = 0; -static bool failed = false; +bool failed = false; -static bool keepServerOutput = false; +bool keepServerOutput = false; static bool valgrind = false; static bool runOnTravis = false; -static const char *natsServerExe = "nats-server"; static const char *serverVersion = NULL; static const char *natsStreamingServerExe = "nats-streaming-server"; -static natsMutex *slMu = NULL; -static natsHash *slMap = NULL; +natsMutex *slMu = NULL; +natsHash *slMap = NULL; + +#define test(s) { printf("#%02d ", ++tests); printf("%s\n", (s)); fflush(stdout); } +#define testf(s, ...) { printf("#%02d ", ++tests); printf((s "\n"), __VA_ARGS__); fflush(stdout); } -#define test(s) { printf("#%02d ", ++tests); printf("%s", (s)); fflush(stdout); } #ifdef _WIN32 -#define NATS_INVALID_PID (NULL) #define testCond(c) if(c) { printf("PASSED\n"); fflush(stdout); } else { printf("FAILED\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; return; } #define testCondNoReturn(c) if(c) { printf("PASSED\n"); fflush(stdout); } else { printf("FAILED\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; } -#define LOGFILE_NAME "wserver.log" #else -#define NATS_INVALID_PID (-1) #define testCond(c) if(c) { printf("\033[0;32mPASSED\033[0;0m\n"); fflush(stdout); } else { printf("\033[0;31mFAILED\033[0;0m\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; return; } #define testCondNoReturn(c) if(c) { printf("\033[0;32mPASSED\033[0;0m\n"); fflush(stdout); } else { printf("\033[0;31mFAILED\033[0;0m\n"); nats_PrintLastErrorStack(stdout); fflush(stdout); failed=true; } -#define LOGFILE_NAME "server.log" #endif -#define FAIL(m) { printf("@@ %s @@\n", (m)); failed=true; return; } - -#define CHECK_SERVER_STARTED(p) if ((p) == NATS_INVALID_PID) FAIL("Unable to start or verify that the server was started!") static const char *testServers[] = {"nats://127.0.0.1:1222", "nats://127.0.0.1:1223", @@ -87,11 +105,6 @@ static const char *testServers[] = {"nats://127.0.0.1:1222", "nats://127.0.0.1:1227", "nats://127.0.0.1:1228"}; -#if defined(NATS_HAS_STREAMING) -static const char *clusterName = "test-cluster"; -static const char *clientName = "client"; -#endif - // Forward declaration static void _startMockupServerThread(void *closure); static void _createConfFile(char *buf, int bufLen, const char *content); @@ -112,6 +125,7 @@ struct threadArg natsStrHash *inboxes; natsStatus status; const char* string; + int N; bool connected; bool disconnected; int64_t disconnectedAt[4]; @@ -221,8 +235,7 @@ _destroyDefaultThreadArgs(struct threadArg *args) natsCondition_Destroy(args->c); } -static void -test_natsNowAndSleep(void) +void test_natsNowAndSleep(void) { int64_t start; int64_t end; @@ -234,8 +247,7 @@ test_natsNowAndSleep(void) testCond(((end - start) >= 990) && ((end - start) <= 1010)); } -static void -test_natsAllocSprintf(void) +void test_natsAllocSprintf(void) { char smallStr[20]; char mediumStr[256]; // This is the size of the temp buffer in nats_asprintf @@ -277,8 +289,7 @@ test_natsAllocSprintf(void) ptr = NULL; } -static void -test_natsStrCaseStr(void) +void test_natsStrCaseStr(void) { const char *s1 = "Hello World!"; const char *s2 = "wo"; @@ -301,8 +312,7 @@ test_natsStrCaseStr(void) testCond(res == NULL); } -static void -test_natsSnprintf(void) +void test_natsSnprintf(void) { #if _WIN32 // This test is specific to older version of Windows @@ -318,7 +328,7 @@ test_natsSnprintf(void) #endif } -static void test_natsBuffer(void) +void test_natsBuffer(void) { natsStatus s; char backend[10]; @@ -595,8 +605,7 @@ static void test_natsBuffer(void) buf = NULL; } -static void -test_natsParseInt64(void) +void test_natsParseInt64(void) { int64_t n; @@ -657,8 +666,7 @@ test_natsParseInt64(void) testCond(n == -1); } -static void -test_natsParseControl(void) +void test_natsParseControl(void) { natsStatus s; natsControl c; @@ -734,8 +742,7 @@ test_natsParseControl(void) c.args = NULL; } -static void -test_natsNormalizeErr(void) +void test_natsNormalizeErr(void) { char error[1024]; char expected[256]; @@ -783,8 +790,7 @@ test_natsNormalizeErr(void) testCond(error[0] == '\0'); } -static void -test_natsMutex(void) +void test_natsMutex(void) { natsStatus s; natsMutex *m = NULL; @@ -842,8 +848,7 @@ sumThread(void *arg) static int NUM_THREADS = 1000; -static void -test_natsThread(void) +void test_natsThread(void) { natsStatus s = NATS_OK; natsMutex *m = NULL; @@ -967,8 +972,7 @@ _unblockLongWait(void *closure) natsMutex_Unlock(args->m); } -static void -test_natsCondition(void) +void test_natsCondition(void) { natsStatus s; natsMutex *m = NULL; @@ -1184,8 +1188,7 @@ _timerStopCB(natsTimer *timer, void *arg) natsCondition_Wait(tArg.c, tArg.m); \ natsMutex_Unlock(tArg.m) -static void -test_natsTimer(void) +void test_natsTimer(void) { natsStatus s; natsTimer *t = NULL; @@ -1405,8 +1408,7 @@ test_natsTimer(void) testCond(s == NATS_OK); } -static void -test_natsUrl(void) +void test_natsUrl(void) { natsStatus s; natsUrl *u = NULL; @@ -1815,8 +1817,7 @@ test_natsUrl(void) && (strstr(nats_GetLastError(NULL), "invalid port '2147483648'") != NULL)); } -static void -test_natsCreateStringFromBuffer(void) +void test_natsCreateStringFromBuffer(void) { natsStatus s = NATS_OK; natsBuffer buf; @@ -1888,8 +1889,7 @@ _testInbox(void *closure) args->status = s; } -static void -test_natsInbox(void) +void test_natsInbox(void) { natsStatus s = NATS_OK; natsThread *threads[INBOX_THREADS_COUNT]; @@ -1956,8 +1956,7 @@ test_natsInbox(void) static int HASH_ITER = 10000000; -static void -test_natsHashing(void) +void test_natsHashing(void) { const char *keys[] = {"foo", "bar", @@ -2005,8 +2004,7 @@ test_natsHashing(void) testCond((s == NATS_OK) && ((end - start) < 1000)); } -static void -test_natsHash(void) +void test_natsHash(void) { natsStatus s; natsHash *hash = NULL; @@ -2271,8 +2269,7 @@ test_natsHash(void) natsHash_Destroy(hash); } -static void -test_natsStrHash(void) +void test_natsStrHash(void) { natsStatus s; natsStrHash *hash = NULL; @@ -2588,8 +2585,7 @@ _dummySigCb(char **customErrTxt, unsigned char **psig, int *sigLen, const char* return NATS_OK; } -static void -test_natsOptions(void) +void test_natsOptions(void) { natsStatus s; natsOptions *opts = NULL; @@ -2618,10 +2614,11 @@ test_natsOptions(void) && (opts->token == NULL) && (opts->tokenCb == NULL) && (opts->orderIP == 0) - && (opts->writeDeadline == natsLib_defaultWriteDeadline()) + && (opts->writeDeadline == nats_lib()->config.DefaultWriteDeadline) && !opts->noEcho && !opts->retryOnFailedConnect - && !opts->ignoreDiscoveredServers) + && !opts->ignoreDiscoveredServers + && !opts->tlsHandshakeFirst); test("Add URL: "); s = natsOptions_SetURL(opts, "test"); @@ -2771,6 +2768,14 @@ test_natsOptions(void) testCond((s == NATS_ILLEGAL_STATE) && (opts->secure == false)); #endif + test("Set TLSHandshakeFirst: "); + s = natsOptions_TLSHandshakeFirst(opts); +#if defined(NATS_HAS_TLS) + testCond((s == NATS_OK) && (opts->tlsHandshakeFirst == true) && (opts->secure == true)); +#else + testCond((s == NATS_ILLEGAL_STATE) && (opts->secure == false) && (opts->tlsHandshakeFirst == false)); +#endif + test("Set Pedantic: "); s = natsOptions_SetPedantic(opts, true); testCond((s == NATS_OK) && (opts->pedantic == true)); @@ -3145,8 +3150,7 @@ test_natsOptions(void) natsOptions_Destroy(opts); } -static void -test_natsSock_ReadLine(void) +void test_natsSock_ReadLine(void) { char buffer[20]; natsStatus s; @@ -3183,8 +3187,7 @@ _dummyJSONCb(void *userInfo, const char *fieldName, nats_JSONField *f) return NATS_OK; } -static void -test_natsJSON(void) +void test_natsJSON(void) { natsStatus s; nats_JSON *json = NULL; @@ -4232,8 +4235,7 @@ test_natsJSON(void) } -static void -test_natsEncodeTimeUTC(void) +void test_natsEncodeTimeUTC(void) { natsStatus s; char buf[36] = {'\0'}; @@ -4282,8 +4284,7 @@ test_natsEncodeTimeUTC(void) } } -static void -test_natsErrWithLongText(void) +void test_natsErrWithLongText(void) { natsStatus s; char errTxt[300]; @@ -4321,8 +4322,7 @@ test_natsErrWithLongText(void) nats_clearLastError(); } -static void -test_natsErrStackMoreThanMaxFrames(void) +void test_natsErrStackMoreThanMaxFrames(void) { int i; const int total = MAX_FRAMES+10; @@ -4375,8 +4375,7 @@ test_natsErrStackMoreThanMaxFrames(void) testCond(s == NATS_OK); } -static void -test_natsMsg(void) +void test_natsMsg(void) { natsMsg *msg = NULL; natsStatus s = NATS_OK; @@ -4412,8 +4411,7 @@ test_natsMsg(void) natsMsg_Destroy(msg); } -static void -test_natsBase32Decode(void) +void test_natsBase32Decode(void) { natsStatus s; const char *src = "KRUGS4ZANFZSA5DIMUQHEZLTOVWHIIDPMYQGCIDCMFZWKMZSEBSGKY3PMRUW4ZY"; @@ -4439,8 +4437,7 @@ test_natsBase32Decode(void) && (strstr(nats_GetLastError(NULL), "invalid") != NULL)); } -static void -test_natsBase64Encode(void) +void test_natsBase64Encode(void) { natsStatus s; char *enc = NULL; @@ -4586,8 +4583,7 @@ test_natsBase64Encode(void) free(dec); } -static void -test_natsCRC16(void) +void test_natsCRC16(void) { unsigned char a[] = {153, 209, 36, 74, 103, 32, 65, 34, 111, 68, 104, 156, 50, 14, 164, 140, 144, 230}; uint16_t crc = 0; @@ -4605,8 +4601,7 @@ test_natsCRC16(void) testCond(!nats_CRC16_Validate(a, (int)sizeof(a), expected)); } -static void -test_natsKeys(void) +void test_natsKeys(void) { natsStatus s; unsigned char sig[NATS_CRYPTO_SIGN_BYTES]; @@ -4662,8 +4657,7 @@ test_natsKeys(void) && (memcmp(sig, expected, sizeof(expected)) == 0)); } -static void -test_natsReadFile(void) +void test_natsReadFile(void) { natsStatus s = NATS_OK; natsBuffer *buf = NULL; @@ -4757,8 +4751,7 @@ test_natsReadFile(void) remove(fn); } -static void -test_natsGetJWTOrSeed(void) +void test_natsGetJWTOrSeed(void) { natsStatus s; char *val = NULL; @@ -4804,8 +4797,7 @@ test_natsGetJWTOrSeed(void) } } -static void -test_natsHostIsIP(void) +void test_natsHostIsIP(void) { struct _testHost { const char *host; @@ -4896,8 +4888,7 @@ _testSockShutdownThread(void *closure) natsSock_Shutdown(ctx->fd); } -static void -test_natsWaitReady(void) +void test_natsWaitReady(void) { natsStatus s = NATS_OK; natsThread *t = NULL; @@ -4986,8 +4977,7 @@ test_natsWaitReady(void) _destroyDefaultThreadArgs(&arg); } -static void -test_natsSign(void) +void test_natsSign(void) { unsigned char *sig = NULL; int sigLen = 0; @@ -5076,8 +5066,7 @@ _testStatus(const char *testName, char *buf, const char *expectedStatus, const c natsMsg_Destroy(msg); } -static void -test_natsMsgHeadersLift(void) +void test_HeadersLift(void) { char buf[512]; @@ -5138,8 +5127,7 @@ test_natsMsgHeadersLift(void) _testStatus("Status with description (extra spaces): ", buf, "404", "No Messages"); } -static void -test_natsMsgHeaderAPIs(void) +void test_natsMsgHeaderAPIs(void) { natsStatus s = NATS_OK; natsMsg *msg = NULL; @@ -5367,8 +5355,7 @@ test_natsMsgHeaderAPIs(void) natsMsg_Destroy(msg); } -static void -test_natsMsgIsJSCtrl(void) +void test_natsMsgIsJSCtrl(void) { struct testCase { const char *buf; @@ -5426,8 +5413,7 @@ test_natsMsgIsJSCtrl(void) } } -static void -test_natsSrvVersionAtLeast(void) +void test_natsSrvVersionAtLeast(void) { natsOptions *opts = NULL; natsConnection *nc = NULL; @@ -5477,6 +5463,7 @@ test_natsSrvVersionAtLeast(void) { s = NATS_ERR; } + natsConn_Unlock(nc); } } testCond(s == NATS_OK); @@ -5496,8 +5483,7 @@ test_natsSrvVersionAtLeast(void) natsConnection_Destroy(nc); } -static void -test_natsFormatStringArray(void) +void test_natsFormatStringArray(void) { natsStatus s; size_t i, N; @@ -5531,331 +5517,13 @@ test_natsFormatStringArray(void) NATS_FREE_STRINGS(out, N); } -static natsStatus -_checkStart(const char *url, int orderIP, int maxAttempts) -{ - natsStatus s = NATS_OK; - natsUrl *nUrl = NULL; - int attempts = 0; - natsSockCtx ctx; - - natsSock_Init(&ctx); - ctx.orderIP = orderIP; - - natsDeadline_Init(&(ctx.writeDeadline), 2000); - - s = natsUrl_Create(&nUrl, url); - if (s == NATS_OK) - { - while (((s = natsSock_ConnectTcp(&ctx, - nUrl->host, nUrl->port)) != NATS_OK) - && (attempts++ < maxAttempts)) - { - nats_Sleep(200); - } - - natsUrl_Destroy(nUrl); - - if (s == NATS_OK) - natsSock_Close(ctx.fd); - else - s = NATS_NO_SERVER; - } - - nats_clearLastError(); - - return s; -} - -static natsStatus -_checkStreamingStart(const char *url, int maxAttempts) -{ - natsStatus s = NATS_NOT_PERMITTED; - -#if defined(NATS_HAS_STREAMING) - - stanConnOptions *opts = NULL; - stanConnection *sc = NULL; - int attempts = 0; - - s = stanConnOptions_Create(&opts); - IFOK(s, stanConnOptions_SetURL(opts, url)); - IFOK(s, stanConnOptions_SetConnectionWait(opts, 250)); - if (s == NATS_OK) - { - while (((s = stanConnection_Connect(&sc, clusterName, "checkStart", opts)) != NATS_OK) - && (attempts++ < maxAttempts)) - { - nats_Sleep(200); - } - } - - stanConnection_Destroy(sc); - stanConnOptions_Destroy(opts); - - if (s != NATS_OK) - nats_clearLastError(); -#else -#endif - return s; -} - -#ifdef _WIN32 - -typedef PROCESS_INFORMATION *natsPid; - -static HANDLE logHandle = NULL; - -static void -_stopServer(natsPid pid) -{ - if (pid == NATS_INVALID_PID) - return; - - TerminateProcess(pid->hProcess, 0); - WaitForSingleObject(pid->hProcess, INFINITE); - - CloseHandle(pid->hProcess); - CloseHandle(pid->hThread); - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Remove(slMap, (int64_t) pid); - natsMutex_Unlock(slMu); - - free(pid); -} - -static natsPid -_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) -{ - SECURITY_ATTRIBUTES sa; - STARTUPINFO si; - HANDLE h; - PROCESS_INFORMATION *pid; - DWORD flags = 0; - BOOL createdOk = FALSE; - BOOL hInheritance = FALSE; - char *exeAndCmdLine = NULL; - int ret; - - pid = calloc(1, sizeof(PROCESS_INFORMATION)); - if (pid == NULL) - return NATS_INVALID_PID; - - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - - ret = nats_asprintf(&exeAndCmdLine, "%s%s%s", serverExe, - (cmdLineOpts != NULL ? " " : ""), - (cmdLineOpts != NULL ? cmdLineOpts : "")); - if (ret < 0) - { - printf("No memory allocating command line string!\n"); - free(pid); - return NATS_INVALID_PID; - } - - if (!keepServerOutput) - { - ZeroMemory(&sa, sizeof(sa)); - sa.nLength = sizeof(sa); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = TRUE; - - h = logHandle; - if (h == NULL) - { - h = CreateFile(LOGFILE_NAME, - GENERIC_WRITE, - FILE_SHARE_WRITE | FILE_SHARE_READ, - &sa, - CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, - NULL); - } - - si.dwFlags |= STARTF_USESTDHANDLES; - si.hStdInput = NULL; - si.hStdError = h; - si.hStdOutput = h; - - hInheritance = TRUE; - flags = CREATE_NO_WINDOW; - - if (logHandle == NULL) - logHandle = h; - } - - // Start the child process. - if (!CreateProcess(NULL, - (LPSTR) exeAndCmdLine, - NULL, // Process handle not inheritable - NULL, // Thread handle not inheritable - hInheritance, // Set handle inheritance - flags, // Creation flags - NULL, // Use parent's environment block - NULL, // Use parent's starting directory - &si, // Pointer to STARTUPINFO structure - pid)) // Pointer to PROCESS_INFORMATION structure - { - - printf("Unable to start '%s': error (%d).\n", - exeAndCmdLine, GetLastError()); - free(exeAndCmdLine); - return NATS_INVALID_PID; - } - - free(exeAndCmdLine); - - if (checkStart) - { - natsStatus s; - - if (strcmp(serverExe, natsServerExe) == 0) - s = _checkStart(url, 46, 10); - else - s = _checkStreamingStart(url, 10); - - if (s != NATS_OK) - { - _stopServer(pid); - return NATS_INVALID_PID; - } - } - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Set(slMap, (int64_t) pid, NULL, NULL); - natsMutex_Unlock(slMu); - - return (natsPid) pid; -} - -#else - -typedef pid_t natsPid; - -static void -_stopServer(natsPid pid) -{ - int status = 0; - - if (pid == NATS_INVALID_PID) - return; - - if (kill(pid, SIGINT) < 0) - { - perror("kill with SIGINT"); - if (kill(pid, SIGKILL) < 0) - { - perror("kill with SIGKILL"); - } - } - - waitpid(pid, &status, 0); - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Remove(slMap, (int64_t) pid); - natsMutex_Unlock(slMu); -} - -static natsPid -_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) -{ - natsPid pid = fork(); - if (pid == -1) - { - perror("fork"); - return NATS_INVALID_PID; - } - - if (pid == 0) - { - char *exeAndCmdLine = NULL; - char *argvPtrs[64]; - char *line = NULL; - int index = 0; - int ret = 0; - bool overrideAddr = false; - - if ((cmdLineOpts == NULL) || (strstr(cmdLineOpts, "-a ") == NULL)) - overrideAddr = true; - - ret = nats_asprintf(&exeAndCmdLine, "%s%s%s%s%s", serverExe, - (cmdLineOpts != NULL ? " " : ""), - (cmdLineOpts != NULL ? cmdLineOpts : ""), - (overrideAddr ? " -a 127.0.0.1" : ""), - (keepServerOutput ? "" : " -l " LOGFILE_NAME)); - if (ret < 0) - { - perror("No memory allocating command line string!\n"); - exit(1); - } - - memset(argvPtrs, 0, sizeof(argvPtrs)); - line = exeAndCmdLine; - - while (*line != '\0') - { - while ((*line == ' ') || (*line == '\t') || (*line == '\n')) - *line++ = '\0'; - - argvPtrs[index++] = line; - while ((*line != '\0') && (*line != ' ') - && (*line != '\t') && (*line != '\n')) - { - line++; - } - } - argvPtrs[index++] = NULL; - - // Child process. Replace with NATS server - execvp(argvPtrs[0], argvPtrs); - perror("Exec failed: "); - exit(1); - } - else if (checkStart) - { - natsStatus s; - - if (strcmp(serverExe, natsServerExe) == 0) - s = _checkStart(url, 46, 10); - else - s = _checkStreamingStart(url, 10); - - if (s != NATS_OK) - { - _stopServer(pid); - return NATS_INVALID_PID; - } - } - - natsMutex_Lock(slMu); - if (slMap != NULL) - natsHash_Set(slMap, (int64_t) pid, NULL, NULL); - natsMutex_Unlock(slMu); - - // parent, return the child's PID back. - return pid; -} -#endif - -static natsPid -_startServer(const char *url, const char *cmdLineOpts, bool checkStart) -{ - return _startServerImpl(natsServerExe, url, cmdLineOpts, checkStart); -} - static natsPid _startStreamingServer(const char* url, const char *cmdLineOpts, bool checkStart) { return _startServerImpl(natsStreamingServerExe, url, cmdLineOpts, checkStart); } -static void -test_natsSock_IPOrder(void) +void test_natsSock_IPOrder(void) { natsStatus s; natsPid serverPid; @@ -5913,8 +5581,7 @@ test_natsSock_IPOrder(void) } } -static void -test_natsSock_ConnectTcp(void) +void test_natsSock_ConnectTcp(void) { natsPid serverPid = NATS_INVALID_PID; @@ -5953,8 +5620,7 @@ listOrder(struct addrinfo *head, bool ordered) return true; } -static void -test_natsSock_ShuffleIPs(void) +void test_natsSock_ShuffleIPs(void) { struct addrinfo *tmp[10]; struct addrinfo *head = NULL; @@ -6072,8 +5738,7 @@ _reconnectedCb(natsConnection *nc, void *closure) natsMutex_Unlock(arg->m); } -static void -test_ReconnectServerStats(void) +void test_ReconnectServerStats(void) { natsStatus s; natsConnection *nc = NULL; @@ -6166,6 +5831,7 @@ _recvTestString(natsConnection *nc, natsSubscription *sub, natsMsg *msg, natsMutex_Lock(arg->m); + // up to 12 now switch (arg->control) { case 0: @@ -6200,15 +5866,26 @@ _recvTestString(natsConnection *nc, natsSubscription *sub, natsMsg *msg, } case 3: case 9: + case 12: { doSignal = false; arg->sum++; - if ((arg->control != 9) && (arg->sum == 10)) + if ((arg->control == 3) && (arg->sum == 10)) { arg->status = natsSubscription_Unsubscribe(sub); doSignal = true; } + else if ((arg->control == 12) && (arg->sum == arg->N)) + { + arg->status = NATS_OK; + doSignal = true; + } + else if ((arg->control == 12) && (arg->sum > arg->N)) + { + doSignal = true; + arg->status = NATS_ERR; + } break; } case 11: @@ -6307,8 +5984,7 @@ _waitForConnClosed(struct threadArg *arg) return s; } -static void -test_ParseStateReconnectFunctionality(void) +void test_ParseStateReconnectFunctionality(void) { natsStatus s; natsConnection *nc = NULL; @@ -6395,8 +6071,7 @@ test_ParseStateReconnectFunctionality(void) _stopServer(serverPid); } -static void -test_ServersRandomize(void) +void test_ServersRandomize(void) { natsStatus s; natsOptions *opts = NULL; @@ -6530,8 +6205,7 @@ test_ServersRandomize(void) _stopServer(pid); } -static void -test_SelectNextServer(void) +void test_SelectNextServer(void) { natsStatus s; natsOptions *opts = NULL; @@ -6658,8 +6332,7 @@ parserNegTest(int lineNum) #define PARSER_START_TEST parserNegTest(__LINE__) -static void -test_ParserPing(void) +void test_ParserPing(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6724,8 +6397,7 @@ test_ParserPing(void) natsConnection_Destroy(nc); } -static void -test_ParserErr(void) +void test_ParserErr(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6811,8 +6483,7 @@ test_ParserErr(void) natsConnection_Destroy(nc); } -static void -test_ParserOK(void) +void test_ParserOK(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6849,8 +6520,7 @@ test_ParserOK(void) natsConnection_Destroy(nc); } -static void -test_ParseINFO(void) +void test_ParseINFO(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -6881,8 +6551,7 @@ test_ParseINFO(void) natsConnection_Destroy(nc); } -static void -test_ParserShouldFail(void) +void test_ParserShouldFail(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7039,8 +6708,7 @@ test_ParserShouldFail(void) natsConnection_Destroy(nc); } -static void -test_ParserSplitMsg(void) +void test_ParserSplitMsg(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7207,8 +6875,7 @@ test_ParserSplitMsg(void) if (s != NATS_OK) \ FAIL("Unable to setup test"); \ -static void -test_ProcessMsgArgs(void) +void test_ProcessMsgArgs(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7581,8 +7248,7 @@ checkNewURLsAddedRandomly(natsConnection *nc, char **urlsAfterPoolSetup, int ini return s; } -static void -test_AsyncINFO(void) +void test_AsyncINFO(void) { natsConnection *nc = NULL; natsOptions *opts = NULL; @@ -7819,8 +7485,7 @@ _parallelRequests(void *closure) natsConnection_RequestString(&msg, nc, "foo", "test", 500); } -static void -test_RequestPool(void) +void test_RequestPool(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -7883,8 +7548,7 @@ test_RequestPool(void) _stopServer(pid); } -static void -test_NoFlusherIfSendAsap(void) +void test_NoFlusherIfSendAsap(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -7952,8 +7616,7 @@ test_NoFlusherIfSendAsap(void) _stopServer(pid); } -static void -test_HeadersAndSubPendingBytes(void) +void test_HeadersAndSubPendingBytes(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -8032,153 +7695,214 @@ _dummyMsgHandler(natsConnection *nc, natsSubscription *sub, natsMsg *msg, natsMsg_Destroy(msg); } -static void -test_LibMsgDelivery(void) +static int _numRunningThreads(natsDispatcherPool *pool) { - natsStatus s; - natsPid serverPid = NATS_INVALID_PID; - natsOptions *opts = NULL; - natsConnection *nc = NULL; - natsSubscription *s1 = NULL; - natsSubscription *s2 = NULL; - natsSubscription *s3 = NULL; - natsSubscription *s4 = NULL; - natsSubscription *s5 = NULL; - natsMsgDlvWorker *lmd1 = NULL; - natsMsgDlvWorker *lmd2 = NULL; - natsMsgDlvWorker *lmd3 = NULL; - natsMsgDlvWorker *lmd4 = NULL; - natsMsgDlvWorker *lmd5 = NULL; - natsMsgDlvWorker **pwks = NULL; - int psize = 0; - int pmaxSize = 0; - int pidx = 0; + int i, n; + for (n = 0, i = 0; i < pool->cap; i++) + n += (pool->dispatchers[i].thread != NULL); + return n; +} - // First, close the library and re-open, to reset things - nats_Close(); +void test_AssignSubToDispatch(void) +{ + natsStatus s; +#define MAX_SUBS 500 + natsPid serverPid = NATS_INVALID_PID; + natsConnection *nc = NULL; + natsSubscription *subs[MAX_SUBS]; + natsClientConfig config4 = { + .DefaultToThreadPool = true, + .ThreadPoolMax = 4, + }; + natsClientConfig config50 = { + .DefaultToThreadPool = true, + .ThreadPoolMax = 50, + }; + natsDispatcherPool *pool = NULL; + natsDispatcherPool *rpool = NULL; + natsClientConfig c; + struct threadArg arg; + char subSubj[32]; + char pubSubj[32]; + int i=0, n=0; - nats_Sleep(100); + const int numMsgs = 5; - nats_Open(-1); + typedef struct + { + int numSubs; + natsClientConfig *config; + int expectedMax; + int expectedDispatchers; + bool expectedDefaultPool; + } TC; - // Check some pre-conditions that need to be met for the test to work. - test("Check initial values: ") - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - testCond((pmaxSize == 1) && (psize == 0) && (pidx == 0)); + TC tcs[] = { + { + .numSubs = 10, + .expectedMax = 1, + .expectedDispatchers = 0, // all should be dispatched with dedicated threads + }, + { + .config = &config4, + .numSubs = 3, + .expectedMax = 4, + .expectedDispatchers = 3, + .expectedDefaultPool = true, + }, + { + .config = &config4, + .numSubs = 23, + .expectedMax = 4, + .expectedDispatchers = 4, + .expectedDefaultPool = true, + }, + { + .numSubs = 77, + .config = &config50, + .expectedMax = 50, + .expectedDispatchers = 50, + .expectedDefaultPool = true, + }, + }; + TC *tc, *end = tcs + (sizeof(tcs) / sizeof(tcs[0])); - test("Check pool size not negative: ") + test("Check pool size not negative: "); s = nats_SetMessageDeliveryPoolSize(-1); testCond(s != NATS_OK); - test("Check pool size not zero: ") + test("Check pool size not zero: "); s = nats_SetMessageDeliveryPoolSize(0); testCond(s != NATS_OK); // Reset stack since we know the above generated errors. nats_clearLastError(); - test("Increase size to 2: ") - s = nats_SetMessageDeliveryPoolSize(2); - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - testCond((s == NATS_OK) && (pmaxSize == 2) && (psize == 0)); - - test("Check pool size decreased (no error): ") - s = nats_SetMessageDeliveryPoolSize(1); - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - testCond((s == NATS_OK) && (pmaxSize == 2) && (psize == 0)); + s = _createDefaultThreadArgsForCbTests(&arg); + if (s != NATS_OK) + FAIL("Unable to setup test"); serverPid = _startServer("nats://127.0.0.1:4222", NULL, true); CHECK_SERVER_STARTED(serverPid); - s = natsOptions_Create(&opts); - IFOK(s, natsOptions_UseGlobalMessageDelivery(opts, true)); - IFOK(s, natsConnection_Connect(&nc, opts)); - IFOK(s, natsConnection_Subscribe(&s1, nc, "foo", _dummyMsgHandler, NULL)); - if (s == NATS_OK) - { - natsMutex_Lock(s1->mu); - lmd1 = s1->libDlvWorker; - natsMutex_Unlock(s1->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 1st sub assigned 1st worker: ") - testCond((s == NATS_OK) && (psize == 1) && (lmd1 != NULL) - && (pidx == 1) && (pwks != NULL) && (lmd1 == pwks[0])); + // First, close the library and re-open, to reset things + test("Reset the library's global state: "); + nats_CloseAndWait(100); + testCond(true); - s = natsConnection_Subscribe(&s2, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) + for (tc = tcs; tc < end; tc++, n++) { - natsMutex_Lock(s2->mu); - lmd2 = s2->libDlvWorker; - natsMutex_Unlock(s2->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 2nd sub assigned 2nd worker: ") - testCond((s == NATS_OK) && (psize == 2) && (lmd2 != lmd1) - && (pidx == 0) && (pwks != NULL) && (lmd2 == pwks[1])); + testf("%d subs over %d threads: Re-open, check settings, ensure no shared dispatchers created: ", tc->numSubs, tc->expectedDispatchers); + s = nats_OpenWithConfig(tc->config); + IFOK(s, (c = nats_lib()->config, NATS_OK)); + if (s == NATS_OK) + { + pool = &nats_lib()->messageDispatchers; + rpool = &nats_lib()->replyDispatchers; + } + natsMutex_Lock(pool->lock); + natsMutex_Lock(rpool->lock); + testCond((s == NATS_OK) && + (c.DefaultToThreadPool == tc->expectedDefaultPool) && + (c.ThreadPoolMax == tc->expectedMax) && + (c.DefaultRepliesToThreadPool == false) && + (c.ReplyThreadPoolMax == 0) && + (c.UseSeparatePoolForReplies == false) && + (_numRunningThreads(pool) == 0) && + (pool->cap == tc->expectedMax) && + (pool->dispatchers != NULL) && + (_numRunningThreads(rpool) == 0) && + (rpool->dispatchers == NULL)); + natsMutex_Unlock(rpool->lock); + natsMutex_Unlock(pool->lock); + + testf("%d subs over %d threads: Initial connect: ", tc->numSubs, tc->expectedDispatchers); + s = natsConnection_ConnectTo(&nc, NATS_DEFAULT_URL); + testCond(s == NATS_OK); - s = natsConnection_Subscribe(&s3, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) - { - natsMutex_Lock(s3->mu); - lmd3 = s3->libDlvWorker; - natsMutex_Unlock(s3->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 3rd sub assigned 1st worker: ") - testCond((s == NATS_OK) && (psize == 2) && (lmd3 == lmd1) - && (pidx == 1) && (pwks != NULL) && (lmd3 == pwks[0])); + testf("%d subs over %d threads: Create %d subscriptions: ", tc->numSubs, tc->expectedDispatchers, tc->numSubs); + if (tc->numSubs > MAX_SUBS) + FAIL("Too many subscriptions for this test's buffer"); - // Bump the pool size to 4 - s = nats_SetMessageDeliveryPoolSize(4); - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check increase of pool size: "); - testCond((s == NATS_OK) && (psize == 2) && (pidx == 1) - && (pmaxSize == 4) && (pwks != NULL)); + arg.string = "test"; + arg.status = NATS_OK; + arg.control = 12; + arg.N = tc->numSubs * numMsgs; + arg.sum = 0; + snprintf(subSubj, sizeof(subSubj), "foo.%d.*", n); + for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) + { + s = natsConnection_Subscribe(&subs[i], nc, subSubj, _recvTestString, &arg); + } + testCond(s == NATS_OK); - s = natsConnection_Subscribe(&s4, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) - { - natsMutex_Lock(s4->mu); - lmd4 = s4->libDlvWorker; - natsMutex_Unlock(s4->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 4th sub assigned 2nd worker: ") - testCond((s == NATS_OK) && (psize == 2) && (lmd4 == lmd2) - && (pidx == 2) && (pwks != NULL) && (lmd4 == pwks[1])); + testf("%d subs over %d threads: Check shared dispatchers assigned as expected: ", tc->numSubs, tc->expectedDispatchers); + natsMutex_Lock(pool->lock); + natsMutex_Lock(rpool->lock); + testCond((pool->dispatchers != NULL) && + (_numRunningThreads(pool) == tc->expectedDispatchers) && + (_numRunningThreads(rpool) == 0)); - s = natsConnection_Subscribe(&s5, nc, "foo", _dummyMsgHandler, NULL); - if (s == NATS_OK) - { - natsMutex_Lock(s5->mu); - lmd5 = s5->libDlvWorker; - natsMutex_Unlock(s5->mu); - } - natsLib_getMsgDeliveryPoolInfo(&pmaxSize, &psize, &pidx, &pwks); - test("Check 5th sub assigned 3rd worker: ") - testCond((s == NATS_OK) && (psize == 3) && (lmd5 != lmd2) - && (pidx == 3) && (pwks != NULL) && (lmd5 == pwks[2])); + if (tc->config != NULL) + { + testf("%d subs over %d threads: Verify that the dispatchers have been assigned: ", tc->numSubs, tc->expectedDispatchers); + for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) + { + natsSub_Lock(subs[i]); + if (subs[i]->dispatcher != &pool->dispatchers[i % tc->expectedDispatchers]) + s = NATS_ERR; + natsSub_Unlock(subs[i]); + } + testCond(s == NATS_OK); + } + natsMutex_Unlock(rpool->lock); + natsMutex_Unlock(pool->lock); - natsSubscription_Destroy(s5); - natsSubscription_Destroy(s4); - natsSubscription_Destroy(s3); - natsSubscription_Destroy(s2); - natsSubscription_Destroy(s1); - natsConnection_Destroy(nc); - natsOptions_Destroy(opts); - _stopServer(serverPid); + testf("%d subs over %d threads: Publish %d messages and verify received %d times: ", tc->numSubs, tc->expectedDispatchers, numMsgs, arg.N); + for (i = 0; (s == NATS_OK) && (i < numMsgs); i++) + { + snprintf(pubSubj, sizeof(pubSubj), "foo.%d.%d", n, i); + s = natsConnection_PublishString(nc, pubSubj, "test"); + } + if (s == NATS_OK) + { + natsMutex_Lock(arg.m); + while ((s != NATS_TIMEOUT) && !arg.msgReceived) + s = natsCondition_TimedWait(arg.c, arg.m, 1500); + arg.msgReceived = false; + arg.status = NATS_OK; + natsMutex_Unlock(arg.m); + if (s == NATS_OK) + s = arg.status; + } + testCond(s == NATS_OK); - // Close the library and re-open, to reset things - nats_Close(); + testf("%d subs over %d threads: Drain the subs: ", tc->numSubs, tc->expectedDispatchers); + for (i = 0; (s == NATS_OK) && (i < tc->numSubs); i++) + { + s = natsSubscription_Drain(subs[i]); + natsSubscription_Destroy(subs[i]); + } + testCond(s == NATS_OK); - nats_Sleep(100); + testf("%d subs over %d threads: Make sure we didn't get unexpected messages: ", tc->numSubs, tc->expectedDispatchers); + natsMutex_Lock(arg.m); + while ((s != NATS_TIMEOUT) && !arg.msgReceived) + s = natsCondition_TimedWait(arg.c, arg.m, 50); + natsMutex_Unlock(arg.m); + testCond((s == NATS_TIMEOUT) && !arg.msgReceived && arg.status != NATS_ERR); - nats_Open(-1); + testf("%d subs over %d threads: cleanup before next test: ", tc->numSubs, tc->expectedDispatchers); + natsConnection_Destroy(nc); + nats_Close(); + testCond(true); + } + _stopServer(serverPid); + + _destroyDefaultThreadArgs(&arg); } -static void -test_DefaultConnection(void) +void test_DefaultConnection(void) { natsStatus s; natsConnection *nc = NULL; @@ -8225,8 +7949,7 @@ test_DefaultConnection(void) _stopServer(serverPid); } -static void -test_SimplifiedURLs(void) +void test_SimplifiedURLs(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -8287,8 +8010,7 @@ test_SimplifiedURLs(void) #endif } -static void -test_IPResolutionOrder(void) +void test_IPResolutionOrder(void) { natsStatus s; natsConnection *nc = NULL; @@ -8444,8 +8166,7 @@ test_IPResolutionOrder(void) natsOptions_Destroy(opts); } -static void -test_UseDefaultURLIfNoServerSpecified(void) +void test_UseDefaultURLIfNoServerSpecified(void) { natsStatus s; natsOptions *opts = NULL; @@ -8469,8 +8190,7 @@ test_UseDefaultURLIfNoServerSpecified(void) _stopServer(serverPid); } -static void -test_ConnectToWithMultipleURLs(void) +void test_ConnectToWithMultipleURLs(void) { natsStatus s; natsConnection *nc = NULL; @@ -8501,8 +8221,7 @@ test_ConnectToWithMultipleURLs(void) _stopServer(serverPid); } -static void -test_ConnectionToWithNullURLs(void) +void test_ConnectionToWithNullURLs(void) { natsStatus s; natsConnection *nc = NULL; @@ -8523,8 +8242,7 @@ test_ConnectionToWithNullURLs(void) _stopServer(serverPid); } -static void -test_ConnectionWithNullOptions(void) +void test_ConnectionWithNullOptions(void) { natsStatus s; natsConnection *nc = NULL; @@ -8542,8 +8260,7 @@ test_ConnectionWithNullOptions(void) _stopServer(serverPid); } -static void -test_ConnectionStatus(void) +void test_ConnectionStatus(void) { natsStatus s; natsConnection *nc = NULL; @@ -8569,8 +8286,7 @@ test_ConnectionStatus(void) _stopServer(serverPid); } -static void -test_ConnClosedCB(void) +void test_ConnClosedCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -8614,8 +8330,7 @@ test_ConnClosedCB(void) _stopServer(serverPid); } -static void -test_CloseDisconnectedCB(void) +void test_CloseDisconnectedCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -8660,8 +8375,7 @@ test_CloseDisconnectedCB(void) _stopServer(serverPid); } -static void -test_ServerStopDisconnectedCB(void) +void test_ServerStopDisconnectedCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -8704,8 +8418,7 @@ test_ServerStopDisconnectedCB(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ClosedConnections(void) +void test_ClosedConnections(void) { natsStatus s; natsConnection *nc = NULL; @@ -8774,8 +8487,7 @@ test_ClosedConnections(void) _stopServer(serverPid); } -static void -test_ConnectVerboseOption(void) +void test_ConnectVerboseOption(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -8828,8 +8540,7 @@ test_ConnectVerboseOption(void) _stopServer(serverPid); } -static void -test_ReconnectThreadLeak(void) +void test_ReconnectThreadLeak(void) { natsStatus s; natsOptions *opts = NULL; @@ -8890,8 +8601,7 @@ test_ReconnectThreadLeak(void) _stopServer(serverPid); } -static void -test_ReconnectTotalTime(void) +void test_ReconnectTotalTime(void) { natsStatus s; natsOptions *opts = NULL; @@ -8904,8 +8614,7 @@ test_ReconnectTotalTime(void) natsOptions_Destroy(opts); } -static void -test_ReconnectDisallowedFlags(void) +void test_ReconnectDisallowedFlags(void) { natsStatus s; natsConnection *nc = NULL; @@ -8940,8 +8649,7 @@ test_ReconnectDisallowedFlags(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ReconnectAllowedFlags(void) +void test_ReconnectAllowedFlags(void) { natsStatus s; natsConnection *nc = NULL; @@ -8998,8 +8706,7 @@ _closeConn(void *arg) natsConnection_Close(nc); } -static void -test_ConnCloseBreaksReconnectLoop(void) +void test_ConnCloseBreaksReconnectLoop(void) { natsStatus s; natsConnection *nc = NULL; @@ -9067,8 +8774,7 @@ test_ConnCloseBreaksReconnectLoop(void) _destroyDefaultThreadArgs(&arg); } -static void -test_BasicReconnectFunctionality(void) +void test_BasicReconnectFunctionality(void) { natsStatus s; natsConnection *nc = NULL; @@ -9154,8 +8860,7 @@ _doneCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure) natsMsg_Destroy(msg); } -static void -test_ExtendedReconnectFunctionality(void) +void test_ExtendedReconnectFunctionality(void) { natsStatus s; natsConnection *nc = NULL; @@ -9260,8 +8965,7 @@ test_ExtendedReconnectFunctionality(void) _stopServer(serverPid); } -static void -test_QueueSubsOnReconnect(void) +void test_QueueSubsOnReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -9372,8 +9076,7 @@ test_QueueSubsOnReconnect(void) _stopServer(serverPid); } -static void -test_IsClosed(void) +void test_IsClosed(void) { natsStatus s; natsConnection *nc = NULL; @@ -9407,8 +9110,7 @@ test_IsClosed(void) _stopServer(serverPid); } -static void -test_IsReconnectingAndStatus(void) +void test_IsReconnectingAndStatus(void) { natsStatus s; natsConnection *nc = NULL; @@ -9494,8 +9196,7 @@ test_IsReconnectingAndStatus(void) _stopServer(serverPid); } -static void -test_ReconnectBufSize(void) +void test_ReconnectBufSize(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -9602,8 +9303,7 @@ _testCustomReconnectDelayOnInitialConnect(natsConnection *nc, int attempts, void return 50; } -static void -test_RetryOnFailedConnect(void) +void test_RetryOnFailedConnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -9970,8 +9670,7 @@ _startMockupServer(natsSock *serverSock, const char *host, const char *port) return s; } -static void -test_ErrOnConnectAndDeadlock(void) +void test_ErrOnConnectAndDeadlock(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -10036,8 +9735,7 @@ test_ErrOnConnectAndDeadlock(void) natsSock_Close(sock); } -static void -test_ErrOnMaxPayloadLimit(void) +void test_ErrOnMaxPayloadLimit(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -10114,8 +9812,7 @@ test_ErrOnMaxPayloadLimit(void) _destroyDefaultThreadArgs(&arg); } -static void -test_Auth(void) +void test_Auth(void) { natsStatus s; natsConnection *nc = NULL; @@ -10164,8 +9861,7 @@ test_Auth(void) _stopServer(serverPid); } -static void -test_AuthFailNoDisconnectCB(void) +void test_AuthFailNoDisconnectCB(void) { natsStatus s; natsOptions *opts = NULL; @@ -10205,8 +9901,7 @@ test_AuthFailNoDisconnectCB(void) _stopServer(serverPid); } -static void -test_AuthToken(void) +void test_AuthToken(void) { natsStatus s; natsConnection *nc = NULL; @@ -10256,8 +9951,7 @@ _tokenHandler(void* closure) return (char*) closure; } -static void -test_AuthTokenHandler(void) +void test_AuthTokenHandler(void) { natsStatus s; natsConnection *nc = NULL; @@ -10327,8 +10021,7 @@ _permsViolationHandler(natsConnection *nc, natsSubscription *sub, natsStatus err } } -static void -test_PermViolation(void) +void test_PermViolation(void) { natsStatus s; natsConnection *conn = NULL; @@ -10423,8 +10116,7 @@ _authViolationHandler(natsConnection *nc, natsSubscription *sub, natsStatus err, } } -static void -test_AuthViolation(void) +void test_AuthViolation(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -10627,8 +10319,7 @@ _authExpiredHandler(natsConnection *nc, natsSubscription *sub, natsStatus err, v } } -static void -test_AuthenticationExpired(void) +void test_AuthenticationExpired(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -10765,8 +10456,7 @@ _startServerSendErr2Thread(void *closure) natsSock_Close(sock); } -static void -test_AuthenticationExpiredReconnect(void) +void test_AuthenticationExpiredReconnect(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -10854,8 +10544,7 @@ test_AuthenticationExpiredReconnect(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ConnectedServer(void) +void test_ConnectedServer(void) { natsStatus s; natsConnection *nc = NULL; @@ -10898,8 +10587,7 @@ test_ConnectedServer(void) _stopServer(serverPid); } -static void -test_MultipleClose(void) +void test_MultipleClose(void) { natsStatus s; natsConnection *nc = NULL; @@ -10928,8 +10616,7 @@ test_MultipleClose(void) _stopServer(serverPid); } -static void -test_SimplePublish(void) +void test_SimplePublish(void) { natsStatus s; natsConnection *nc = NULL; @@ -10950,8 +10637,7 @@ test_SimplePublish(void) _stopServer(serverPid); } -static void -test_SimplePublishNoData(void) +void test_SimplePublishNoData(void) { natsStatus s; natsConnection *nc = NULL; @@ -10972,8 +10658,7 @@ test_SimplePublishNoData(void) _stopServer(serverPid); } -static void -test_PublishMsg(void) +void test_PublishMsg(void) { natsStatus s; natsConnection *nc = NULL; @@ -11026,8 +10711,7 @@ test_PublishMsg(void) _destroyDefaultThreadArgs(&arg); } -static void -test_InvalidSubsArgs(void) +void test_InvalidSubsArgs(void) { natsStatus s; natsConnection *nc = NULL; @@ -11190,8 +10874,7 @@ test_InvalidSubsArgs(void) _stopServer(serverPid); } -static void -test_AsyncSubscribe(void) +void test_AsyncSubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11313,8 +10996,7 @@ _asyncTimeoutCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *c natsMutex_Unlock(ai->arg->m); } -static void -test_AsyncSubscribeTimeout(void) +void test_AsyncSubscribeTimeout(void) { natsStatus s; natsConnection *nc = NULL; @@ -11324,7 +11006,6 @@ test_AsyncSubscribeTimeout(void) struct threadArg arg; bool useLibDlv = false; int i; - char testText[128]; int64_t timeout = 100; _asyncTimeoutInfo ai; @@ -11346,10 +11027,9 @@ test_AsyncSubscribeTimeout(void) serverPid = _startServer("nats://127.0.0.1:4222", NULL, true); CHECK_SERVER_STARTED(serverPid); - snprintf(testText, sizeof(testText), "Test async %ssubscriber timeout%s: ", + testf("Test async %ssubscriber timeout%s: ", ((i == 1 || i == 3) ? "queue " : ""), (i > 1 ? " (lib msg delivery)" : "")); - test(testText); s = natsConnection_Connect(&nc, opts); if (s == NATS_OK) { @@ -11420,8 +11100,7 @@ test_AsyncSubscribeTimeout(void) } } -static void -test_SyncSubscribe(void) +void test_SyncSubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11449,8 +11128,7 @@ test_SyncSubscribe(void) _stopServer(serverPid); } -static void -test_PubSubWithReply(void) +void test_PubSubWithReply(void) { natsStatus s; natsConnection *nc = NULL; @@ -11479,8 +11157,7 @@ test_PubSubWithReply(void) _stopServer(serverPid); } -static void -test_NoResponders(void) +void test_NoResponders(void) { natsStatus s; natsConnection *nc = NULL; @@ -11581,8 +11258,7 @@ _doFlush(void *arg) } } -static void -test_Flush(void) +void test_Flush(void) { natsStatus s; natsOptions *opts = NULL; @@ -11705,8 +11381,7 @@ test_Flush(void) _stopServer(serverPid); } -static void -test_ConnCloseDoesFlush(void) +void test_ConnCloseDoesFlush(void) { natsStatus s = NATS_OK; natsPid pid = NATS_INVALID_PID; @@ -11756,8 +11431,7 @@ test_ConnCloseDoesFlush(void) _stopServer(pid); } -static void -test_QueueSubscriber(void) +void test_QueueSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -11825,8 +11499,7 @@ test_QueueSubscriber(void) _stopServer(serverPid); } -static void -test_ReplyArg(void) +void test_ReplyArg(void) { natsStatus s; natsConnection *nc = NULL; @@ -11868,8 +11541,7 @@ test_ReplyArg(void) _stopServer(serverPid); } -static void -test_SyncReplyArg(void) +void test_SyncReplyArg(void) { natsStatus s; natsConnection *nc = NULL; @@ -11898,8 +11570,7 @@ test_SyncReplyArg(void) _stopServer(serverPid); } -static void -test_Unsubscribe(void) +void test_Unsubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11962,8 +11633,7 @@ test_Unsubscribe(void) _stopServer(serverPid); } -static void -test_DoubleUnsubscribe(void) +void test_DoubleUnsubscribe(void) { natsStatus s; natsConnection *nc = NULL; @@ -11992,14 +11662,13 @@ test_DoubleUnsubscribe(void) _stopServer(serverPid); } -static void -test_SubRemovedWhileProcessingMsg(void) +void test_SubRemovedWhileProcessingMsg(void) { - natsStatus s; - natsConnection *nc = NULL; - natsOptions *opts = NULL; - natsSubscription *sub = NULL; - natsPid serverPid = NATS_INVALID_PID; + natsStatus s; + natsConnection *nc = NULL; + natsOptions *opts = NULL; + natsSubscription *sub = NULL; + natsPid serverPid = NATS_INVALID_PID; serverPid = _startServer("nats://127.0.0.1:4222", NULL, true); CHECK_SERVER_STARTED(serverPid); @@ -12024,7 +11693,7 @@ test_SubRemovedWhileProcessingMsg(void) test("Check msg not given: "); natsSub_Lock(sub); - testCond(sub->msgList.msgs == 0); + testCond(sub->ownDispatcher.queue.msgs == 0); natsSub_Unlock(sub); natsSubscription_Destroy(sub); @@ -12043,24 +11712,20 @@ test_SubRemovedWhileProcessingMsg(void) IFOK(s, natsConnection_Subscribe(&sub, nc, "foo", _dummyMsgHandler, NULL)); testCond(s == NATS_OK); - natsSub_Lock(sub); - natsMutex_Lock(sub->libDlvWorker->lock); + nats_lockSubAndDispatcher(sub); test("Send message: "); s = natsConnection_PublishString(nc, "foo", "hello"); testCond(s == NATS_OK); test("Close sub: "); - natsMutex_Unlock(sub->libDlvWorker->lock); - natsSub_Unlock(sub); + nats_unlockSubAndDispatcher(sub); natsSub_close(sub, false); testCond(s == NATS_OK); test("Check msg not given: "); - natsSub_Lock(sub); - natsMutex_Lock(sub->libDlvWorker->lock); - testCond(sub->msgList.msgs == 0); - natsMutex_Unlock(sub->libDlvWorker->lock); - natsSub_Unlock(sub); + nats_lockSubAndDispatcher(sub); + testCond(sub->ownDispatcher.queue.msgs == 0); + nats_unlockSubAndDispatcher(sub); natsSubscription_Destroy(sub); natsConnection_Destroy(nc); @@ -12069,8 +11734,7 @@ test_SubRemovedWhileProcessingMsg(void) _stopServer(serverPid); } -static void -test_RequestTimeout(void) +void test_RequestTimeout(void) { natsStatus s; natsConnection *nc = NULL; @@ -12090,8 +11754,7 @@ test_RequestTimeout(void) _stopServer(serverPid); } -static void -test_Request(void) +void test_Request(void) { natsStatus s; natsConnection *nc = NULL; @@ -12204,8 +11867,7 @@ test_Request(void) _stopServer(serverPid); } -static void -test_RequestNoBody(void) +void test_RequestNoBody(void) { natsStatus s; natsConnection *nc = NULL; @@ -12320,8 +11982,7 @@ _serverForMuxWithMappedSubject(void *closure) natsSock_Close(sock); } -static void -test_RequestMuxWithMappedSubject(void) +void test_RequestMuxWithMappedSubject(void) { natsStatus s; natsConnection *nc = NULL; @@ -12370,8 +12031,7 @@ test_RequestMuxWithMappedSubject(void) _destroyDefaultThreadArgs(&arg); } -static void -test_OldRequest(void) +void test_OldRequest(void) { natsStatus s; natsConnection *nc = NULL; @@ -12455,8 +12115,7 @@ _sendRequest(void *closure) natsMsg_Destroy(msg); } -static void -test_SimultaneousRequest(void) +void test_SimultaneousRequest(void) { natsStatus s; natsConnection *nc = NULL; @@ -12519,8 +12178,7 @@ test_SimultaneousRequest(void) _stopServer(serverPid); } -static void -test_RequestClose(void) +void test_RequestClose(void) { natsStatus s; natsConnection *nc = NULL; @@ -12557,8 +12215,7 @@ test_RequestClose(void) } -static void -test_CustomInbox(void) +void test_CustomInbox(void) { natsStatus s; natsConnection *nc = NULL; @@ -12689,8 +12346,7 @@ test_CustomInbox(void) _stopServer(serverPid); } -static void -test_MessageBufferPadding(void) +void test_MessageBufferPadding(void) { natsStatus s; natsConnection *nc = NULL; @@ -12746,8 +12402,7 @@ test_MessageBufferPadding(void) _stopServer(serverPid); } -static void -test_FlushInCb(void) +void test_FlushInCb(void) { natsStatus s; natsConnection *nc = NULL; @@ -12790,8 +12445,7 @@ test_FlushInCb(void) _stopServer(serverPid); } -static void -test_ReleaseFlush(void) +void test_ReleaseFlush(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -12869,8 +12523,7 @@ test_ReleaseFlush(void) } -static void -test_FlushErrOnDisconnect(void) +void test_FlushErrOnDisconnect(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -12945,8 +12598,7 @@ test_FlushErrOnDisconnect(void) _destroyDefaultThreadArgs(&arg); } -static void -test_Inbox(void) +void test_Inbox(void) { natsStatus s; natsInbox *inbox = NULL; @@ -12960,8 +12612,7 @@ test_Inbox(void) natsInbox_Destroy(inbox); } -static void -test_Stats(void) +void test_Stats(void) { natsStatus s; natsConnection *nc = NULL; @@ -13024,8 +12675,7 @@ test_Stats(void) _stopServer(serverPid); } -static void -test_BadSubject(void) +void test_BadSubject(void) { natsStatus s; natsConnection *nc = NULL; @@ -13050,8 +12700,7 @@ test_BadSubject(void) _stopServer(serverPid); } -static void -test_SubBadSubjectAndQueueName(void) +void test_SubBadSubjectAndQueueName(void) { natsStatus s; natsConnection *nc = NULL; @@ -13140,8 +12789,7 @@ _subComplete(void *closure) natsMutex_Unlock(arg->m); } -static void -test_ClientAsyncAutoUnsub(void) +void test_ClientAsyncAutoUnsub(void) { natsStatus s; natsConnection *nc = NULL; @@ -13216,8 +12864,7 @@ test_ClientAsyncAutoUnsub(void) _stopServer(serverPid); } -static void -test_ClientSyncAutoUnsub(void) +void test_ClientSyncAutoUnsub(void) { natsStatus s; natsConnection *nc = NULL; @@ -13282,8 +12929,7 @@ test_ClientSyncAutoUnsub(void) _stopServer(serverPid); } -static void -test_ClientAutoUnsubAndReconnect(void) +void test_ClientAutoUnsubAndReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -13337,7 +12983,10 @@ test_ClientAutoUnsubAndReconnect(void) nats_Sleep(10); test("Received no more than max: "); - testCond((s == NATS_OK) && (arg.sum == 10)); + natsMutex_Lock(arg.m); + int sum = arg.sum; + natsMutex_Unlock(arg.m); + testCond((s == NATS_OK) && (sum == 10)); natsSubscription_Destroy(sub); natsConnection_Destroy(nc); @@ -13362,8 +13011,7 @@ _autoUnsub(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closur } -static void -test_AutoUnsubNoUnsubOnDestroy(void) +void test_AutoUnsubNoUnsubOnDestroy(void) { natsStatus s; natsConnection *nc = NULL; @@ -13390,6 +13038,7 @@ test_AutoUnsubNoUnsubOnDestroy(void) natsMutex_Lock(arg.m); while ((s != NATS_TIMEOUT) && !arg.done) s = natsCondition_TimedWait(arg.c, arg.m, 2000); + natsMutex_Unlock(arg.m); testCond(s == NATS_OK); natsConnection_Destroy(nc); @@ -13408,8 +13057,7 @@ test_AutoUnsubNoUnsubOnDestroy(void) natsBuf_Destroy(buf); } -static void -test_NextMsgOnClosedSub(void) +void test_NextMsgOnClosedSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -13445,8 +13093,7 @@ _nextMsgKickedOut(void *closure) (void) natsSubscription_NextMsg(&msg, sub, 10000); } -static void -test_CloseSubRelease(void) +void test_CloseSubRelease(void) { natsStatus s; natsConnection *nc = NULL; @@ -13494,8 +13141,7 @@ test_CloseSubRelease(void) _stopServer(serverPid); } -static void -test_IsValidSubscriber(void) +void test_IsValidSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -13539,8 +13185,7 @@ test_IsValidSubscriber(void) _stopServer(serverPid); } -static void -test_SlowSubscriber(void) +void test_SlowSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -13593,8 +13238,7 @@ test_SlowSubscriber(void) _stopServer(serverPid); } -static void -test_SlowAsyncSubscriber(void) +void test_SlowAsyncSubscriber(void) { natsStatus s; natsConnection *nc = NULL; @@ -13622,12 +13266,14 @@ test_SlowAsyncSubscriber(void) CHECK_SERVER_STARTED(serverPid); s = natsConnection_Connect(&nc, opts); - IFOK(s, natsConnection_Subscribe(&sub, nc, "foo", _recvTestString, (void*) &arg)); + IFOK(s, natsConnection_Subscribe(&sub, nc, "foo.*", _recvTestString, (void*) &arg)); for (int i=0; - (s == NATS_OK) && (i < (total + 100)); i++) + (s == NATS_OK) && (i < (2 * total)); i++) { - s = natsConnection_PublishString(nc, "foo", "hello"); + char subj[32]; + snprintf(subj, sizeof(subj), "foo.%d", i); + s = natsConnection_PublishString(nc, subj, "hello"); } test("Check Publish does not fail due to SlowConsumer: "); @@ -13660,21 +13306,21 @@ test_SlowAsyncSubscriber(void) // Release the sub natsMutex_Lock(arg.m); - // Unblock the wait - arg.closed = true; - // And destroy the subscription here so that the next msg callback // is not invoked. natsSubscription_Destroy(sub); + // Unblock the wait, let the callback finish + test("Unblock and wait for the callback to finish: "); + arg.closed = true; natsCondition_Signal(arg.c); arg.msgReceived = false; natsMutex_Unlock(arg.m); - // Let the callback finish natsMutex_Lock(arg.m); while (!arg.msgReceived) natsCondition_TimedWait(arg.c, arg.m, 5000); + testCond(arg.msgReceived); natsMutex_Unlock(arg.m); natsOptions_Destroy(opts); @@ -13702,8 +13348,7 @@ _slowConsErrCB(natsConnection *nc, natsSubscription *sub, natsStatus err, void * natsMutex_Unlock(arg->m); } -static void -test_SlowConsumerCB(void) +void test_SlowConsumerCB(void) { natsStatus s; natsConnection *nc = NULL; @@ -13761,8 +13406,7 @@ test_SlowConsumerCB(void) _destroyDefaultThreadArgs(&arg); } -static void -test_PendingLimitsDeliveredAndDropped(void) +void test_PendingLimitsDeliveredAndDropped(void) { natsStatus s; natsConnection *nc = NULL; @@ -13991,8 +13635,7 @@ test_PendingLimitsDeliveredAndDropped(void) _stopServer(serverPid); } -static void -test_PendingLimitsWithSyncSub(void) +void test_PendingLimitsWithSyncSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -14059,8 +13702,7 @@ test_PendingLimitsWithSyncSub(void) _stopServer(serverPid); } -static void -test_AsyncSubscriptionPending(void) +void test_AsyncSubscriptionPending(void) { natsStatus s; natsConnection *nc = NULL; @@ -14172,8 +13814,7 @@ test_AsyncSubscriptionPending(void) _stopServer(serverPid); } -static void -test_AsyncSubscriptionPendingDrain(void) +void test_AsyncSubscriptionPendingDrain(void) { natsStatus s; natsConnection *nc = NULL; @@ -14237,8 +13878,7 @@ test_AsyncSubscriptionPendingDrain(void) _stopServer(serverPid); } -static void -test_SyncSubscriptionPending(void) +void test_SyncSubscriptionPending(void) { natsStatus s; natsConnection *nc = NULL; @@ -14323,8 +13963,7 @@ test_SyncSubscriptionPending(void) _stopServer(serverPid); } -static void -test_SyncSubscriptionPendingDrain(void) +void test_SyncSubscriptionPendingDrain(void) { natsStatus s; natsConnection *nc = NULL; @@ -14416,8 +14055,7 @@ _asyncErrCb(natsConnection *nc, natsSubscription *sub, natsStatus err, void* clo natsMutex_Unlock(arg->m); } -static void -test_AsyncErrHandler_MaxPendingMsgs(void) +void test_AsyncErrHandlerMaxPendingMsgs(void) { natsStatus s; natsConnection *nc = NULL; @@ -14479,8 +14117,7 @@ test_AsyncErrHandler_MaxPendingMsgs(void) _stopServer(serverPid); } -static void -test_AsyncErrHandler_MaxPendingBytes(void) +void test_AsyncErrHandlerMaxPendingBytes(void) { natsStatus s; natsConnection* nc = NULL; @@ -14575,8 +14212,7 @@ _asyncErrBlockingCb(natsConnection *nc, natsSubscription *sub, natsStatus err, v natsMutex_Unlock(arg->m); } -static void -test_AsyncErrHandlerSubDestroyed(void) +void test_AsyncErrHandlerSubDestroyed(void) { natsStatus s; natsConnection *nc = NULL; @@ -14711,8 +14347,7 @@ _startCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void *closure) natsMsg_Destroy(msg); } -static void -test_AsyncSubscriberStarvation(void) +void test_AsyncSubscriberStarvation(void) { natsStatus s; natsConnection *nc = NULL; @@ -14761,8 +14396,7 @@ test_AsyncSubscriberStarvation(void) _stopServer(serverPid); } -static void -test_AsyncSubscriberOnClose(void) +void test_AsyncSubscriberOnClose(void) { natsStatus s; natsConnection *nc = NULL; @@ -14833,8 +14467,7 @@ test_AsyncSubscriberOnClose(void) _stopServer(serverPid); } -static void -test_NextMsgCallOnAsyncSub(void) +void test_NextMsgCallOnAsyncSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -14894,8 +14527,7 @@ testOnCompleteMsgHandler(natsConnection *nc, natsSubscription *sub, natsMsg *msg natsMsg_Destroy(msg); } -static void -test_SubOnComplete(void) +void test_SubOnComplete(void) { natsStatus s; natsConnection *nc = NULL; @@ -14978,8 +14610,7 @@ test_SubOnComplete(void) _stopServer(serverPid); } -static void -test_ServersOption(void) +void test_ServersOption(void) { natsStatus s; natsConnection *nc = NULL; @@ -15043,8 +14674,7 @@ test_ServersOption(void) _stopServer(serverPid); } -static void -test_AuthServers(void) +void test_AuthServers(void) { natsStatus s; natsConnection *nc = NULL; @@ -15098,8 +14728,7 @@ test_AuthServers(void) _stopServer(serverPid2); } -static void -test_AuthFailToReconnect(void) +void test_AuthFailToReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -15186,8 +14815,7 @@ test_AuthFailToReconnect(void) _stopServer(serverPid3); } -static void -test_BasicClusterReconnect(void) +void test_BasicClusterReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -15304,8 +14932,7 @@ _reconnectTokenHandler(void* closure) return token; } -static void -test_ReconnectWithTokenHandler(void) +void test_ReconnectWithTokenHandler(void) { natsStatus s; natsConnection *nc = NULL; @@ -15414,8 +15041,7 @@ struct hashCount }; -static void -test_HotSpotReconnect(void) +void test_HotSpotReconnect(void) { natsStatus s; natsConnection *nc[NUM_CLIENTS]; @@ -15568,8 +15194,7 @@ test_HotSpotReconnect(void) _stopServer(serverPid3); } -static void -test_ProperReconnectDelay(void) +void test_ProperReconnectDelay(void) { natsStatus s; natsConnection *nc = NULL; @@ -15638,8 +15263,7 @@ test_ProperReconnectDelay(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ProperFalloutAfterMaxAttempts(void) +void test_ProperFalloutAfterMaxAttempts(void) { natsStatus s; natsConnection *nc = NULL; @@ -15710,8 +15334,7 @@ test_ProperFalloutAfterMaxAttempts(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StopReconnectAfterTwoAuthErr(void) +void test_StopReconnectAfterTwoAuthErr(void) { natsStatus s; natsConnection *nc = NULL; @@ -15802,8 +15425,7 @@ test_StopReconnectAfterTwoAuthErr(void) _stopServer(serverPid2); } -static void -test_TimeoutOnNoServer(void) +void test_TimeoutOnNoServer(void) { natsStatus s; natsConnection *nc = NULL; @@ -15881,8 +15503,7 @@ test_TimeoutOnNoServer(void) _destroyDefaultThreadArgs(&arg); } -static void -test_PingReconnect(void) +void test_PingReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -15966,8 +15587,7 @@ test_PingReconnect(void) } -static void -test_GetServers(void) +void test_GetServers(void) { natsStatus s; natsConnection *conn = NULL; @@ -16051,8 +15671,7 @@ test_GetServers(void) _stopServer(s1Pid); } -static void -test_GetDiscoveredServers(void) +void test_GetDiscoveredServers(void) { natsStatus s; natsConnection *conn = NULL; @@ -16111,8 +15730,7 @@ _discoveredServersCb(natsConnection *nc, void *closure) natsMutex_Unlock(arg->m); } -static void -test_DiscoveredServersCb(void) +void test_DiscoveredServersCb(void) { natsStatus s; natsConnection *conn = NULL; @@ -16174,8 +15792,7 @@ test_DiscoveredServersCb(void) _destroyDefaultThreadArgs(&arg); } -static void -test_IgnoreDiscoveredServers(void) +void test_IgnoreDiscoveredServers(void) { natsStatus s; natsConnection *conn = NULL; @@ -16293,8 +15910,7 @@ _serverSendsINFOAfterPONG(void *closure) natsSock_Close(sock); } -static void -test_ReceiveINFORightAfterFirstPONG(void) +void test_ReceiveINFORightAfterFirstPONG(void) { natsStatus s = NATS_OK; natsThread *t = NULL; @@ -16370,8 +15986,7 @@ test_ReceiveINFORightAfterFirstPONG(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ServerPoolUpdatedOnClusterUpdate(void) +void test_ServerPoolUpdatedOnClusterUpdate(void) { natsStatus s; natsConnection *conn = NULL; @@ -16637,8 +16252,7 @@ test_ServerPoolUpdatedOnClusterUpdate(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ReconnectJitter(void) +void test_ReconnectJitter(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -16760,8 +16374,7 @@ _customReconnectDelayCB(natsConnection *nc, int attempts, void *closure) return delay; } -static void -test_CustomReconnectDelay(void) +void test_CustomReconnectDelay(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -16917,8 +16530,7 @@ _lameDuckMockupServerThread(void *closure) natsSock_Close(sock); } -static void -test_LameDuckMode(void) +void test_LameDuckMode(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -16999,8 +16611,7 @@ test_LameDuckMode(void) _destroyDefaultThreadArgs(&arg); } -static void -test_Version(void) +void test_Version(void) { const char *str = NULL; @@ -17016,8 +16627,7 @@ test_Version(void) testCond(nats_GetVersionNumber() == LIB_NATS_VERSION_NUMBER); } -static void -test_VersionMatchesTag(void) +void test_VersionMatchesTag(void) { natsStatus s = NATS_OK; const char *tag; @@ -17084,8 +16694,7 @@ _openCloseAndWaitThread(void *closure) natsLib_Release(); } -static void -test_OpenCloseAndWait(void) +void test_OpenCloseAndWait(void) { natsStatus s; natsConnection *nc = NULL; @@ -17211,8 +16820,7 @@ _testGetLastErrInThread(void *arg) natsOptions_Destroy(opts); } -static void -test_GetLastError(void) +void test_GetLastError(void) { natsStatus s, getLastErrSts; natsOptions *opts = NULL; @@ -17326,8 +16934,7 @@ test_GetLastError(void) nats_clearLastError(); } -static void -test_StaleConnection(void) +void test_StaleConnection(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -17448,8 +17055,7 @@ test_StaleConnection(void) _destroyDefaultThreadArgs(&arg); } -static void -test_ServerErrorClosesConnection(void) +void test_ServerErrorClosesConnection(void) { natsStatus s = NATS_OK; natsSock sock = NATS_SOCK_INVALID; @@ -17553,8 +17159,7 @@ test_ServerErrorClosesConnection(void) _destroyDefaultThreadArgs(&arg); } -static void -test_NoEcho(void) +void test_NoEcho(void) { natsStatus s; natsOptions *opts = NULL; @@ -17661,8 +17266,7 @@ _startMockupServerThread(void *closure) natsSock_Close(sock); } -static void -test_NoEchoOldServer(void) +void test_NoEchoOldServer(void) { natsStatus s; natsConnection *conn = NULL; @@ -17718,8 +17322,7 @@ test_NoEchoOldServer(void) _destroyDefaultThreadArgs(&arg); } -static void -test_DrainSub(void) +void test_DrainSub(void) { natsStatus s; natsConnection *nc = NULL; @@ -17954,6 +17557,7 @@ test_DrainSub(void) arg.closed = true; natsCondition_Signal(arg.c); natsMutex_Unlock(arg.m); + testCond(s == NATS_OK); test("Wait for Drain to complete: "); s = natsSubscription_WaitForDrainCompletion(sub, -1); @@ -18064,8 +17668,7 @@ _drainSubCompleteCB(void *closure) natsMutex_Unlock(arg->m); } -static void -test_DrainSubStops(void) +void test_DrainSubStops(void) { natsStatus s; natsConnection *nc = NULL; @@ -18182,8 +17785,7 @@ test_DrainSubStops(void) _stopServer(pid); } -static void -test_DrainSubRaceOnAutoUnsub(void) +void test_DrainSubRaceOnAutoUnsub(void) { natsStatus s; natsConnection *nc = NULL; @@ -18234,8 +17836,7 @@ test_DrainSubRaceOnAutoUnsub(void) _stopServer(pid); } -static void -test_DrainSubNotResentOnReconnect(void) +void test_DrainSubNotResentOnReconnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -18381,8 +17982,7 @@ _drainConnErrHandler(natsConnection *nc, natsSubscription *sub, natsStatus err, natsMutex_Unlock(args->m); } -static void -test_DrainConn(void) +void test_DrainConn(void) { natsStatus s; natsConnection *nc = NULL; @@ -18667,8 +18267,7 @@ _noDoubleCbSubCb(natsConnection *nc, natsSubscription *sub, natsMsg *msg, void * natsMsg_Destroy(msg); } -static void -test_NoDoubleConnClosedOnDrain(void) +void test_NoDoubleConnClosedOnDrain(void) { natsStatus s; natsConnection *nc = NULL; @@ -18731,8 +18330,7 @@ test_NoDoubleConnClosedOnDrain(void) _stopServer(pid); } -static void -test_GetClientID(void) +void test_GetClientID(void) { natsStatus s; natsPid pid1 = NATS_INVALID_PID; @@ -18754,7 +18352,7 @@ test_GetClientID(void) testCond(true); return; } - pid1 = _startServer("nats://127.0.0.1:4222", "-cluster nats://127.0.0.1:6222 -cluster_name abc", true); + pid1 = _startServer("nats://127.0.0.1:4222", "-a 127.0.0.1 -p 4222 -cluster nats://127.0.0.1:6222 -cluster_name abc", true); CHECK_SERVER_STARTED(pid1); test("Create nc1: "); @@ -18773,7 +18371,7 @@ test_GetClientID(void) testCond((s == NATS_OK) && (cid != 0)); test("Wait for discovered callback: "); - pid2 = _startServer("nats://127.0.0.1:4223", "-p 4223 -cluster nats://127.0.0.1:6223 -cluster_name abc -routes nats://127.0.0.1:6222", true); + pid2 = _startServer("nats://127.0.0.1:4223", "-a 127.0.0.1 -p 4223 -cluster nats://127.0.0.1:6223 -cluster_name abc -routes nats://127.0.0.1:6222", true); CHECK_SERVER_STARTED(pid2); natsMutex_Lock(arg.m); @@ -18858,8 +18456,7 @@ test_GetClientID(void) _destroyDefaultThreadArgs(&arg); } -static void -test_GetClientIP(void) +void test_GetClientIP(void) { natsStatus s; natsConnection *nc = NULL; @@ -18966,8 +18563,7 @@ test_GetClientIP(void) _destroyDefaultThreadArgs(&arg); } -static void -test_GetRTT(void) +void test_GetRTT(void) { natsStatus s; natsConnection *nc = NULL; @@ -19009,8 +18605,7 @@ test_GetRTT(void) natsOptions_Destroy(opts); } -static void -test_GetLocalIPAndPort(void) +void test_GetLocalIPAndPort(void) { natsStatus s; natsConnection *nc = NULL; @@ -19170,8 +18765,7 @@ _checkJWTAndSigCB(char *buffer) return NATS_OK; } -static void -test_UserCredsCallbacks(void) +void test_UserCredsCallbacks(void) { natsStatus s; natsConnection *nc = NULL; @@ -19355,8 +18949,7 @@ test_UserCredsCallbacks(void) _destroyDefaultThreadArgs(&arg); } -static void -test_UserCredsFromMemory(void) +void test_UserCredsFromMemory(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -19464,8 +19057,7 @@ test_UserCredsFromMemory(void) natsOptions_Destroy(opts); } -static void -test_UserCredsFromFiles(void) +void test_UserCredsFromFiles(void) { natsStatus s = NATS_OK; natsConnection *nc = NULL; @@ -19589,7 +19181,7 @@ test_UserCredsFromFiles(void) // Use a file that contains no userJWT.. test("UserOrChainedFile has no JWT: "); - s = natsOptions_SetUserCredentialsFromFiles(opts, "list.txt", NULL); + s = natsOptions_SetUserCredentialsFromFiles(opts, "list_test.txt", NULL); IFOK(s, natsConnection_Connect(&nc, opts)); // Since we return the whole content of the file when we don't find // the key for the user, but we don't for seed, the error we'll get @@ -19605,7 +19197,7 @@ test_UserCredsFromFiles(void) // Use a seed file that contains no seed.. test("SeedFile has no seed: "); - s = natsOptions_SetUserCredentialsFromFiles(opts, ucfn, "list.txt"); + s = natsOptions_SetUserCredentialsFromFiles(opts, ucfn, "list_test.txt"); IFOK(s, natsConnection_Connect(&nc, opts)); testCond((s == NATS_ERR) && (strstr(nats_GetLastError(NULL), "no nkey user seed found") != NULL)); @@ -19705,8 +19297,7 @@ _checkNKeyAndSig(char *buffer) return NATS_OK; } -static void -test_NKey(void) +void test_NKey(void) { natsStatus s; natsOptions *opts = NULL; @@ -19835,8 +19426,7 @@ _checkNKeyFromSeed(char *buffer) return NATS_OK; } -static void -test_NKeyFromSeed(void) +void test_NKeyFromSeed(void) { natsStatus s; natsOptions *opts = NULL; @@ -19971,8 +19561,7 @@ test_NKeyFromSeed(void) remove("seed.file"); } -static void -test_ConnSign(void) +void test_ConnSign(void) { natsStatus s; natsConnection *nc = NULL; @@ -20049,8 +19638,7 @@ test_ConnSign(void) remove(ucfn); } -static void -test_WriteDeadline(void) +void test_WriteDeadline(void) { natsStatus s; natsOptions *opts = NULL; @@ -20132,8 +19720,7 @@ _publish(void *arg) } -static void -test_NoPartialOnReconnect(void) +void test_NoPartialOnReconnect(void) { natsStatus s; natsOptions *opts = NULL; @@ -20232,8 +19819,7 @@ test_NoPartialOnReconnect(void) _stopServer(pid); } -static void -test_ForcedReconnect(void) +void test_ForcedReconnect(void) { natsStatus s; struct threadArg arg; @@ -20252,6 +19838,7 @@ test_ForcedReconnect(void) CHECK_SERVER_STARTED(pid); IFOK(s, natsOptions_Create(&opts)); IFOK(s, natsOptions_SetReconnectedCB(opts, _reconnectedCb, &arg)); + IFOK(s, natsOptions_SetReconnectWait(opts, 100)); IFOK(s, natsConnection_Connect(&nc, opts)); IFOK(s, natsConnection_SubscribeSync(&sub, nc, "foo")); testCond(s == NATS_OK); @@ -20325,8 +19912,7 @@ _stopServerInThread(void *closure) _stopServer(pid); } -static void -test_ReconnectFailsPendingRequest(void) +void test_ReconnectFailsPendingRequest(void) { natsStatus s; natsOptions *opts = NULL; @@ -20380,8 +19966,7 @@ test_ReconnectFailsPendingRequest(void) } } -static void -test_HeadersNotSupported(void) +void test_HeadersNotSupported(void) { natsStatus s; natsConnection *conn = NULL; @@ -20453,8 +20038,7 @@ test_HeadersNotSupported(void) _destroyDefaultThreadArgs(&arg); } -static void -test_HeadersBasic(void) +void test_HeadersBasic(void) { natsStatus s; natsConnection *nc = NULL; @@ -20563,8 +20147,7 @@ _msgFilterDropMsg(natsConnection *nc, natsMsg **msg, void *closure) natsConn_setFilter(nc, NULL); } -static void -test_natsMsgsFilter(void) +void test_natsMsgsFilter(void) { natsStatus s; natsConnection *nc = NULL; @@ -20709,8 +20292,7 @@ _eventLoop(void *closure) } } -static void -test_EventLoop(void) +void test_EventLoop(void) { natsStatus s; natsConnection *nc = NULL; @@ -20800,6 +20382,7 @@ test_EventLoop(void) natsMutex_Lock(arg.m); if (arg.attached != 2 || !arg.detached) s = NATS_ERR; + natsMutex_Unlock(arg.m); testCond(s == NATS_OK); natsSubscription_Destroy(sub); @@ -20811,8 +20394,7 @@ test_EventLoop(void) _stopServer(pid); } -static void -test_EventLoopRetryOnFailedConnect(void) +void test_EventLoopRetryOnFailedConnect(void) { natsStatus s; natsConnection *nc = NULL; @@ -20899,8 +20481,7 @@ test_EventLoopRetryOnFailedConnect(void) _stopServer(pid); } -static void -test_EventLoopTLS(void) +void test_EventLoopTLS(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -20990,8 +20571,7 @@ test_EventLoopTLS(void) #endif } -static void -test_SSLBasic(void) +void test_SSLBasic(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21059,8 +20639,7 @@ test_SSLBasic(void) #endif } -static void -test_SSLVerify(void) +void test_SSLVerify(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21125,8 +20704,7 @@ test_SSLVerify(void) #endif } -static void -test_SSLLoadCAFromMemory(void) +void test_SSLLoadCAFromMemory(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21206,8 +20784,7 @@ test_SSLLoadCAFromMemory(void) #endif } -static void -test_SSLCertAndKeyFromMemory(void) +void test_SSLCertAndKeyFromMemory(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21300,8 +20877,7 @@ test_SSLCertAndKeyFromMemory(void) #endif } -static void -test_SSLVerifyHostname(void) +void test_SSLVerifyHostname(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21319,20 +20895,31 @@ test_SSLVerifyHostname(void) serverPid = _startServer("nats://127.0.0.1:4443", "-config tls_noip.conf", true); CHECK_SERVER_STARTED(serverPid); +#if defined(NATS_FORCE_HOST_VERIFICATION) test("Check that connect fails if url is IP: "); +#else + test("Check that connect succeeds if url is IP: "); +#endif s = natsOptions_SetURL(opts, "nats://127.0.0.1:4443"); IFOK(s, natsOptions_SetSecure(opts, true)); - // For test purposes, we provide the CA trusted certs IFOK(s, natsOptions_LoadCATrustedCertificates(opts, "certs/ca.pem")); IFOK(s, natsOptions_SetReconnectedCB(opts, _reconnectedCb, &args)); IFOK(s, natsConnection_Connect(&nc, opts)); +#if defined(NATS_FORCE_HOST_VERIFICATION) testCond(s == NATS_SSL_ERROR); + nats_clearLastError(); +#else + testCond(s == NATS_OK); + natsConnection_Destroy(nc); + nc = NULL; +#endif test("Check that connect fails if wrong expected hostname: "); s = natsOptions_SetURL(opts, "nats://localhost:4443"); IFOK(s, natsOptions_SetExpectedHostname(opts, "foo")); IFOK(s, natsConnection_Connect(&nc, opts)); testCond(s == NATS_SSL_ERROR); + nats_clearLastError(); test("Check that connect succeeds if hostname ok and no expected hostname set: "); s = natsOptions_SetURL(opts, "nats://localhost:4443"); @@ -21379,8 +20966,7 @@ test_SSLVerifyHostname(void) #endif } -static void -test_SSLSkipServerVerification(void) +void test_SSLSkipServerVerification(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21439,8 +21025,7 @@ test_SSLSkipServerVerification(void) #endif } -static void -test_SSLCiphers(void) +void test_SSLCiphers(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21573,8 +21158,7 @@ _sslMT(void *closure) #define SSL_THREADS (3) #endif -static void -test_SSLMultithreads(void) +void test_SSLMultithreads(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21641,8 +21225,7 @@ test_SSLMultithreads(void) #endif } -static void -test_SSLConnectVerboseOption(void) +void test_SSLConnectVerboseOption(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21709,6 +21292,166 @@ test_SSLConnectVerboseOption(void) #endif } +void test_SSLHandshakeFirst(void) +{ +#if defined(NATS_HAS_TLS) + natsStatus s; + natsConnection *nc = NULL; + natsOptions *opts = NULL; + natsPid serverPid = NATS_INVALID_PID; + + if (!serverVersionAtLeast(2, 10, 0)) + { + char txt[200]; + + snprintf(txt, sizeof(txt), "Skipping since requires server version of at least 2.10.0, got %s: ", serverVersion); + test(txt); + testCond(true); + return; + } + + serverPid = _startServer("nats://127.0.0.1:4443", "-config tlsfirst.conf", true); + CHECK_SERVER_STARTED(serverPid); + + test("Set options: "); + s = natsOptions_Create(&opts); + IFOK(s, natsOptions_SetURL(opts, "nats://127.0.0.1:4443")); + IFOK(s, natsOptions_SetSecure(opts, true)); + IFOK(s, natsOptions_SkipServerVerification(opts, true)); + IFOK(s, natsOptions_SetTimeout(opts, 500)); + testCond(s == NATS_OK); + + test("Check that connect fails if option not set: "); + s = natsConnection_Connect(&nc, opts); + testCond(s != NATS_OK); + nats_clearLastError(); + + test("Set TLSHandshakeFirst option error: "); + s = natsOptions_TLSHandshakeFirst(NULL); + testCond(s == NATS_INVALID_ARG); + nats_clearLastError(); + + test("Set TLSHandshakeFirst option: "); + s = natsOptions_TLSHandshakeFirst(opts); + testCond(s == NATS_OK); + + test("Set TLSHandshakeFirst option without setting secure: "); + { + // we start with a new natsOptions struct so that we can test + // that it does not crash with a minimal config + natsOptions *no_secure_opts = NULL; + s = natsOptions_Create(&no_secure_opts); + IFOK(s, natsOptions_SetURL(no_secure_opts, "nats://127.0.0.1:4443")); + IFOK(s, natsOptions_SetTimeout(no_secure_opts, 500)); + IFOK(s, natsOptions_TLSHandshakeFirst(no_secure_opts)); + IFOK(s, natsConnection_Connect(&nc, no_secure_opts)); + // expecting an error because cert valiation will fail; the goal here is to avoid a crash + testCond(s == NATS_SSL_ERROR); + natsOptions_Destroy(no_secure_opts); + nats_clearLastError(); + } + + test("Check that connect succeeds: "); + s = natsConnection_Connect(&nc, opts); + testCond(s == NATS_OK); + natsConnection_Destroy(nc); + nc = NULL; + + _stopServer(serverPid); + serverPid = _startServer("nats://127.0.0.1:4443", "-config tls.conf", true); + CHECK_SERVER_STARTED(serverPid); + + test("Check that connect fails if option is set but not in the server: "); + s = natsConnection_Connect(&nc, opts); + testCond(s != NATS_OK); + nats_clearLastError(); + + natsOptions_Destroy(opts); + +#else + test("Skipped when built with no SSL support: "); + testCond(true); +#endif +} + +void test_SSLServerNameIndication(void) +{ +#if defined(NATS_HAS_TLS) + natsStatus s = NATS_OK; + natsSock sock = NATS_SOCK_INVALID; + natsThread *t = NULL; + struct threadArg arg; + natsSockCtx ctx; + static const char *server = "tls://localhost:4222"; + + memset(&ctx, 0, sizeof(natsSockCtx)); + + s = _createDefaultThreadArgsForCbTests(&arg); + IFOK(s, natsOptions_Create(&(arg.opts))); + IFOK(s, natsOptions_SetSecure(arg.opts, true)); + IFOK(s, natsOptions_TLSHandshakeFirst(arg.opts)); + IFOK(s, natsOptions_SetServers(arg.opts, &server, 1)); + if (s != NATS_OK) + FAIL("@@ Unable to setup test!"); + + test("Start server and connect client: ") + + arg.control = 3; + + s = _startMockupServer(&sock, "localhost", "4222"); + + // Start the thread that will try to connect to our server... + IFOK(s, natsThread_Create(&t, _connectToMockupServer, (void*) &arg)); + + if ((s == NATS_OK) + && (((ctx.fd = accept(sock, NULL, NULL)) == NATS_SOCK_INVALID) + || natsSock_SetCommonTcpOptions(ctx.fd) != NATS_OK)) + { + s = NATS_SYS_ERROR; + } + + testCond((s == NATS_OK) && (ctx.fd > 0)); + + test("Read ClientHello from client: "); + char buffer[1024]; + memset(buffer, 0, sizeof(buffer)); + + int size = recv(ctx.fd, buffer, sizeof(buffer), 0); + testCond(size > 0); + + // remove all null chars to allow the use of strstr on the result + for (int i = 0; i < size; ++i) { + if (buffer[i] == 0) + buffer[i] = '0'; + } + + test("Check hostname is found in ClientHello: "); + bool found = strstr(buffer, "localhost"); +#if defined(NATS_USE_OPENSSL_1_1) + testCond(found == true); +#else + testCond(found == false); +#endif + + // Need to close those for the client side to unblock. + natsSock_Close(ctx.fd); + natsSock_Close(sock); + + // Wait for the client to finish. + if (t != NULL) + { + natsThread_Join(t); + natsThread_Destroy(t); + } + + _destroyDefaultThreadArgs(&arg); + +#else + test("Skipped when built with no SSL support: "); + testCond(true); +#endif +} + #if defined(NATS_HAS_TLS) static natsStatus _elDummyAttach(void **userData, void *loop, natsConnection *nc, natsSock socket) { return NATS_OK; } @@ -21723,8 +21466,7 @@ static natsStatus _elDummyDetach(void *userData) { return NATS_OK; } #endif -static void -test_SSLSocketLeakWithEventLoop(void) +void test_SSLSocketLeakWithEventLoop(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21767,8 +21509,7 @@ test_SSLSocketLeakWithEventLoop(void) #endif } -static void -test_SSLReconnectWithAuthError(void) +void test_SSLReconnectWithAuthError(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21790,10 +21531,10 @@ test_SSLReconnectWithAuthError(void) if (opts == NULL) FAIL("Unable to create reconnect options!"); - pid1 = _startServer("nats://127.0.0.1:4443", "-p 4443 -cluster_name abc -cluster nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); + pid1 = _startServer("nats://127.0.0.1:4443", "-a 127.0.0.1 -p 4443 -cluster_name abc -cluster nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); CHECK_SERVER_STARTED(pid1); - pid2 = _startServer("nats://127.0.0.1:4444", "-p 4444 -cluster_name abc -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); + pid2 = _startServer("nats://127.0.0.1:4444", "-a 127.0.0.1 -p 4444 -cluster_name abc -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -tls -tlscert certs/server-cert.pem -tlskey certs/server-key.pem -tlscacert certs/ca.pem -user user -pass pwd", true); CHECK_SERVER_STARTED(pid2); test("Connect to server1: "); @@ -21823,8 +21564,7 @@ test_SSLReconnectWithAuthError(void) #endif } -static void -test_SSLAvailable(void) +void test_SSLAvailable(void) { #if defined(NATS_HAS_TLS) natsStatus s; @@ -21983,8 +21723,7 @@ _createConfFile(char *buf, int bufLen, const char *content) fclose(f); } -static void -test_ReconnectImplicitUserInfo(void) +void test_ReconnectImplicitUserInfo(void) { natsStatus s; natsPid pid1 = NATS_INVALID_PID; @@ -22009,7 +21748,7 @@ test_ReconnectImplicitUserInfo(void) "}\n"\ "no_auth_user: b\n"); test("Start server1: "); - snprintf(cmdLine, sizeof(cmdLine), "-cluster_name \"local\" -cluster nats://127.0.0.1:6222 -c %s", conf); + snprintf(cmdLine, sizeof(cmdLine), "-a 127.0.0.1 -p 4222 -cluster_name \"local\" -cluster nats://127.0.0.1:6222 -c %s", conf); pid1 = _startServer("nats://127.0.0.1:4222", cmdLine, true); CHECK_SERVER_STARTED(pid1); testCond(true); @@ -22025,7 +21764,7 @@ test_ReconnectImplicitUserInfo(void) testCond(s == NATS_OK); test("Start server2: "); - snprintf(cmdLine, sizeof(cmdLine), "-p 4223 -cluster_name \"local\" -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -c %s", conf); + snprintf(cmdLine, sizeof(cmdLine), "-a 127.0.0.1 -p 4223 -cluster_name \"local\" -cluster nats://127.0.0.1:6223 -routes nats://127.0.0.1:6222 -c %s", conf); pid2 = _startServer("nats://127.0.0.1:4223", cmdLine, true); CHECK_SERVER_STARTED(pid2); testCond(true); @@ -22075,8 +21814,7 @@ test_ReconnectImplicitUserInfo(void) remove(conf); } -static void -test_JetStreamUnmarshalAccountInfo(void) +void test_JetStreamUnmarshalAccountInfo(void) { natsStatus s; nats_JSON *json = NULL; @@ -22199,8 +21937,7 @@ test_JetStreamUnmarshalAccountInfo(void) jsAccountInfo_Destroy(ai); } -static void -test_JetStreamUnmarshalStreamState(void) +void test_JetStreamUnmarshalStreamState(void) { natsStatus s; nats_JSON *json = NULL; @@ -22277,8 +22014,7 @@ test_JetStreamUnmarshalStreamState(void) nats_JSONDestroy(json); } -static void -test_JetStreamUnmarshalStreamConfig(void) +void test_JetStreamUnmarshalStreamConfig(void) { natsStatus s; nats_JSON *json = NULL; @@ -22428,8 +22164,7 @@ test_JetStreamUnmarshalStreamConfig(void) json = NULL; } -static void -test_JetStreamUnmarshalStreamInfo(void) +void test_JetStreamUnmarshalStreamInfo(void) { natsStatus s; nats_JSON *json = NULL; @@ -22526,8 +22261,7 @@ test_JetStreamUnmarshalStreamInfo(void) } } -static void -test_JetStreamMarshalStreamConfig(void) +void test_JetStreamMarshalStreamConfig(void) { natsStatus s; jsStreamConfig sc; @@ -22779,8 +22513,7 @@ test_JetStreamMarshalStreamConfig(void) testCond((s == NATS_INVALID_ARG) && (buf == NULL)); } -static void -test_JetStreamUnmarshalConsumerInfo(void) +void test_JetStreamUnmarshalConsumerInfo(void) { natsStatus s; jsConsumerInfo *ci = NULL; @@ -23024,8 +22757,7 @@ natsConnection_Destroy(nc); \ _stopServer(pid); \ rmtree(datastore); -static void -test_JetStreamContext(void) +void test_JetStreamContext(void) { natsStatus s; natsConnection *nc = NULL; @@ -23201,8 +22933,7 @@ test_JetStreamContext(void) remove(confFile); } -static void -test_JetStreamContextDomain(void) +void test_JetStreamContextDomain(void) { natsStatus s; natsConnection *nc = NULL; @@ -23414,8 +23145,7 @@ _streamsNamesListReq(natsConnection *nc, natsMsg **msg, void *closure) } } -static void -test_JetStreamMgtStreams(void) +void test_JetStreamMgtStreams(void) { natsStatus s; jsCtx *js2= NULL; @@ -24189,8 +23919,7 @@ _consumerNamesListReq(natsConnection *nc, natsMsg **msg, void *closure) } } -static void -test_JetStreamMgtConsumers(void) +void test_JetStreamMgtConsumers(void) { natsStatus s; jsConsumerInfo *ci = NULL; @@ -25161,8 +24890,7 @@ test_JetStreamMgtConsumers(void) JS_TEARDOWN; } -static void -test_JetStreamPublish(void) +void test_JetStreamPublish(void) { natsStatus s; natsConnection *nc = NULL; @@ -25515,8 +25243,7 @@ _jsPubAckErrHandler(jsCtx *js, jsPubAckErr *pae, void *closure) natsMutex_Unlock(args->m); } -static void -test_JetStreamPublishAsync(void) +void test_JetStreamPublishAsync(void) { natsStatus s; natsSubscription *sub= NULL; @@ -25835,11 +25562,11 @@ test_JetStreamPublishAsync(void) _waitSubPending(rsub, 0); natsSub_Lock(rsub); - rsub->msgList.head = msg; - rsub->msgList.tail = msg; - rsub->msgList.msgs = 1; - rsub->msgList.bytes = natsMsg_dataAndHdrLen(msg); - natsCondition_Signal(rsub->cond); + rsub->ownDispatcher.queue.head = msg; + rsub->ownDispatcher.queue.tail = msg; + rsub->ownDispatcher.queue.msgs = 1; + rsub->ownDispatcher.queue.bytes = natsMsg_dataAndHdrLen(msg); + natsCondition_Signal(rsub->ownDispatcher.cond); natsSub_Unlock(rsub); // Message is owned by subscription, do not destroy it here. @@ -26091,8 +25818,7 @@ _checkPubAckResult(natsStatus s, struct threadArg *args) return s; } -static void -test_JetStreamPublishAckHandler(void) +void test_JetStreamPublishAckHandler(void) { natsStatus s; jsOptions o; @@ -26291,8 +26017,7 @@ _jsDrainErrCb(natsConnection *nc, natsSubscription *sub, natsStatus err, void *c natsMutex_Unlock(args->m); } -static void -test_JetStreamSubscribe(void) +void test_JetStreamSubscribe(void) { natsStatus s; natsOptions *ncOpts = NULL; @@ -26966,8 +26691,7 @@ test_JetStreamSubscribe(void) _destroyDefaultThreadArgs(&args); } -static void -test_JetStreamSubscribeSync(void) +void test_JetStreamSubscribeSync(void) { natsStatus s; natsSubscription *sub= NULL; @@ -27468,8 +27192,7 @@ test_JetStreamSubscribeSync(void) JS_TEARDOWN; } -static void -test_JetStreamSubscribeConfigCheck(void) +void test_JetStreamSubscribeConfigCheck(void) { natsStatus s; natsSubscription *sub= NULL; @@ -27869,8 +27592,7 @@ _setMsgReply(natsConnection *nc, natsMsg **msg, void* closure) natsConn_setFilter(nc, NULL); } -static void -test_JetStreamSubscribeIdleHearbeat(void) +void test_JetStreamSubscribeIdleHearbeat(void) { natsStatus s; natsConnection *nc = NULL; @@ -27944,9 +27666,9 @@ test_JetStreamSubscribeIdleHearbeat(void) test("Check HB received: "); nats_Sleep(300); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); s = (sub->jsi->mismatch.dseq == 1 ? NATS_OK : NATS_ERR); - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); testCond(s == NATS_OK); test("Check HB is not given to app: "); @@ -27986,10 +27708,10 @@ test_JetStreamSubscribeIdleHearbeat(void) // server state. #define PUBLISH_FAKE_JS_MSG_WITH_SEQ(_reply, _msg) \ { \ - natsSub_Lock(sub); \ + nats_lockSubAndDispatcher(sub); \ inbox = sub->subject; \ sub->jsi->ackNone = true; \ - natsSub_Unlock(sub); \ + nats_unlockSubAndDispatcher(sub); \ \ natsConn_setFilterWithClosure(nc, _setMsgReply, (void *)(_reply)); \ s = natsConnection_PublishString(nc, inbox, (_msg)); \ @@ -28037,9 +27759,9 @@ test_JetStreamSubscribeIdleHearbeat(void) // Send real message so that all clears up s = js_Publish(NULL, js, "foo", "msg3", 4, NULL, &jerr); nats_Sleep(300); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); s = (sub->jsi->ssmn == false ? NATS_OK : NATS_ERR); - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); testCond(s == NATS_OK); test("Skip again: "); @@ -28141,9 +27863,9 @@ test_JetStreamSubscribeIdleHearbeat(void) // Send real message so that all clears up s = js_Publish(NULL, js, "foo", "msg4", 4, NULL, &jerr); nats_Sleep(300); - natsSubAndLdw_Lock(sub); + nats_lockSubAndDispatcher(sub); s = (sub->jsi->ssmn == false && sub->jsi->sm == false ? NATS_OK : NATS_ERR); - natsSubAndLdw_Unlock(sub); + nats_unlockSubAndDispatcher(sub); testCond(s == NATS_OK); test("Skip again: "); @@ -28200,8 +27922,7 @@ test_JetStreamSubscribeIdleHearbeat(void) natsOptions_Destroy(opts); } -static void -test_JetStreamSubscribeFlowControl(void) +void test_JetStreamSubscribeFlowControl(void) { natsStatus s; natsSubscription *sub= NULL; @@ -28475,8 +28196,7 @@ _dropTimeoutProto(natsConnection *nc, natsMsg **msg, void* closure) *msg = NULL; } -static void -test_JetStreamSubscribePull(void) +void test_JetStreamSubscribePull(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29021,8 +28741,7 @@ test_JetStreamSubscribePull(void) _destroyDefaultThreadArgs(&args); } -static void -test_JetStreamSubscribeHeadersOnly(void) +void test_JetStreamSubscribeHeadersOnly(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29252,8 +28971,7 @@ _lastOnlyLoss(natsConnection *nc, natsMsg **msg, void* closure) } } -static void -test_JetStreamOrderedConsumer(void) +void test_JetStreamOrderedConsumer(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29485,14 +29203,16 @@ _jsOrderedErrHandler(natsConnection *nc, natsSubscription *subscription, natsSta { struct threadArg *args = (struct threadArg*) closure; + if (err != NATS_MISSED_HEARTBEAT) + return; + natsMutex_Lock(args->m); args->status = err; natsCondition_Signal(args->c); natsMutex_Unlock(args->m); } -static void -test_JetStreamOrderedConsumerWithErrors(void) +void test_JetStreamOrderedConsumerWithErrors(void) { natsStatus s; natsConnection *nc = NULL; @@ -29626,8 +29346,7 @@ _dropMsgFive(natsConnection *nc, natsMsg **msg, void* closure) } } -static void -test_JetStreamOrderedConsumerWithAutoUnsub(void) +void test_JetStreamOrderedConsumerWithAutoUnsub(void) { natsStatus s; natsConnection *nc2= NULL; @@ -29745,8 +29464,7 @@ test_JetStreamOrderedConsumerWithAutoUnsub(void) JS_TEARDOWN; } -static void -test_JetStreamOrderedConsSrvRestart(void) +void test_JetStreamOrderedConsSrvRestart(void) { natsStatus s; natsSubscription *sub = NULL; @@ -29824,6 +29542,7 @@ test_JetStreamOrderedConsSrvRestart(void) natsMutex_Lock(args.m); while ((s != NATS_TIMEOUT) && !args.reconnected) s = natsCondition_TimedWait(args.c, args.m, 2000); + natsMutex_Unlock(args.m); testCond(s == NATS_OK); test("Send 1 message: "); @@ -29861,8 +29580,7 @@ test_JetStreamOrderedConsSrvRestart(void) JS_TEARDOWN; } -static void -test_JetStreamSubscribeWithFWC(void) +void test_JetStreamSubscribeWithFWC(void) { natsStatus s; natsSubscription *sub= NULL; @@ -29899,8 +29617,7 @@ test_JetStreamSubscribeWithFWC(void) JS_TEARDOWN; } -static void -test_JetStreamStreamsSealAndRollup(void) +void test_JetStreamStreamsSealAndRollup(void) { natsStatus s; jsStreamInfo *si = NULL; @@ -30068,8 +29785,7 @@ test_JetStreamStreamsSealAndRollup(void) JS_TEARDOWN; } -static void -test_JetStreamGetMsgAndLastMsg(void) +void test_JetStreamGetMsgAndLastMsg(void) { natsStatus s; natsMsg *msg = NULL; @@ -30183,8 +29899,7 @@ test_JetStreamGetMsgAndLastMsg(void) JS_TEARDOWN; } -static void -test_JetStreamConvertDirectMsg(void) +void test_JetStreamConvertDirectMsg(void) { natsStatus s; natsMsg *msg = NULL; @@ -30295,8 +30010,7 @@ _checkDirectGet(jsCtx *js, uint64_t seq, const char *nextBySubj, const char *las return s; } -static void -test_JetStreamDirectGetMsg(void) +void test_JetStreamDirectGetMsg(void) { natsStatus s; natsMsg *msg = NULL; @@ -30391,8 +30105,7 @@ test_JetStreamDirectGetMsg(void) JS_TEARDOWN; } -static void -test_JetStreamNakWithDelay(void) +void test_JetStreamNakWithDelay(void) { natsStatus s; natsSubscription *sub= NULL; @@ -30469,8 +30182,7 @@ test_JetStreamNakWithDelay(void) JS_TEARDOWN; } -static void -test_JetStreamBackOffRedeliveries(void) +void test_JetStreamBackOffRedeliveries(void) { natsStatus s; natsSubscription *sub= NULL; @@ -30612,8 +30324,7 @@ _subjectsInfoReq(natsConnection *nc, natsMsg **msg, void *closure) } } -static void -test_JetStreamInfoWithSubjects(void) +void test_JetStreamInfoWithSubjects(void) { natsStatus s; jsStreamInfo *si = NULL; @@ -30743,45 +30454,7 @@ test_JetStreamInfoWithSubjects(void) JS_TEARDOWN; } -static natsStatus -_checkJSClusterReady(const char *url) -{ - natsStatus s = NATS_OK; - natsConnection *nc = NULL; - jsCtx *js = NULL; - jsErrCode jerr= 0; - int i; - jsOptions jo; - - jsOptions_Init(&jo); - jo.Wait = 1000; - - s = natsConnection_ConnectTo(&nc, url); - IFOK(s, natsConnection_JetStream(&js, nc, &jo)); - for (i=0; (s == NATS_OK) && (i<10); i++) - { - jsStreamInfo *si = NULL; - - s = js_GetStreamInfo(&si, js, "CHECK_CLUSTER", &jo, &jerr); - if (jerr == JSStreamNotFoundErr) - { - nats_clearLastError(); - s = NATS_OK; - break; - } - if ((s != NATS_OK) && (i < 9)) - { - s = NATS_OK; - nats_Sleep(500); - } - } - jsCtx_Destroy(js); - natsConnection_Destroy(nc); - return s; -} - -static void -test_JetStreamInfoAlternates(void) +void test_JetStreamInfoAlternates(void) { char datastore1[256] = {'\0'}; char datastore2[256] = {'\0'}; @@ -30796,6 +30469,7 @@ test_JetStreamInfoAlternates(void) jsStreamConfig sc; jsStreamSource ss; natsStatus s; + int i; ENSURE_JS_VERSION(2, 9, 0); @@ -30816,10 +30490,6 @@ test_JetStreamInfoAlternates(void) CHECK_SERVER_STARTED(pid1); testCond(true); - test("Check cluster: "); - s = _checkJSClusterReady("nats://127.0.0.1:4224"); - testCond(s == NATS_OK); - test("Connect: "); s = natsConnection_ConnectTo(&nc, NATS_DEFAULT_URL); testCond(s == NATS_OK); @@ -30833,7 +30503,18 @@ test_JetStreamInfoAlternates(void) sc.Name = "TEST"; sc.Subjects = (const char*[1]){"foo"}; sc.SubjectsLen = 1; - s = js_AddStream(NULL, js, &sc, NULL, NULL); + // We will try up to 10 times to create the stream. It may fail if the cluster + // is not ready to accept the request. + for (i=0; (s == NATS_OK) && (i < 10); i++) + { + s = js_AddStream(NULL, js, &sc, NULL, NULL); + if ((s != NATS_OK) && (i < 9)) + { + nats_clearLastError(); + s = NATS_OK; + nats_Sleep(100); + } + } testCond(s == NATS_OK); test("Create mirror: "); @@ -30874,8 +30555,7 @@ test_JetStreamInfoAlternates(void) rmtree(datastore3); } -static void -test_KeyValueManager(void) +void test_KeyValueManager(void) { natsStatus s; kvStore *kv = NULL; @@ -31005,8 +30685,7 @@ test_KeyValueManager(void) JS_TEARDOWN; } -static void -test_KeyValueBasics(void) +void test_KeyValueBasics(void) { natsStatus s; kvStore *kv = NULL; @@ -31412,8 +31091,7 @@ _stopWatcher(void *closure) kvWatcher_Stop(w); } -static void -test_KeyValueWatch(void) +void test_KeyValueWatch(void) { natsStatus s; kvStore *kv = NULL; @@ -31559,8 +31237,7 @@ test_KeyValueWatch(void) JS_TEARDOWN; } -static void -test_KeyValueWatchMulti(void) +void test_KeyValueWatchMulti(void) { natsStatus s; kvStore *kv = NULL; @@ -31607,8 +31284,7 @@ test_KeyValueWatchMulti(void) JS_TEARDOWN; } -static void -test_KeyValueHistory(void) +void test_KeyValueHistory(void) { natsStatus s; kvStore *kv = NULL; @@ -31692,8 +31368,7 @@ test_KeyValueHistory(void) JS_TEARDOWN; } -static void -test_KeyValueKeys(void) +void test_KeyValueKeys(void) { natsStatus s; kvStore *kv = NULL; @@ -31795,8 +31470,7 @@ test_KeyValueKeys(void) JS_TEARDOWN; } -static void -test_KeyValueDeleteVsPurge(void) +void test_KeyValueDeleteVsPurge(void) { natsStatus s; kvStore *kv = NULL; @@ -31866,8 +31540,7 @@ test_KeyValueDeleteVsPurge(void) JS_TEARDOWN; } -static void -test_KeyValueDeleteTombstones(void) +void test_KeyValueDeleteTombstones(void) { natsStatus s; kvStore *kv = NULL; @@ -31936,8 +31609,7 @@ test_KeyValueDeleteTombstones(void) JS_TEARDOWN; } -static void -test_KeyValuePurgeDeletesMarkerThreshold(void) +void test_KeyValuePurgeDeletesMarkerThreshold(void) { natsStatus s; kvStore *kv = NULL; @@ -31997,8 +31669,7 @@ test_KeyValuePurgeDeletesMarkerThreshold(void) JS_TEARDOWN; } -static void -test_KeyValueCrossAccount(void) +void test_KeyValueCrossAccount(void) { natsStatus s; natsOptions *opts= NULL; @@ -32269,8 +31940,7 @@ _checkDiscard(jsCtx *js, jsDiscardPolicy expected, kvStore **newKV) return s; } -static void -test_KeyValueDiscardOldToNew(void) +void test_KeyValueDiscardOldToNew(void) { kvStore *kv = NULL; kvConfig kvc; @@ -32328,8 +31998,7 @@ test_KeyValueDiscardOldToNew(void) JS_TEARDOWN; } -static void -test_KeyValueRePublish(void) +void test_KeyValueRePublish(void) { kvStore *kv = NULL; jsStreamInfo *si = NULL; @@ -32400,8 +32069,7 @@ test_KeyValueRePublish(void) JS_TEARDOWN; } -static void -test_KeyValueMirrorDirectGet(void) +void test_KeyValueMirrorDirectGet(void) { kvStore *kv = NULL; kvConfig kvc; @@ -32490,8 +32158,7 @@ _connectToHubAndCheckLeaf(natsConnection **hub, natsConnection *lnc) return s; } -static void -test_KeyValueMirrorCrossDomains(void) +void test_KeyValueMirrorCrossDomains(void) { natsStatus s; natsConnection *nc = NULL; @@ -32815,8 +32482,7 @@ test_KeyValueMirrorCrossDomains(void) remove(lconfFile); } -static void -test_MicroMatchEndpointSubject(void) +void test_MicroMatchEndpointSubject(void) { // endpoint, actual, match const char *test_cases[] = { @@ -32995,8 +32661,7 @@ typedef struct int expected_num_subjects; } add_service_test_case_t; -static void -test_MicroAddService(void) +void test_MicroAddService(void) { natsStatus s = NATS_OK; microError *err = NULL; @@ -33231,8 +32896,7 @@ test_MicroAddService(void) _stopServer(serverPid); } -static void -test_MicroGroups(void) +void test_MicroGroups(void) { natsStatus s = NATS_OK; microError *err = NULL; @@ -33341,8 +33005,7 @@ test_MicroGroups(void) #define NUM_MICRO_SERVICES 5 -static void -test_MicroBasics(void) +void test_MicroBasics(void) { natsStatus s = NATS_OK; microError *err = NULL; @@ -33622,8 +33285,7 @@ test_MicroBasics(void) _stopServer(serverPid); } -static void -test_MicroStartStop(void) +void test_MicroStartStop(void) { natsStatus s = NATS_OK; struct threadArg arg; @@ -33691,8 +33353,7 @@ test_MicroStartStop(void) _stopServer(serverPid); } -static void -test_MicroServiceStopsOnClosedConn(void) +void test_MicroServiceStopsOnClosedConn(void) { natsStatus s; natsConnection *nc = NULL; @@ -33764,8 +33425,7 @@ test_MicroServiceStopsOnClosedConn(void) _stopServer(serverPid); } -static void -test_MicroServiceStopsWhenServerStops(void) +void test_MicroServiceStopsWhenServerStops(void) { natsStatus s; natsConnection *nc = NULL; @@ -33857,8 +33517,7 @@ _microAsyncErrorRequestHandler(microRequest *req) return NULL; } -static void -test_MicroAsyncErrorHandler_MaxPendingMsgs(void) +void test_MicroAsyncErrorHandlerMaxPendingMsgs(void) { natsStatus s; struct threadArg arg; @@ -33921,8 +33580,8 @@ test_MicroAsyncErrorHandler_MaxPendingMsgs(void) natsMutex_Lock(arg.m); while ((s != NATS_TIMEOUT) && !arg.closed) s = natsCondition_TimedWait(arg.c, arg.m, 1000); - natsMutex_Unlock(arg.m); testCond((s == NATS_OK) && arg.closed && (arg.status == NATS_SLOW_CONSUMER)); + natsMutex_Unlock(arg.m); microService_Destroy(m); _waitForMicroservicesAllDone(&arg); @@ -33936,8 +33595,7 @@ test_MicroAsyncErrorHandler_MaxPendingMsgs(void) _stopServer(serverPid); } -static void -test_MicroAsyncErrorHandler_MaxPendingBytes(void) +void test_MicroAsyncErrorHandlerMaxPendingBytes(void) { natsStatus s; struct threadArg arg; @@ -34027,8 +33685,7 @@ _roundUp(int val) return ((val + (MEMALIGN-1))/MEMALIGN)*MEMALIGN; } -static void -test_StanPBufAllocator(void) +void test_StanPBufAllocator(void) { natsPBufAllocator *a = NULL; natsStatus s; @@ -34161,8 +33818,7 @@ _stanConnLostCB(stanConnection *sc, const char *errorTxt, void *closure) natsMutex_Unlock(arg->m); } -static void -test_StanConnOptions(void) +void test_StanConnOptions(void) { natsStatus s; stanConnOptions *opts = NULL; @@ -34329,8 +33985,7 @@ test_StanConnOptions(void) stanConnOptions_Destroy(clone); } -static void -test_StanSubOptions(void) +void test_StanSubOptions(void) { natsStatus s; stanSubOptions *opts = NULL; @@ -34452,8 +34107,7 @@ test_StanSubOptions(void) stanSubOptions_Destroy(clone); } -static void -test_StanMsg(void) +void test_StanMsg(void) { test("GetSequence with NULL msg: "); testCond(stanMsg_GetSequence(NULL) == 0); @@ -34473,8 +34127,7 @@ test_StanMsg(void) stanMsg_Destroy(NULL); } -static void -test_StanServerNotReachable(void) +void test_StanServerNotReachable(void) { natsStatus s; stanConnection *sc = NULL; @@ -34513,8 +34166,7 @@ test_StanServerNotReachable(void) _stopServer(serverPid); } -static void -test_StanBasicConnect(void) +void test_StanBasicConnect(void) { natsStatus s; stanConnection *sc = NULL; @@ -34581,8 +34233,7 @@ test_StanBasicConnect(void) _stopServer(pid); } -static void -test_StanConnectError(void) +void test_StanConnectError(void) { natsStatus s; stanConnection *sc = NULL; @@ -34618,8 +34269,7 @@ test_StanConnectError(void) } -static void -test_StanBasicPublish(void) +void test_StanBasicPublish(void) { natsStatus s; stanConnection *sc = NULL; @@ -34662,8 +34312,7 @@ _stanPubAckHandler(const char *guid, const char *errTxt, void* closure) natsMutex_Unlock(args->m); } -static void -test_StanBasicPublishAsync(void) +void test_StanBasicPublishAsync(void) { natsStatus s; stanConnection *sc = NULL; @@ -34701,8 +34350,7 @@ test_StanBasicPublishAsync(void) _stopServer(pid); } -static void -test_StanPublishTimeout(void) +void test_StanPublishTimeout(void) { natsStatus s; stanConnection *sc = NULL; @@ -34778,8 +34426,7 @@ _stanPublishSyncThread(void *closure) stanConnection_Publish(sc, "foo", (const void*)"hello", 5); } -static void -test_StanPublishMaxAcksInflight(void) +void test_StanPublishMaxAcksInflight(void) { natsStatus s; stanConnection *sc1 = NULL; @@ -34927,8 +34574,7 @@ _stanMsgHandlerBumpSum(stanConnection *sc, stanSubscription *sub, const char *ch stanMsg_Destroy(msg); } -static void -test_StanBasicSubscription(void) +void test_StanBasicSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -34971,8 +34617,7 @@ test_StanBasicSubscription(void) _stopServer(pid); } -static void -test_StanSubscriptionCloseAndUnsubscribe(void) +void test_StanSubscriptionCloseAndUnsubscribe(void) { natsStatus s; stanConnection *sc = NULL; @@ -35072,8 +34717,7 @@ test_StanSubscriptionCloseAndUnsubscribe(void) _stopServer(pid); } -static void -test_StanDurableSubscription(void) +void test_StanDurableSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -35160,8 +34804,7 @@ test_StanDurableSubscription(void) _stopServer(pid); } -static void -test_StanBasicQueueSubscription(void) +void test_StanBasicQueueSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -35224,8 +34867,7 @@ test_StanBasicQueueSubscription(void) _stopServer(pid); } -static void -test_StanDurableQueueSubscription(void) +void test_StanDurableQueueSubscription(void) { natsStatus s; stanConnection *sc = NULL; @@ -35335,8 +34977,7 @@ _stanCheckRecvStanMsg(stanConnection *sc, stanSubscription *sub, const char *cha natsMutex_Unlock(args->m); } -static void -test_StanCheckReceivedvMsg(void) +void test_StanCheckReceivedMsg(void) { natsStatus s; stanConnection *sc = NULL; @@ -35424,8 +35065,7 @@ _stanGetMsg(stanConnection *sc, stanSubscription *sub, const char *channel, natsMutex_Unlock(args->m); } -static void -test_StanSubscriptionAckMsg(void) +void test_StanSubscriptionAckMsg(void) { natsStatus s; stanConnection *sc = NULL; @@ -35545,8 +35185,7 @@ test_StanSubscriptionAckMsg(void) _stopServer(pid); } -static void -test_StanPings(void) +void test_StanPings(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35635,8 +35274,7 @@ test_StanPings(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanPingsNoResponder(void) +void test_StanPingsNoResponder(void) { natsStatus s; natsPid nPid = NATS_INVALID_PID; @@ -35691,8 +35329,7 @@ test_StanPingsNoResponder(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanConnectionLostHandlerNotSet(void) +void test_StanConnectionLostHandlerNotSet(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35753,8 +35390,7 @@ test_StanConnectionLostHandlerNotSet(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanPingsUnblockPubCalls(void) +void test_StanPingsUnblockPubCalls(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35836,8 +35472,7 @@ test_StanPingsUnblockPubCalls(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanGetNATSConnection(void) +void test_StanGetNATSConnection(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -35939,8 +35574,7 @@ test_StanGetNATSConnection(void) _stopServer(pid); } -static void -test_StanNoRetryOnFailedConnect(void) +void test_StanNoRetryOnFailedConnect(void) { natsStatus s; natsOptions *opts = NULL; @@ -35964,13 +35598,12 @@ _subDlvThreadPooled(natsSubscription *sub) { bool pooled; natsSub_Lock(sub); - pooled = (sub->libDlvWorker != NULL); + pooled = (sub->dispatcher->dedicatedTo == NULL); natsSub_Unlock(sub); return pooled; } -static void -test_StanInternalSubsNotPooled(void) +void test_StanInternalSubsNotPooled(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -36068,8 +35701,7 @@ _stanSubOnCompleteMsgCB(stanConnection *sc, stanSubscription *sub, const char *c stanMsg_Destroy(msg); } -static void -test_StanSubOnComplete(void) +void test_StanSubOnComplete(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -36148,8 +35780,7 @@ test_StanSubOnComplete(void) _destroyDefaultThreadArgs(&arg); } -static void -test_StanSubTimeout(void) +void test_StanSubTimeout(void) { natsStatus s; natsPid pid = NATS_INVALID_PID; @@ -36232,363 +35863,6 @@ test_StanSubTimeout(void) #endif -typedef void (*testFunc)(void); - -typedef struct __testInfo -{ - const char *name; - testFunc func; - -} testInfo; - -static testInfo allTests[] = -{ - // Building blocks - {"Version", test_Version}, - {"VersionMatchesTag", test_VersionMatchesTag}, - {"OpenCloseAndWait", test_OpenCloseAndWait}, - {"natsNowAndSleep", test_natsNowAndSleep}, - {"natsAllocSprintf", test_natsAllocSprintf}, - {"natsStrCaseStr", test_natsStrCaseStr}, - {"natsSnprintf", test_natsSnprintf}, - {"natsBuffer", test_natsBuffer}, - {"natsParseInt64", test_natsParseInt64}, - {"natsParseControl", test_natsParseControl}, - {"natsNormalizeErr", test_natsNormalizeErr}, - {"natsMutex", test_natsMutex}, - {"natsThread", test_natsThread}, - {"natsCondition", test_natsCondition}, - {"natsTimer", test_natsTimer}, - {"natsUrl", test_natsUrl}, - {"natsCreateStringFromBuffer", test_natsCreateStringFromBuffer}, - {"natsHash", test_natsHash}, - {"natsHashing", test_natsHashing}, - {"natsStrHash", test_natsStrHash}, - {"natsInbox", test_natsInbox}, - {"natsOptions", test_natsOptions}, - {"natsSock_ConnectTcp", test_natsSock_ConnectTcp}, - {"natsSock_ShuffleIPs", test_natsSock_ShuffleIPs}, - {"natsSock_IPOrder", test_natsSock_IPOrder}, - {"natsSock_ReadLine", test_natsSock_ReadLine}, - {"natsJSON", test_natsJSON}, - {"natsEncodeTimeUTC", test_natsEncodeTimeUTC}, - {"natsErrWithLongText", test_natsErrWithLongText}, - {"natsErrStackMoreThanMaxFrames", test_natsErrStackMoreThanMaxFrames}, - {"natsMsg", test_natsMsg}, - {"natsBase32", test_natsBase32Decode}, - {"natsBase64", test_natsBase64Encode}, - {"natsCRC16", test_natsCRC16}, - {"natsKeys", test_natsKeys}, - {"natsReadFile", test_natsReadFile}, - {"natsGetJWTOrSeed", test_natsGetJWTOrSeed}, - {"natsHostIsIP", test_natsHostIsIP}, - {"natsWaitReady", test_natsWaitReady}, - {"natsSign", test_natsSign}, - {"HeadersLift", test_natsMsgHeadersLift}, - {"HeadersAPIs", test_natsMsgHeaderAPIs}, - {"MsgIsJSControl", test_natsMsgIsJSCtrl}, - {"SrvVersionAtLeast", test_natsSrvVersionAtLeast}, - {"FormatStringArray", test_natsFormatStringArray}, - - // Package Level Tests - - {"ReconnectServerStats", test_ReconnectServerStats}, - {"ParseStateReconnectFunctionality",test_ParseStateReconnectFunctionality}, - {"ServersRandomize", test_ServersRandomize}, - {"SelectNextServer", test_SelectNextServer}, - {"ParserPing", test_ParserPing}, - {"ParserErr", test_ParserErr}, - {"ParserOK", test_ParserOK}, - {"ParseINFO", test_ParseINFO}, - {"ParserShouldFail", test_ParserShouldFail}, - {"ParserSplitMsg", test_ParserSplitMsg}, - {"ProcessMsgArgs", test_ProcessMsgArgs}, - {"LibMsgDelivery", test_LibMsgDelivery}, - {"AsyncINFO", test_AsyncINFO}, - {"RequestPool", test_RequestPool}, - {"NoFlusherIfSendAsapOption", test_NoFlusherIfSendAsap}, - {"HeadersAndSubPendingBytes", test_HeadersAndSubPendingBytes}, - - // Public API Tests - - {"DefaultConnection", test_DefaultConnection}, - {"SimplifiedURLs", test_SimplifiedURLs}, - {"IPResolutionOrder", test_IPResolutionOrder}, - {"UseDefaultURLIfNoServerSpecified",test_UseDefaultURLIfNoServerSpecified}, - {"ConnectToWithMultipleURLs", test_ConnectToWithMultipleURLs}, - {"ConnectionWithNULLOptions", test_ConnectionWithNullOptions}, - {"ConnectionToWithNullURLs", test_ConnectionToWithNullURLs}, - {"ConnectionStatus", test_ConnectionStatus}, - {"ConnClosedCB", test_ConnClosedCB}, - {"CloseDisconnectedCB", test_CloseDisconnectedCB}, - {"ServerStopDisconnectedCB", test_ServerStopDisconnectedCB}, - {"ClosedConnections", test_ClosedConnections}, - {"ConnectVerboseOption", test_ConnectVerboseOption}, - {"ReconnectThreadLeak", test_ReconnectThreadLeak}, - {"ReconnectTotalTime", test_ReconnectTotalTime}, - {"ReconnectDisallowedFlags", test_ReconnectDisallowedFlags}, - {"ReconnectAllowedFlags", test_ReconnectAllowedFlags}, - {"ConnCloseBreaksReconnectLoop", test_ConnCloseBreaksReconnectLoop}, - {"BasicReconnectFunctionality", test_BasicReconnectFunctionality}, - {"ExtendedReconnectFunctionality", test_ExtendedReconnectFunctionality}, - {"QueueSubsOnReconnect", test_QueueSubsOnReconnect}, - {"IsClosed", test_IsClosed}, - {"IsReconnectingAndStatus", test_IsReconnectingAndStatus}, - {"ReconnectBufSize", test_ReconnectBufSize}, - {"RetryOnFailedConnect", test_RetryOnFailedConnect}, - {"NoPartialOnReconnect", test_NoPartialOnReconnect}, - {"ReconnectFailsPendingRequests", test_ReconnectFailsPendingRequest}, - {"ForcedReconnect", test_ForcedReconnect}, - - {"ErrOnConnectAndDeadlock", test_ErrOnConnectAndDeadlock}, - {"ErrOnMaxPayloadLimit", test_ErrOnMaxPayloadLimit}, - - {"Auth", test_Auth}, - {"AuthFailNoDisconnectCB", test_AuthFailNoDisconnectCB}, - {"AuthToken", test_AuthToken}, - {"AuthTokenHandler", test_AuthTokenHandler}, - {"PermViolation", test_PermViolation}, - {"AuthViolation", test_AuthViolation}, - {"AuthenticationExpired", test_AuthenticationExpired}, - {"AuthenticationExpiredReconnect", test_AuthenticationExpiredReconnect}, - {"ConnectedServer", test_ConnectedServer}, - {"MultipleClose", test_MultipleClose}, - {"SimplePublish", test_SimplePublish}, - {"SimplePublishNoData", test_SimplePublishNoData}, - {"PublishMsg", test_PublishMsg}, - {"InvalidSubsArgs", test_InvalidSubsArgs}, - {"AsyncSubscribe", test_AsyncSubscribe}, - {"AsyncSubscribeTimeout", test_AsyncSubscribeTimeout}, - {"SyncSubscribe", test_SyncSubscribe}, - {"PubSubWithReply", test_PubSubWithReply}, - {"NoResponders", test_NoResponders}, - {"Flush", test_Flush}, - {"ConnCloseDoesFlush", test_ConnCloseDoesFlush}, - {"QueueSubscriber", test_QueueSubscriber}, - {"ReplyArg", test_ReplyArg}, - {"SyncReplyArg", test_SyncReplyArg}, - {"Unsubscribe", test_Unsubscribe}, - {"DoubleUnsubscribe", test_DoubleUnsubscribe}, - {"SubRemovedWhileProcessingMsg", test_SubRemovedWhileProcessingMsg}, - {"RequestTimeout", test_RequestTimeout}, - {"Request", test_Request}, - {"RequestNoBody", test_RequestNoBody}, - {"RequestMuxWithMappedSubject", test_RequestMuxWithMappedSubject}, - {"OldRequest", test_OldRequest}, - {"SimultaneousRequests", test_SimultaneousRequest}, - {"RequestClose", test_RequestClose}, - {"CustomInbox", test_CustomInbox}, - {"MessagePadding", test_MessageBufferPadding}, - {"FlushInCb", test_FlushInCb}, - {"ReleaseFlush", test_ReleaseFlush}, - {"FlushErrOnDisconnect", test_FlushErrOnDisconnect}, - {"Inbox", test_Inbox}, - {"Stats", test_Stats}, - {"BadSubject", test_BadSubject}, - {"SubBadSubjectAndQueueNames", test_SubBadSubjectAndQueueName}, - {"ClientAsyncAutoUnsub", test_ClientAsyncAutoUnsub}, - {"ClientSyncAutoUnsub", test_ClientSyncAutoUnsub}, - {"ClientAutoUnsubAndReconnect", test_ClientAutoUnsubAndReconnect}, - {"AutoUnsubNoUnsubOnDestroy", test_AutoUnsubNoUnsubOnDestroy}, - {"NextMsgOnClosedSub", test_NextMsgOnClosedSub}, - {"CloseSubRelease", test_CloseSubRelease}, - {"IsValidSubscriber", test_IsValidSubscriber}, - {"SlowSubscriber", test_SlowSubscriber}, - {"SlowAsyncSubscriber", test_SlowAsyncSubscriber}, - {"SlowConsumerCb", test_SlowConsumerCB}, - {"PendingLimitsDeliveredAndDropped",test_PendingLimitsDeliveredAndDropped}, - {"PendingLimitsWithSyncSub", test_PendingLimitsWithSyncSub}, - {"AsyncSubscriptionPending", test_AsyncSubscriptionPending}, - {"AsyncSubscriptionPendingDrain", test_AsyncSubscriptionPendingDrain}, - {"SyncSubscriptionPending", test_SyncSubscriptionPending}, - {"SyncSubscriptionPendingDrain", test_SyncSubscriptionPendingDrain}, - {"AsyncErrHandlerMaxPendingMsgs", test_AsyncErrHandler_MaxPendingMsgs}, - {"AsyncErrHandlerMaxPendingBytes", test_AsyncErrHandler_MaxPendingBytes }, - {"AsyncErrHandlerSubDestroyed", test_AsyncErrHandlerSubDestroyed}, - {"AsyncSubscriberStarvation", test_AsyncSubscriberStarvation}, - {"AsyncSubscriberOnClose", test_AsyncSubscriberOnClose}, - {"NextMsgCallOnAsyncSub", test_NextMsgCallOnAsyncSub}, - {"SubOnComplete", test_SubOnComplete}, - {"GetLastError", test_GetLastError}, - {"StaleConnection", test_StaleConnection}, - {"ServerErrorClosesConnection", test_ServerErrorClosesConnection}, - {"NoEcho", test_NoEcho}, - {"NoEchoOldServer", test_NoEchoOldServer}, - {"DrainSub", test_DrainSub}, - {"DrainSubStops", test_DrainSubStops}, - {"DrainSubRaceOnAutoUnsub", test_DrainSubRaceOnAutoUnsub}, - {"DrainSubNotResentOnReconnect", test_DrainSubNotResentOnReconnect}, - {"DrainConn", test_DrainConn}, - {"NoDoubleCloseCbOnDrain", test_NoDoubleConnClosedOnDrain}, - {"GetClientID", test_GetClientID}, - {"GetClientIP", test_GetClientIP}, - {"GetRTT", test_GetRTT}, - {"GetLocalIPAndPort", test_GetLocalIPAndPort}, - {"UserCredsCallbacks", test_UserCredsCallbacks}, - {"UserCredsFromFiles", test_UserCredsFromFiles}, - {"UserCredsFromMemory", test_UserCredsFromMemory}, - {"NKey", test_NKey}, - {"NKeyFromSeed", test_NKeyFromSeed}, - {"ConnSign", test_ConnSign}, - {"WriteDeadline", test_WriteDeadline}, - {"HeadersNotSupported", test_HeadersNotSupported}, - {"HeadersBasic", test_HeadersBasic}, - {"MsgsFilter", test_natsMsgsFilter}, - {"EventLoop", test_EventLoop}, - {"EventLoopRetryOnFailedConnect", test_EventLoopRetryOnFailedConnect}, - {"EventLoopTLS", test_EventLoopTLS}, - {"SSLBasic", test_SSLBasic}, - {"SSLVerify", test_SSLVerify}, - {"SSLCAFromMemory", test_SSLLoadCAFromMemory}, - {"SSLCertAndKeyFromMemory", test_SSLCertAndKeyFromMemory}, - {"SSLVerifyHostname", test_SSLVerifyHostname}, - {"SSLSkipServerVerification", test_SSLSkipServerVerification}, - {"SSLCiphers", test_SSLCiphers}, - {"SSLMultithreads", test_SSLMultithreads}, - {"SSLConnectVerboseOption", test_SSLConnectVerboseOption}, - {"SSLSocketLeakEventLoop", test_SSLSocketLeakWithEventLoop}, - {"SSLReconnectWithAuthError", test_SSLReconnectWithAuthError}, - {"SSLAvailable", test_SSLAvailable}, - - // Clusters Tests - - {"ServersOption", test_ServersOption}, - {"AuthServers", test_AuthServers}, - {"AuthFailToReconnect", test_AuthFailToReconnect}, - {"ReconnectWithTokenHandler", test_ReconnectWithTokenHandler}, - {"BasicClusterReconnect", test_BasicClusterReconnect}, - {"HotSpotReconnect", test_HotSpotReconnect}, - {"ProperReconnectDelay", test_ProperReconnectDelay}, - {"ProperFalloutAfterMaxAttempts", test_ProperFalloutAfterMaxAttempts}, - {"StopReconnectAfterTwoAuthErr", test_StopReconnectAfterTwoAuthErr}, - {"TimeoutOnNoServer", test_TimeoutOnNoServer}, - {"PingReconnect", test_PingReconnect}, - {"GetServers", test_GetServers}, - {"GetDiscoveredServers", test_GetDiscoveredServers}, - {"DiscoveredServersCb", test_DiscoveredServersCb}, - {"IgnoreDiscoveredServers", test_IgnoreDiscoveredServers}, - {"INFOAfterFirstPONGisProcessedOK", test_ReceiveINFORightAfterFirstPONG}, - {"ServerPoolUpdatedOnClusterUpdate",test_ServerPoolUpdatedOnClusterUpdate}, - {"ReconnectJitter", test_ReconnectJitter}, - {"CustomReconnectDelay", test_CustomReconnectDelay}, - {"LameDuckMode", test_LameDuckMode}, - {"ReconnectImplicitUserInfo", test_ReconnectImplicitUserInfo}, - - {"JetStreamUnmarshalAccInfo", test_JetStreamUnmarshalAccountInfo}, - {"JetStreamUnmarshalStreamState", test_JetStreamUnmarshalStreamState}, - {"JetStreamUnmarshalStreamCfg", test_JetStreamUnmarshalStreamConfig}, - {"JetStreamUnmarshalStreamInfo", test_JetStreamUnmarshalStreamInfo}, - {"JetStreamMarshalStreamCfg", test_JetStreamMarshalStreamConfig}, - {"JetStreamUnmarshalConsumerInfo", test_JetStreamUnmarshalConsumerInfo}, - {"JetStreamContext", test_JetStreamContext}, - {"JetStreamDomain", test_JetStreamContextDomain}, - {"JetStreamMgtStreams", test_JetStreamMgtStreams}, - {"JetStreamMgtConsumers", test_JetStreamMgtConsumers}, - {"JetStreamPublish", test_JetStreamPublish}, - {"JetStreamPublishAsync", test_JetStreamPublishAsync}, - {"JetStreamPublishAckHandler", test_JetStreamPublishAckHandler}, - {"JetStreamSubscribe", test_JetStreamSubscribe}, - {"JetStreamSubscribeSync", test_JetStreamSubscribeSync}, - {"JetStreamSubscribeConfigCheck", test_JetStreamSubscribeConfigCheck}, - {"JetStreamSubscribeIdleHeartbeat", test_JetStreamSubscribeIdleHearbeat}, - {"JetStreamSubscribeFlowControl", test_JetStreamSubscribeFlowControl}, - {"JetStreamSubscribePull", test_JetStreamSubscribePull}, - {"JetStreamSubscribeHeadersOnly", test_JetStreamSubscribeHeadersOnly}, - {"JetStreamOrderedCons", test_JetStreamOrderedConsumer}, - {"JetStreamOrderedConsWithErrors", test_JetStreamOrderedConsumerWithErrors}, - {"JetStreamOrderedConsAutoUnsub", test_JetStreamOrderedConsumerWithAutoUnsub}, - {"JetStreamOrderedConsSrvRestart", test_JetStreamOrderedConsSrvRestart}, - {"JetStreamSubscribeWithFWC", test_JetStreamSubscribeWithFWC}, - {"JetStreamStreamsSealAndRollup", test_JetStreamStreamsSealAndRollup}, - {"JetStreamGetMsgAndLastMsg", test_JetStreamGetMsgAndLastMsg}, - {"JetStreamConvertDirectMsg", test_JetStreamConvertDirectMsg}, - {"JetStreamDirectGetMsg", test_JetStreamDirectGetMsg}, - {"JetStreamNakWithDelay", test_JetStreamNakWithDelay}, - {"JetStreamBackOffRedeliveries", test_JetStreamBackOffRedeliveries}, - {"JetStreamInfoWithSubjects", test_JetStreamInfoWithSubjects}, - {"JetStreamInfoAlternates", test_JetStreamInfoAlternates}, - - {"KeyValueManager", test_KeyValueManager}, - {"KeyValueBasics", test_KeyValueBasics}, - {"KeyValueWatch", test_KeyValueWatch}, - {"KeyValueWatchMulti", test_KeyValueWatchMulti}, - {"KeyValueHistory", test_KeyValueHistory}, - {"KeyValueKeys", test_KeyValueKeys}, - {"KeyValueDeleteVsPurge", test_KeyValueDeleteVsPurge}, - {"KeyValueDeleteTombstones", test_KeyValueDeleteTombstones}, - {"KeyValueDeleteMarkerThreshold", test_KeyValuePurgeDeletesMarkerThreshold}, - {"KeyValueCrossAccount", test_KeyValueCrossAccount}, - {"KeyValueDiscardOldToNew", test_KeyValueDiscardOldToNew}, - {"KeyValueRePublish", test_KeyValueRePublish}, - {"KeyValueMirrorDirectGet", test_KeyValueMirrorDirectGet}, - {"KeyValueMirrorCrossDomains", test_KeyValueMirrorCrossDomains}, - - {"MicroMatchEndpointSubject", test_MicroMatchEndpointSubject}, - {"MicroAddService", test_MicroAddService}, - {"MicroGroups", test_MicroGroups}, - {"MicroBasics", test_MicroBasics}, - {"MicroStartStop", test_MicroStartStop}, - {"MicroServiceStopsOnClosedConn", test_MicroServiceStopsOnClosedConn}, - {"MicroServiceStopsWhenServerStops", test_MicroServiceStopsWhenServerStops}, - {"MicroAsyncErrorHandlerMaxPendingMsgs", test_MicroAsyncErrorHandler_MaxPendingMsgs}, - {"MicroAsyncErrorHandlerMaxPendingBytes", test_MicroAsyncErrorHandler_MaxPendingBytes }, - -#if defined(NATS_HAS_STREAMING) - {"StanPBufAllocator", test_StanPBufAllocator}, - {"StanConnOptions", test_StanConnOptions}, - {"StanSubOptions", test_StanSubOptions}, - {"StanMsg", test_StanMsg}, - {"StanServerNotReachable", test_StanServerNotReachable}, - {"StanBasicConnect", test_StanBasicConnect}, - {"StanConnectError", test_StanConnectError}, - {"StanBasicPublish", test_StanBasicPublish}, - {"StanBasicPublishAsync", test_StanBasicPublishAsync}, - {"StanPublishTimeout", test_StanPublishTimeout}, - {"StanPublishMaxAcksInflight", test_StanPublishMaxAcksInflight}, - {"StanBasicSubscription", test_StanBasicSubscription}, - {"StanSubscriptionCloseAndUnsub", test_StanSubscriptionCloseAndUnsubscribe}, - {"StanDurableSubscription", test_StanDurableSubscription}, - {"StanBasicQueueSubscription", test_StanBasicQueueSubscription}, - {"StanDurableQueueSubscription", test_StanDurableQueueSubscription}, - {"StanCheckReceivedMsg", test_StanCheckReceivedvMsg}, - {"StanSubscriptionAckMsg", test_StanSubscriptionAckMsg}, - {"StanPings", test_StanPings}, - {"StanPingsNoResponder", test_StanPingsNoResponder}, - {"StanConnectionLostHandlerNotSet", test_StanConnectionLostHandlerNotSet}, - {"StanPingsUnblockPublishCalls", test_StanPingsUnblockPubCalls}, - {"StanGetNATSConnection", test_StanGetNATSConnection}, - {"StanNoRetryOnFailedConnect", test_StanNoRetryOnFailedConnect}, - {"StanInternalSubsNotPooled", test_StanInternalSubsNotPooled}, - {"StanSubOnComplete", test_StanSubOnComplete}, - {"StanSubTimeout", test_StanSubTimeout}, - -#endif - -}; - -static int maxTests = (int) (sizeof(allTests)/sizeof(testInfo)); - -static void -generateList(void) -{ - FILE *list = fopen("list.txt", "w"); - int i; - - if (list == NULL) - { - printf("@@ Unable to create file 'list.txt': %d\n", errno); - return; - } - - printf("Number of tests: %d\n", maxTests); - - for (i=0; i= maxTests) - || (testEnd < 0) || (testEnd >= maxTests) - || (testStart > testEnd)) - { - printf("@@ Usage: %s [start] [end] (0 .. %d)\n", argv[0], (maxTests - 1)); + printf("@@ Usage: %s [testname]\n", argv[0]); return 1; } + testName = argv[1]; #ifndef _WIN32 signal(SIGSEGV, _sigsegv_handler); @@ -36691,14 +35950,25 @@ int main(int argc, char **argv) } // Execute tests - for (i=testStart; (i<=testEnd) && !failed; i++) + testFunc f = NULL; + for (i = 0; i < (int)(sizeof(allTests) / sizeof(*allTests)); i++) { + if (strcmp(testName, allTests[i].name) != 0) + continue; + #ifdef _WIN32 printf("\n== %s ==\n", allTests[i].name); #else printf("\033[0;34m\n== %s ==\n\033[0;0m", allTests[i].name); #endif - (*(allTests[i].func))(); + f = allTests[i].func; + f(); + break; + } + if (f == NULL) + { + printf("@@ Test '%s' not found!\n", testName); + return 1; } #ifdef _WIN32 diff --git a/test/test.h b/test/test.h new file mode 100644 index 000000000..6a9bc8edf --- /dev/null +++ b/test/test.h @@ -0,0 +1,360 @@ +// Copyright 2024 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "natsp.h" +#include "comsock.h" + +#if defined(NATS_HAS_STREAMING) +static const char *clusterName = "test-cluster"; +#endif + +#ifdef _WIN32 +#define NATS_INVALID_PID (NULL) +#define LOGFILE_NAME "wserver.log" +#else +#define NATS_INVALID_PID (-1) +#define LOGFILE_NAME "server.log" +#endif + +#define FAIL(m) \ + { \ + printf("@@ %s @@\n", (m)); \ + failed = true; \ + return; \ + } + +#define CHECK_SERVER_STARTED(p) \ + if ((p) == NATS_INVALID_PID) \ + FAIL("Unable to start or verify that the server was started!") + +extern natsMutex *slMu; +extern natsHash *slMap; +extern bool keepServerOutput; +extern bool failed; + +static const char *natsServerExe = "nats-server"; + +static natsStatus +_checkStreamingStart(const char *url, int maxAttempts) +{ + natsStatus s = NATS_NOT_PERMITTED; + +#if defined(NATS_HAS_STREAMING) + + stanConnOptions *opts = NULL; + stanConnection *sc = NULL; + int attempts = 0; + + s = stanConnOptions_Create(&opts); + IFOK(s, stanConnOptions_SetURL(opts, url)); + IFOK(s, stanConnOptions_SetConnectionWait(opts, 250)); + if (s == NATS_OK) + { + while (((s = stanConnection_Connect(&sc, clusterName, "checkStart", opts)) != NATS_OK) && (attempts++ < maxAttempts)) + { + nats_Sleep(200); + } + } + + stanConnection_Destroy(sc); + stanConnOptions_Destroy(opts); + + if (s != NATS_OK) + nats_clearLastError(); +#else +#endif + return s; +} + +static natsStatus +_checkStart(const char *url, int orderIP, int maxAttempts) +{ + natsStatus s = NATS_OK; + natsUrl *nUrl = NULL; + int attempts = 0; + natsSockCtx ctx; + + natsSock_Init(&ctx); + ctx.orderIP = orderIP; + + natsDeadline_Init(&(ctx.writeDeadline), 2000); + + s = natsUrl_Create(&nUrl, url); + if (s == NATS_OK) + { + while (((s = natsSock_ConnectTcp(&ctx, + nUrl->host, nUrl->port)) != NATS_OK) && + (attempts++ < maxAttempts)) + { + nats_Sleep(200); + } + + natsUrl_Destroy(nUrl); + + if (s == NATS_OK) + natsSock_Close(ctx.fd); + else + s = NATS_NO_SERVER; + } + + nats_clearLastError(); + + return s; +} + +#ifdef _WIN32 + +typedef PROCESS_INFORMATION *natsPid; + +static HANDLE logHandle = NULL; + +static void +_stopServer(natsPid pid) +{ + if (pid == NATS_INVALID_PID) + return; + + TerminateProcess(pid->hProcess, 0); + WaitForSingleObject(pid->hProcess, INFINITE); + + CloseHandle(pid->hProcess); + CloseHandle(pid->hThread); + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Remove(slMap, (int64_t)pid); + natsMutex_Unlock(slMu); + + free(pid); +} + +static natsPid +_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) +{ + SECURITY_ATTRIBUTES sa; + STARTUPINFO si; + HANDLE h; + PROCESS_INFORMATION *pid; + DWORD flags = 0; + BOOL hInheritance = FALSE; + char *exeAndCmdLine = NULL; + int ret; + + pid = calloc(1, sizeof(PROCESS_INFORMATION)); + if (pid == NULL) + return NATS_INVALID_PID; + + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + + ret = nats_asprintf(&exeAndCmdLine, "%s%s%s", serverExe, + (cmdLineOpts != NULL ? " " : ""), + (cmdLineOpts != NULL ? cmdLineOpts : "")); + if (ret < 0) + { + printf("No memory allocating command line string!\n"); + free(pid); + return NATS_INVALID_PID; + } + + if (!keepServerOutput) + { + ZeroMemory(&sa, sizeof(sa)); + sa.nLength = sizeof(sa); + sa.lpSecurityDescriptor = NULL; + sa.bInheritHandle = TRUE; + + h = logHandle; + if (h == NULL) + { + h = CreateFile(LOGFILE_NAME, + GENERIC_WRITE, + FILE_SHARE_WRITE | FILE_SHARE_READ, + &sa, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + } + + si.dwFlags |= STARTF_USESTDHANDLES; + si.hStdInput = NULL; + si.hStdError = h; + si.hStdOutput = h; + + hInheritance = TRUE; + flags = CREATE_NO_WINDOW; + + if (logHandle == NULL) + logHandle = h; + } + + // Start the child process. + if (!CreateProcess(NULL, + (LPSTR)exeAndCmdLine, + NULL, // Process handle not inheritable + NULL, // Thread handle not inheritable + hInheritance, // Set handle inheritance + flags, // Creation flags + NULL, // Use parent's environment block + NULL, // Use parent's starting directory + &si, // Pointer to STARTUPINFO structure + pid)) // Pointer to PROCESS_INFORMATION structure + { + + printf("Unable to start '%s': error (%d).\n", + exeAndCmdLine, GetLastError()); + free(exeAndCmdLine); + return NATS_INVALID_PID; + } + + free(exeAndCmdLine); + + if (checkStart) + { + natsStatus s; + + if (strcmp(serverExe, natsServerExe) == 0) + s = _checkStart(url, 46, 10); + else + s = _checkStreamingStart(url, 10); + + if (s != NATS_OK) + { + _stopServer(pid); + return NATS_INVALID_PID; + } + } + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Set(slMap, (int64_t)pid, NULL, NULL); + natsMutex_Unlock(slMu); + + return (natsPid)pid; +} + +#else + +typedef pid_t natsPid; + +static void +_stopServer(natsPid pid) +{ + int status = 0; + + if (pid == NATS_INVALID_PID) + return; + + if (kill(pid, SIGINT) < 0) + { + perror("kill with SIGINT"); + if (kill(pid, SIGKILL) < 0) + { + perror("kill with SIGKILL"); + } + } + + waitpid(pid, &status, 0); + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Remove(slMap, (int64_t)pid); + natsMutex_Unlock(slMu); +} + +static natsPid +_startServerImpl(const char *serverExe, const char *url, const char *cmdLineOpts, bool checkStart) +{ + natsPid pid = fork(); + if (pid == -1) + { + perror("fork"); + return NATS_INVALID_PID; + } + + if (pid == 0) + { + char *exeAndCmdLine = NULL; + char *argvPtrs[64]; + char *line = NULL; + int index = 0; + int ret = 0; + bool overrideAddr = false; + + if ((cmdLineOpts == NULL) || (strstr(cmdLineOpts, "-a ") == NULL)) + overrideAddr = true; + + ret = nats_asprintf(&exeAndCmdLine, "%s%s%s%s%s", serverExe, + (cmdLineOpts != NULL ? " " : ""), + (cmdLineOpts != NULL ? cmdLineOpts : ""), + (overrideAddr ? " -a 127.0.0.1" : ""), + (keepServerOutput ? "" : " -l " LOGFILE_NAME)); + if (ret < 0) + { + perror("No memory allocating command line string!\n"); + exit(1); + } + + memset(argvPtrs, 0, sizeof(argvPtrs)); + line = exeAndCmdLine; + + while (*line != '\0') + { + while ((*line == ' ') || (*line == '\t') || (*line == '\n')) + *line++ = '\0'; + + argvPtrs[index++] = line; + while ((*line != '\0') && (*line != ' ') && (*line != '\t') && (*line != '\n')) + { + line++; + } + } + argvPtrs[index++] = NULL; + + // Child process. Replace with NATS server + execvp(argvPtrs[0], argvPtrs); + perror("Exec failed: "); + exit(1); + } + else if (checkStart) + { + natsStatus s; + + if (strcmp(serverExe, natsServerExe) == 0) + s = _checkStart(url, 46, 10); + else + s = _checkStreamingStart(url, 10); + + if (s != NATS_OK) + { + _stopServer(pid); + return NATS_INVALID_PID; + } + } + + natsMutex_Lock(slMu); + if (slMap != NULL) + natsHash_Set(slMap, (int64_t)pid, NULL, NULL); + natsMutex_Unlock(slMu); + + // parent, return the child's PID back. + return pid; +} +#endif + +static natsPid +_startServer(const char *url, const char *cmdLineOpts, bool checkStart) +{ + return _startServerImpl(natsServerExe, url, cmdLineOpts, checkStart); +} + diff --git a/test/tlsfirst.conf b/test/tlsfirst.conf new file mode 100644 index 000000000..5ed521c17 --- /dev/null +++ b/test/tlsfirst.conf @@ -0,0 +1,16 @@ + +# Simple TLS config file + +port: 4443 +net: "0.0.0.0" + +tls { + # Server cert + cert_file: "certs/server-cert.pem" + # Server private key + key_file: "certs/server-key.pem" + # Increase timeout for valgrind tests + timeout: 2 + # Force client to do the handshake first + handshake_first: true +}