core: Link reducer-server against missing dynamic libraries; Upgrade …

…to mongocxx 3.10.2; Add test for shared library build (fixes y-scope#526). (y-scope#527)

Co-authored-by: Lin Zhihao <[email protected]>

.github/actions/clp-core-build/action.yaml

@@ -8,6 +8,9 @@ inputs:
   use_published_image:
     description: "Whether to use the published container image"
     required: true
+  use_shared_libs:
+    description: "Whether to build the core binaries by linking against shared libraries"
+    required: true
   upload_binaries:
     description: "Whether to upload the core binaries"
     required: true
@@ -52,9 +55,11 @@ runs:
         --workdir /mnt/clp
         ${{steps.get_image_props.outputs.qualified_image_name}}
         bash -c "task deps:core &&
-        /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.sh
-        /mnt/clp/components/core
-        /mnt/clp/components/core/build"
+        python3 /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.py
+        ${{inputs.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+        --source-dir /mnt/clp/components/core
+        --build-dir /mnt/clp/components/core/build
+        --num-jobs $(getconf _NPROCESSORS_ONLN)"
       shell: "bash"
 
     - if: "inputs.upload_binaries == 'true'"

.github/workflows/clp-core-build-macos.yaml

@@ -10,7 +10,7 @@ on:
       - "components/core/tests/**"
       - "components/core/tools/scripts/lib_install/macos-12/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
@@ -23,7 +23,7 @@ on:
       - "components/core/tests/**"
       - "components/core/tools/scripts/lib_install/macos-12/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
@@ -36,6 +36,9 @@ concurrency:
 
 jobs:
   build-macos:
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
     runs-on: "macos-12"
     steps:
       - uses: "actions/checkout@v4"
@@ -63,4 +66,10 @@ jobs:
         shell: "bash"
         working-directory: "./components/core"
         # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow
-        run: "./tools/scripts/utils/build-and-run-unit-tests.sh . build ~[Stopwatch]"
+        run: >-
+          python3 ./tools/scripts/utils/build-and-run-unit-tests.py
+          ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+          --source-dir .
+          --build-dir build
+          --num-jobs $(getconf _NPROCESSORS_ONLN)
+          --test-spec "~[Stopwatch]"

.github/workflows/clp-core-build.yaml

@@ -89,7 +89,7 @@ jobs:
               - "components/core/CMakeLists.txt"
               - "components/core/src/**"
               - "components/core/tests/**"
-              - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+              - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
               - "deps-tasks.yml"
               - "Taskfile.yml"
               - "tools/scripts/deps-download/**"
@@ -177,6 +177,11 @@ jobs:
     needs:
       - "centos74-deps-image"
       - "filter-relevant-changes"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "centos74-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -192,6 +197,7 @@ jobs:
           OS_NAME: "centos7.4"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.centos74_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
@@ -205,6 +211,15 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-focal-deps-image"
+    strategy:
+      matrix:
+        include:
+          - use_shared_libs: true
+            upload_binaries: false
+          - use_shared_libs: false
+            upload_binaries: true
+    name: "ubuntu-focal-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -220,10 +235,11 @@ jobs:
           OS_NAME: "ubuntu-focal"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
-          upload_binaries: "true"
+          upload_binaries: "${{matrix.upload_binaries}}"
           binaries_artifact_name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
 
   ubuntu-jammy-binaries:
@@ -234,6 +250,11 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-jammy-deps-image"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "ubuntu-jammy-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -249,6 +270,7 @@ jobs:
           OS_NAME: "ubuntu-jammy"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}

components/core/CMakeLists.txt

@@ -198,6 +198,8 @@ add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL)
 # Add simdjson
 add_subdirectory(submodules/simdjson EXCLUDE_FROM_ALL)
 
+find_package(Threads REQUIRED)
+
 # Add yaml-cpp
 add_subdirectory(submodules/yaml-cpp EXCLUDE_FROM_ALL)
 

components/core/src/reducer/CMakeLists.txt

@@ -44,11 +44,16 @@ target_include_directories(reducer-server PRIVATE "${PROJECT_SOURCE_DIR}/submodu
 target_link_libraries(reducer-server
         PRIVATE
         Boost::program_options
+        Boost::system
         clp::string_utils
         fmt::fmt
         ${MONGOCXX_TARGET}
         msgpack-cxx
         spdlog::spdlog
+        # Threads::Threads is necessary because on Ubuntu Focal (20.04), libssl (a dependency of
+        # libmongocxx) depends on libpthread, but its cmake script doesn't seem to provide the flags
+        # to link against libpthread.
+        Threads::Threads
 )
 # Put the built executable at the root of the build directory
 set_target_properties(

components/core/tools/scripts/lib_install/centos7.4/install-packages-from-source.sh

@@ -28,8 +28,7 @@ lib_install_scripts_dir=$script_dir/..
 "$lib_install_scripts_dir"/libarchive.sh 3.5.1
 "$lib_install_scripts_dir"/lz4.sh 1.8.2
 "$lib_install_scripts_dir"/mariadb-connector-c.sh 3.2.3
-"$lib_install_scripts_dir"/mongoc.sh 1.24.4
-"$lib_install_scripts_dir"/mongocxx.sh 3.8.0
+"$lib_install_scripts_dir"/mongocxx.sh 3.10.2
 "$lib_install_scripts_dir"/msgpack.sh 6.0.0
 "$lib_install_scripts_dir"/spdlog.sh 1.9.2
 "$lib_install_scripts_dir"/zstandard.sh 1.4.9

components/core/tools/scripts/lib_install/mongocxx.sh

@@ -67,12 +67,9 @@ fi
 
 # Set up
 cd "${extracted_dir}/build"
-# NOTE: Although the mongocxx docs indicate we should use
-# '-DMONGOCXX_OVERRIDE_DEFAULT_INSTALL_PREFIX=OFF' to install to the default location (/usr/local),
-# this doesn't seem to work, so we specify CMAKE_INSTALL_PREFIX here
 cmake \
   -DCMAKE_BUILD_TYPE=Release \
-  -DCMAKE_INSTALL_PREFIX=/usr/local \
+  -DMONGOCXX_OVERRIDE_DEFAULT_INSTALL_PREFIX=OFF \
   -DBUILD_SHARED_AND_STATIC_LIBS=ON \
   -DBUILD_SHARED_LIBS_WITH_STATIC_MONGOC=ON \
   -DENABLE_TESTS=OFF \

components/core/tools/scripts/lib_install/ubuntu-focal/install-packages-from-source.sh

@@ -15,8 +15,7 @@ lib_install_scripts_dir=$script_dir/..
 "$lib_install_scripts_dir"/fmtlib.sh 8.0.1
 "$lib_install_scripts_dir"/libarchive.sh 3.5.1
 "$lib_install_scripts_dir"/lz4.sh 1.8.2
-"$lib_install_scripts_dir"/mongoc.sh 1.24.4
-"$lib_install_scripts_dir"/mongocxx.sh 3.8.0
+"$lib_install_scripts_dir"/mongocxx.sh 3.10.2
 "$lib_install_scripts_dir"/msgpack.sh 6.0.0
 "$lib_install_scripts_dir"/spdlog.sh 1.9.2
 "$lib_install_scripts_dir"/zstandard.sh 1.4.9

components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh

@@ -12,8 +12,7 @@ lib_install_scripts_dir=$script_dir/..
 "$lib_install_scripts_dir"/fmtlib.sh 8.0.1
 "$lib_install_scripts_dir"/libarchive.sh 3.5.1
 "$lib_install_scripts_dir"/lz4.sh 1.8.2
-"$lib_install_scripts_dir"/mongoc.sh 1.24.4
-"$lib_install_scripts_dir"/mongocxx.sh 3.8.0
+"$lib_install_scripts_dir"/mongocxx.sh 3.10.2
 "$lib_install_scripts_dir"/msgpack.sh 6.0.0
 "$lib_install_scripts_dir"/spdlog.sh 1.9.2
 "$lib_install_scripts_dir"/zstandard.sh 1.4.9