From 6f6a09f980e86277ce63f69b58ff63fadb3a613b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 00:30:04 -0500 Subject: [PATCH 01/27] Add installation for apple clang 16 and pass it as env vars to build configure --- .github/workflows/clp-core-build-macos.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 8196e75d8..b18e315ff 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -61,6 +61,11 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" + - name: "Install coreutils (for md5sum) and Apple Clang 16 (for C++20)" + run: |- + brew install coreutils + brew install llvm@16 + - run: "./tools/scripts/deps-download/init.sh" shell: "bash" @@ -68,6 +73,20 @@ jobs: shell: "bash" - name: "Build CLP-core and run unit tests" + env: >- + ${{ + 'macos-13' == matrix.runner + && fromJson('{ + "CC": "/usr/local/opt/llvm@16/bin/clang", + "CXX": "/usr/local/opt/llvm@16/bin/clang++" + }') + || 'macos-14' == matrix.runner + && fromJson('{ + "CC": "/opt/homebrew/opt/llvm@16/bin/clang", + "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" + }') + || fromJson('{}') + }} shell: "bash" working-directory: "./components/core" # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow From 7ee9eb3cde5e2de52214e92c728b2f56a858e850 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 02:39:11 -0500 Subject: [PATCH 02/27] Update archive reader --- .github/workflows/clp-core-build-macos.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index b18e315ff..ff6e8913b 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -78,12 +78,14 @@ jobs: 'macos-13' == matrix.runner && fromJson('{ "CC": "/usr/local/opt/llvm@16/bin/clang", - "CXX": "/usr/local/opt/llvm@16/bin/clang++" + "CXX": "/usr/local/opt/llvm@16/bin/clang++", + "CMAKE_AR": "/usr/bin/ar" }') || 'macos-14' == matrix.runner && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", - "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" + "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++", + "CMAKE_AR": "/usr/bin/ar" }') || fromJson('{}') }} From 282a719b72263d449862857ea0684fcc28c36bc0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 14:58:29 -0500 Subject: [PATCH 03/27] print ar info --- .github/workflows/clp-core-build-macos.yaml | 27 ++++++++++++++----- .../scripts/lib_install/macos/install-all.sh | 1 + 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index ff6e8913b..e655990b3 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -61,31 +61,44 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" - - name: "Install coreutils (for md5sum) and Apple Clang 16 (for C++20)" - run: |- - brew install coreutils - brew install llvm@16 - - run: "./tools/scripts/deps-download/init.sh" shell: "bash" - run: "task deps:core" shell: "bash" + - name: "Check AR version" + run: |- + echo "-----------------" + which ar + echo "-----------------" + command -v ar + echo "-----------------" + which -a ar + echo "-----------------" + ar --version + echo "-----------------" + cmake --system-information | grep CMAKE_AR + echo "-----------------" + ls -l $(brew --prefix)/bin/ar + echo "-----------------" + $(brew --prefix)/bin/ar --version + echo "-----------------" + - name: "Build CLP-core and run unit tests" env: >- ${{ 'macos-13' == matrix.runner && fromJson('{ "CC": "/usr/local/opt/llvm@16/bin/clang", - "CXX": "/usr/local/opt/llvm@16/bin/clang++", "CMAKE_AR": "/usr/bin/ar" + "CXX": "/usr/local/opt/llvm@16/bin/clang++", }') || 'macos-14' == matrix.runner && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", - "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++", "CMAKE_AR": "/usr/bin/ar" + "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++", }') || fromJson('{}') }} diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index cb24dd054..b7f16d1f3 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -16,6 +16,7 @@ brew install \ go-task \ java11 \ libarchive \ + llvm@16 \ lz4 \ mariadb-connector-c \ mongo-cxx-driver \ From 4a303c022a5f5e6eed09a0a9c219632a2d4df3e3 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 15:07:40 -0500 Subject: [PATCH 04/27] remove extra line --- .github/workflows/clp-core-build-macos.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index e655990b3..97ca6faba 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -78,8 +78,6 @@ jobs: echo "-----------------" ar --version echo "-----------------" - cmake --system-information | grep CMAKE_AR - echo "-----------------" ls -l $(brew --prefix)/bin/ar echo "-----------------" $(brew --prefix)/bin/ar --version From 8fd147af6566a254fce737e0017275e122d79700 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 15:16:35 -0500 Subject: [PATCH 05/27] remove extra line --- .github/workflows/clp-core-build-macos.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 97ca6faba..a8883db40 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -69,19 +69,8 @@ jobs: - name: "Check AR version" run: |- - echo "-----------------" which ar - echo "-----------------" command -v ar - echo "-----------------" - which -a ar - echo "-----------------" - ar --version - echo "-----------------" - ls -l $(brew --prefix)/bin/ar - echo "-----------------" - $(brew --prefix)/bin/ar --version - echo "-----------------" - name: "Build CLP-core and run unit tests" env: >- From f0df5da483088f71c58695c34687a6d6c7693b28 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 15:26:57 -0500 Subject: [PATCH 06/27] remove extra line --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index a8883db40..5cc500568 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -68,7 +68,7 @@ jobs: shell: "bash" - name: "Check AR version" - run: |- + run: | which ar command -v ar From edc39c26de51f7cf5ebeb4cadc2a8bbbb1464775 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 15:41:51 -0500 Subject: [PATCH 07/27] fix syntax --- .github/workflows/clp-core-build-macos.yaml | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 5cc500568..1389f1f92 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -68,9 +68,22 @@ jobs: shell: "bash" - name: "Check AR version" - run: | + run: |- + echo "-----------------" which ar + echo "-----------------" command -v ar + echo "-----------------" + which -a ar + echo "-----------------" + ar --version + echo "-----------------" + cmake --system-information | grep CMAKE_AR + echo "-----------------" + ls -l $(brew --prefix)/bin/ar + echo "-----------------" + $(brew --prefix)/bin/ar --version + echo "-----------------" - name: "Build CLP-core and run unit tests" env: >- @@ -78,14 +91,14 @@ jobs: 'macos-13' == matrix.runner && fromJson('{ "CC": "/usr/local/opt/llvm@16/bin/clang", - "CMAKE_AR": "/usr/bin/ar" - "CXX": "/usr/local/opt/llvm@16/bin/clang++", + "CMAKE_AR": "/usr/bin/ar", + "CXX": "/usr/local/opt/llvm@16/bin/clang++" }') || 'macos-14' == matrix.runner && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", - "CMAKE_AR": "/usr/bin/ar" - "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++", + "CMAKE_AR": "/usr/bin/ar", + "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" }') || fromJson('{}') }} From 8290d83c74dd6f52ad5f5abd64ba0aeb15aeee31 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 15:50:19 -0500 Subject: [PATCH 08/27] Change cmd --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 1389f1f92..4f570fbab 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -76,7 +76,7 @@ jobs: echo "-----------------" which -a ar echo "-----------------" - ar --version + strings $(which ar) | grep -i cctools echo "-----------------" cmake --system-information | grep CMAKE_AR echo "-----------------" From 79d8ad95c46a12a5482ed37b9f9dcd08976e4232 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 15:53:40 -0500 Subject: [PATCH 09/27] Change cmd --- .github/workflows/clp-core-build-macos.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 4f570fbab..6d2836008 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -76,10 +76,6 @@ jobs: echo "-----------------" which -a ar echo "-----------------" - strings $(which ar) | grep -i cctools - echo "-----------------" - cmake --system-information | grep CMAKE_AR - echo "-----------------" ls -l $(brew --prefix)/bin/ar echo "-----------------" $(brew --prefix)/bin/ar --version From f959846670eaff25188a10ed25a08791032d3b2a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 16:21:27 -0500 Subject: [PATCH 10/27] Change cmd --- .github/workflows/clp-core-build-macos.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 6d2836008..6fb2a1ac1 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -70,15 +70,9 @@ jobs: - name: "Check AR version" run: |- echo "-----------------" - which ar + ls /opt/homebrew/opt/llvm@16/bin/llvm-ar echo "-----------------" - command -v ar - echo "-----------------" - which -a ar - echo "-----------------" - ls -l $(brew --prefix)/bin/ar - echo "-----------------" - $(brew --prefix)/bin/ar --version + ls -l /usr/bin/ar echo "-----------------" - name: "Build CLP-core and run unit tests" From d423104d0efca619fcb11eb21652161c5ccb8e58 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 16:35:03 -0500 Subject: [PATCH 11/27] Change cmd --- .github/workflows/clp-core-build-macos.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 6fb2a1ac1..c1702c31d 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -70,7 +70,9 @@ jobs: - name: "Check AR version" run: |- echo "-----------------" - ls /opt/homebrew/opt/llvm@16/bin/llvm-ar + ls /opt/homebrew/opt/llvm@16/bin/llvm-ar || true + echo "-----------------" + ls /usr/local/opt/llvm@16/bin/llvm-ar || true echo "-----------------" ls -l /usr/bin/ar echo "-----------------" @@ -81,13 +83,13 @@ jobs: 'macos-13' == matrix.runner && fromJson('{ "CC": "/usr/local/opt/llvm@16/bin/clang", - "CMAKE_AR": "/usr/bin/ar", + "CMAKE_AR": "/usr/local/opt/llvm@16/bin/llvm-ar", "CXX": "/usr/local/opt/llvm@16/bin/clang++" }') || 'macos-14' == matrix.runner && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", - "CMAKE_AR": "/usr/bin/ar", + "CMAKE_AR": "/opt/homebrew/opt/llvm@16/bin/llvm-ar", "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" }') || fromJson('{}') From a7e3a7d8978681eed00e314fdfb3589ef15b6ade Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 16:54:01 -0500 Subject: [PATCH 12/27] Change cmd --- .github/workflows/clp-core-build-macos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index c1702c31d..a4d94f21e 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,8 +71,10 @@ jobs: run: |- echo "-----------------" ls /opt/homebrew/opt/llvm@16/bin/llvm-ar || true + ls /opt/homebrew/opt/llvm@16/bin/llvm-ranlib || true echo "-----------------" ls /usr/local/opt/llvm@16/bin/llvm-ar || true + ls /usr/local/opt/llvm@16/bin/llvm-ranlib || true echo "-----------------" ls -l /usr/bin/ar echo "-----------------" @@ -84,12 +86,14 @@ jobs: && fromJson('{ "CC": "/usr/local/opt/llvm@16/bin/clang", "CMAKE_AR": "/usr/local/opt/llvm@16/bin/llvm-ar", + "CMAKE_RANLIB": "/usr/local/opt/llvm@16/bin/llvm-ranlib", "CXX": "/usr/local/opt/llvm@16/bin/clang++" }') || 'macos-14' == matrix.runner && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", "CMAKE_AR": "/opt/homebrew/opt/llvm@16/bin/llvm-ar", + "CMAKE_RANLIB": "/opt/homebrew/opt/llvm@16/bin/llvm-ranlib", "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" }') || fromJson('{}') From ed280abfab60135fbef3b69444526b698c2dc320 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 17:33:53 -0500 Subject: [PATCH 13/27] Change cmd for macos14 --- .github/workflows/clp-core-build-macos.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index a4d94f21e..3bd931e95 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -77,6 +77,7 @@ jobs: ls /usr/local/opt/llvm@16/bin/llvm-ranlib || true echo "-----------------" ls -l /usr/bin/ar + ls -l /usr/bin/ranlib echo "-----------------" - name: "Build CLP-core and run unit tests" @@ -92,8 +93,8 @@ jobs: || 'macos-14' == matrix.runner && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", - "CMAKE_AR": "/opt/homebrew/opt/llvm@16/bin/llvm-ar", - "CMAKE_RANLIB": "/opt/homebrew/opt/llvm@16/bin/llvm-ranlib", + "CMAKE_AR": "/usr/bin/ar", + "CMAKE_RANLIB": "usr/bin/ranlib", "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" }') || fromJson('{}') From 335f4be1fc975a7ab1abcdf187451e881d65ca82 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 5 Mar 2025 18:01:58 -0500 Subject: [PATCH 14/27] Always used shared --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 3bd931e95..ec0c035fe 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -42,7 +42,7 @@ jobs: strategy: matrix: runner: ["macos-13", "macos-14"] - use_shared_libs: [true, false] + use_shared_libs: [true] runs-on: "${{matrix.runner}}" steps: - uses: "actions/checkout@v4" From 718f50b46c98b5e425cbb9f62a4638642dbc47aa Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 03:24:41 -0500 Subject: [PATCH 15/27] Add debugging --- .github/workflows/clp-core-build-macos.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index ec0c035fe..2b598fbd2 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -42,7 +42,7 @@ jobs: strategy: matrix: runner: ["macos-13", "macos-14"] - use_shared_libs: [true] + use_shared_libs: [true, false] runs-on: "${{matrix.runner}}" steps: - uses: "actions/checkout@v4" @@ -72,11 +72,14 @@ jobs: echo "-----------------" ls /opt/homebrew/opt/llvm@16/bin/llvm-ar || true ls /opt/homebrew/opt/llvm@16/bin/llvm-ranlib || true + ls /opt/homebrew/opt/llvm@16/bin/llvm-ar -V || true echo "-----------------" ls /usr/local/opt/llvm@16/bin/llvm-ar || true ls /usr/local/opt/llvm@16/bin/llvm-ranlib || true + ls /usr/local/opt/llvm@16/bin/llvm-ar -V || true echo "-----------------" ls -l /usr/bin/ar + /usr/bin/ar -V || true ls -l /usr/bin/ranlib echo "-----------------" From eafdd3988efcbc9b9a55630d3093c6d3ee5b88d3 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 03:52:27 -0500 Subject: [PATCH 16/27] print debug info --- components/core/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index d8b0352a4..a995cbca3 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -25,6 +25,9 @@ else() message(SEND_ERROR "GENERAL_COMPRESSOR=${GENERAL_COMPRESSOR} is unimplemented.") endif() +message(STATUS "CMAKE_AR: ${CMAKE_AR}") +message(STATUS "CMAKE_RANLIB: ${CMAKE_RANLIB}") + # Add local CMake module directory to CMake's modules path set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") From 1a30e1ab3ced948e0a9a5d3c167434c53fac6050 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 04:42:03 -0500 Subject: [PATCH 17/27] Explicitly override AR and RANLIB via cmake configure --- .github/workflows/clp-core-build-macos.yaml | 2 ++ .../scripts/utils/build-and-run-unit-tests.py | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 2b598fbd2..593d1b112 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -112,3 +112,5 @@ jobs: --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" + ${{ env.CMAKE_AR != '' && format('--archive-tool {0}', env.CMAKE_AR) || '' }} + ${{ env.CMAKE_RANLIB != '' && format('--ranlib-tool {0}', env.CMAKE_RANLIB) || '' }} diff --git a/components/core/tools/scripts/utils/build-and-run-unit-tests.py b/components/core/tools/scripts/utils/build-and-run-unit-tests.py index 7c4b13617..db0ec496f 100644 --- a/components/core/tools/scripts/utils/build-and-run-unit-tests.py +++ b/components/core/tools/scripts/utils/build-and-run-unit-tests.py @@ -21,7 +21,13 @@ logger = logging.getLogger(__name__) -def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): +def _config_cmake_project( + src_dir: Path, + build_dir: Path, + use_shared_libs: bool, + ar: Optional[Path], + ranlib: Optional[Path], +): cmd = [ "cmake", "-S", @@ -31,6 +37,10 @@ def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool) ] if use_shared_libs: cmd.append("-DCLP_USE_STATIC_LIBS=OFF") + if ar: + cmd.append(f"-DCMAKE_AR={ar}") + if ranlib: + cmd.append(f"-DCMAKE_RANLIB={ranlib}") subprocess.run(cmd, check=True) @@ -82,6 +92,8 @@ def main(argv: List[str]) -> int: "--num-jobs", type=int, help="Max number of jobs to run when building." ) args_parser.add_argument("--test-spec", help="Catch2 test specification.") + args_parser.add_argument("--ar", help="Archiver tool to use.") + args_parser.add_argument("--ranlib", help="Ranlib indexing tool to use.") parsed_args = args_parser.parse_args(argv[1:]) src_dir: Path = Path(parsed_args.source_dir) @@ -89,8 +101,10 @@ def main(argv: List[str]) -> int: use_shared_libs: bool = parsed_args.use_shared_libs num_jobs: Optional[int] = parsed_args.num_jobs test_spec: Optional[str] = parsed_args.test_spec + ar: Optional[Path] = parsed_args.ar + ranlib: Optional[Path] = parsed_args.ranlib - _config_cmake_project(src_dir, build_dir, use_shared_libs) + _config_cmake_project(src_dir, build_dir, use_shared_libs, ar, ranlib) _build_project(build_dir, num_jobs) _run_unit_tests(build_dir, test_spec) From 2acfb7a625dd6d230321ce2744ad113ee939c551 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 04:52:44 -0500 Subject: [PATCH 18/27] fix --- .github/workflows/clp-core-build-macos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 593d1b112..60ec29321 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -112,5 +112,5 @@ jobs: --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" - ${{ env.CMAKE_AR != '' && format('--archive-tool {0}', env.CMAKE_AR) || '' }} - ${{ env.CMAKE_RANLIB != '' && format('--ranlib-tool {0}', env.CMAKE_RANLIB) || '' }} + ${{ env.CMAKE_AR != '' && format('--ar {0}', env.CMAKE_AR) || '' }} + ${{ env.CMAKE_RANLIB != '' && format('--ranlib {0}', env.CMAKE_RANLIB) || '' }} From 7644a0ba2b25e0221d4b62e5abe372d7be8bfa41 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 05:02:41 -0500 Subject: [PATCH 19/27] fix --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 60ec29321..6ae7b59f2 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -97,7 +97,7 @@ jobs: && fromJson('{ "CC": "/opt/homebrew/opt/llvm@16/bin/clang", "CMAKE_AR": "/usr/bin/ar", - "CMAKE_RANLIB": "usr/bin/ranlib", + "CMAKE_RANLIB": "/usr/bin/ranlib", "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" }') || fromJson('{}') From dd7012b82d3351a975ee58b13d1ee0c1c5d829ac Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 07:40:03 -0500 Subject: [PATCH 20/27] Refactor --- .github/workflows/clp-core-build-macos.yaml | 47 +++++++------------ .../scripts/lib_install/macos/install-all.sh | 1 - 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 6ae7b59f2..1ababb698 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -41,9 +41,9 @@ jobs: build-macos: strategy: matrix: - runner: ["macos-13", "macos-14"] + os: ["macos-13", "macos-14"] use_shared_libs: [true, false] - runs-on: "${{matrix.runner}}" + runs-on: "${{matrix.os}}" steps: - uses: "actions/checkout@v4" with: @@ -67,38 +67,23 @@ jobs: - run: "task deps:core" shell: "bash" - - name: "Check AR version" - run: |- - echo "-----------------" - ls /opt/homebrew/opt/llvm@16/bin/llvm-ar || true - ls /opt/homebrew/opt/llvm@16/bin/llvm-ranlib || true - ls /opt/homebrew/opt/llvm@16/bin/llvm-ar -V || true - echo "-----------------" - ls /usr/local/opt/llvm@16/bin/llvm-ar || true - ls /usr/local/opt/llvm@16/bin/llvm-ranlib || true - ls /usr/local/opt/llvm@16/bin/llvm-ar -V || true - echo "-----------------" - ls -l /usr/bin/ar - /usr/bin/ar -V || true - ls -l /usr/bin/ranlib - echo "-----------------" + - if: "'macos-13' == matrix.os || 'macos-14' == matrix.os" + name: "Upgrade the default AppleClang to LLVM Clang for C++20 features" + shell: "bash" + run: | + brew install llvm@16 + echo "USE_LLVM_COMPILER=true" >> $GITHUB_ENV + echo "LLVM_INSTALL_BINDIR=$(brew --prefix llvm@16)" >> $GITHUB_ENV - name: "Build CLP-core and run unit tests" env: >- ${{ - 'macos-13' == matrix.runner - && fromJson('{ - "CC": "/usr/local/opt/llvm@16/bin/clang", - "CMAKE_AR": "/usr/local/opt/llvm@16/bin/llvm-ar", - "CMAKE_RANLIB": "/usr/local/opt/llvm@16/bin/llvm-ranlib", - "CXX": "/usr/local/opt/llvm@16/bin/clang++" - }') - || 'macos-14' == matrix.runner + env.USE_LLVM_COMPILER && fromJson('{ - "CC": "/opt/homebrew/opt/llvm@16/bin/clang", - "CMAKE_AR": "/usr/bin/ar", - "CMAKE_RANLIB": "/usr/bin/ranlib", - "CXX": "/opt/homebrew/opt/llvm@16/bin/clang++" + "CC": "${{ env.LLVM_INSTALL_BINDIR }}/clang", + "CMAKE_AR": "${{ env.LLVM_INSTALL_BINDIR }}/llvm-ar", + "CMAKE_RANLIB": "${{ env.LLVM_INSTALL_BINDIR }}/llvm-ranlib", + "CXX": "${{ env.LLVM_INSTALL_BINDIR }}/clang++" }') || fromJson('{}') }} @@ -112,5 +97,5 @@ jobs: --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" - ${{ env.CMAKE_AR != '' && format('--ar {0}', env.CMAKE_AR) || '' }} - ${{ env.CMAKE_RANLIB != '' && format('--ranlib {0}', env.CMAKE_RANLIB) || '' }} + ${{ env.USE_LLVM_COMPILER + && format('--ar {0} --ranlib {1}', env.CMAKE_AR, env.CMAKE_RANLIB) || '' }} diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index b7f16d1f3..cb24dd054 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -16,7 +16,6 @@ brew install \ go-task \ java11 \ libarchive \ - llvm@16 \ lz4 \ mariadb-connector-c \ mongo-cxx-driver \ From edf4d09bb56e2234474d78d6670a733b1e841baa Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 08:09:18 -0500 Subject: [PATCH 21/27] Test env var --- .github/workflows/clp-core-build-macos.yaml | 35 ++++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 1ababb698..0745893ac 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -49,6 +49,33 @@ jobs: with: submodules: "recursive" + - if: "'macos-13' == matrix.os || 'macos-14' == matrix.os" + name: "Upgrade the default AppleClang to LLVM Clang for C++20 features" + shell: "bash" + run: | + brew install llvm@16 + echo "USE_LLVM_COMPILER=true" >> $GITHUB_ENV + echo "LLVM_INSTALL_BINDIR=$(brew --prefix llvm@16)" >> $GITHUB_ENV + + - name: "Test env var" + env: >- + ${{ + env.USE_LLVM_COMPILER + && fromJson('{ + "CC": "$LLVM_INSTALL_BINDIR/clang", + "CMAKE_AR": "$LLVM_INSTALL_BINDIR/llvm-ar", + "CMAKE_RANLIB": "$LLVM_INSTALL_BINDIR/llvm-ranlib", + "CXX": "$LLVM_INSTALL_BINDIR/clang++" + }') + || fromJson('{}') + }} + shell: "bash" + run: | + echo CC=$CC + echo CXX=$CXX + echo AR=$CMAKE_AR + echo RANLIB=$CMAKE_RANLIB + # See https://github.com/actions/setup-python/issues/577 - name: "Remove preinstalled binaries which conflict with brew's installs" run: | @@ -67,14 +94,6 @@ jobs: - run: "task deps:core" shell: "bash" - - if: "'macos-13' == matrix.os || 'macos-14' == matrix.os" - name: "Upgrade the default AppleClang to LLVM Clang for C++20 features" - shell: "bash" - run: | - brew install llvm@16 - echo "USE_LLVM_COMPILER=true" >> $GITHUB_ENV - echo "LLVM_INSTALL_BINDIR=$(brew --prefix llvm@16)" >> $GITHUB_ENV - - name: "Build CLP-core and run unit tests" env: >- ${{ From f2e50cab7df99e6651ae51d1c1e141f37321095d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 08:46:54 -0500 Subject: [PATCH 22/27] Test --- .github/workflows/clp-core-build-macos.yaml | 52 +++++++-------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 0745893ac..c87e22673 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -57,24 +57,31 @@ jobs: echo "USE_LLVM_COMPILER=true" >> $GITHUB_ENV echo "LLVM_INSTALL_BINDIR=$(brew --prefix llvm@16)" >> $GITHUB_ENV - - name: "Test env var" + - name: "Build CLP-core and run unit tests" env: >- ${{ env.USE_LLVM_COMPILER && fromJson('{ - "CC": "$LLVM_INSTALL_BINDIR/clang", - "CMAKE_AR": "$LLVM_INSTALL_BINDIR/llvm-ar", - "CMAKE_RANLIB": "$LLVM_INSTALL_BINDIR/llvm-ranlib", - "CXX": "$LLVM_INSTALL_BINDIR/clang++" + "CC": "${{ env.LLVM_INSTALL_BINDIR }}/clang", + "CXX": "${{ env.LLVM_INSTALL_BINDIR }}/clang++" }') || fromJson('{}') }} shell: "bash" - run: | - echo CC=$CC - echo CXX=$CXX - echo AR=$CMAKE_AR - echo RANLIB=$CMAKE_RANLIB + working-directory: "./components/core" + # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow + run: >- + python3 ./tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}} + --source-dir . + --build-dir build + --num-jobs $(getconf _NPROCESSORS_ONLN) + --test-spec "~[Stopwatch]" + ${{ env.USE_LLVM_COMPILER == 'true' + && format('--ar {0} --ranlib {1}', + 'env.LLVM_INSTALL_BINDIR/llvm-ar', + 'env.LLVM_INSTALL_BINDIR/llvm-ranlib') + || ''}} # See https://github.com/actions/setup-python/issues/577 - name: "Remove preinstalled binaries which conflict with brew's installs" @@ -93,28 +100,3 @@ jobs: - run: "task deps:core" shell: "bash" - - - name: "Build CLP-core and run unit tests" - env: >- - ${{ - env.USE_LLVM_COMPILER - && fromJson('{ - "CC": "${{ env.LLVM_INSTALL_BINDIR }}/clang", - "CMAKE_AR": "${{ env.LLVM_INSTALL_BINDIR }}/llvm-ar", - "CMAKE_RANLIB": "${{ env.LLVM_INSTALL_BINDIR }}/llvm-ranlib", - "CXX": "${{ env.LLVM_INSTALL_BINDIR }}/clang++" - }') - || fromJson('{}') - }} - shell: "bash" - working-directory: "./components/core" - # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow - run: >- - python3 ./tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}} - --source-dir . - --build-dir build - --num-jobs $(getconf _NPROCESSORS_ONLN) - --test-spec "~[Stopwatch]" - ${{ env.USE_LLVM_COMPILER - && format('--ar {0} --ranlib {1}', env.CMAKE_AR, env.CMAKE_RANLIB) || '' }} From 9cffee1649ec5c7a43c32e2d4b3a45376ba1be8a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 08:51:19 -0500 Subject: [PATCH 23/27] quotation fix --- .github/workflows/clp-core-build-macos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index c87e22673..69eaa5bc8 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,8 +79,8 @@ jobs: --test-spec "~[Stopwatch]" ${{ env.USE_LLVM_COMPILER == 'true' && format('--ar {0} --ranlib {1}', - 'env.LLVM_INSTALL_BINDIR/llvm-ar', - 'env.LLVM_INSTALL_BINDIR/llvm-ranlib') + env.LLVM_INSTALL_BINDIR/llvm-ar, + env.LLVM_INSTALL_BINDIR/llvm-ranlib) || ''}} # See https://github.com/actions/setup-python/issues/577 From 625e48cacad7146a5f934f8ca23d3540dea95bb0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 08:57:38 -0500 Subject: [PATCH 24/27] fix --- .github/workflows/clp-core-build-macos.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 69eaa5bc8..04f1c41b5 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -77,11 +77,9 @@ jobs: --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" - ${{ env.USE_LLVM_COMPILER == 'true' - && format('--ar {0} --ranlib {1}', - env.LLVM_INSTALL_BINDIR/llvm-ar, - env.LLVM_INSTALL_BINDIR/llvm-ranlib) - || ''}} + $( [[ "${{ env.USE_LLVM_COMPILER }}" == "true" ]] + && echo "--ar ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ar" + "--ranlib ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ranlib" ) # See https://github.com/actions/setup-python/issues/577 - name: "Remove preinstalled binaries which conflict with brew's installs" From d9c3bd17f7dbf7e6bcc8148fe8108fee8b2296ed Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 09:06:05 -0500 Subject: [PATCH 25/27] fix --- .github/workflows/clp-core-build-macos.yaml | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 04f1c41b5..d58efd6c1 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -55,31 +55,32 @@ jobs: run: | brew install llvm@16 echo "USE_LLVM_COMPILER=true" >> $GITHUB_ENV - echo "LLVM_INSTALL_BINDIR=$(brew --prefix llvm@16)" >> $GITHUB_ENV + echo "LLVM_INSTALL_BINDIR=$(brew --prefix llvm@16)/bin" >> $GITHUB_ENV - name: "Build CLP-core and run unit tests" - env: >- - ${{ - env.USE_LLVM_COMPILER - && fromJson('{ - "CC": "${{ env.LLVM_INSTALL_BINDIR }}/clang", - "CXX": "${{ env.LLVM_INSTALL_BINDIR }}/clang++" - }') - || fromJson('{}') - }} shell: "bash" working-directory: "./components/core" # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow run: >- + $( + [[ "${{ env.USE_LLVM_COMPILER }}" == "true" ]] + && echo + "env" + "CC=${{ env.LLVM_INSTALL_BINDIR }}/clang" + "CXX=${{ env.LLVM_INSTALL_BINDIR }}/clang++" + ) python3 ./tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}} --source-dir . --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" - $( [[ "${{ env.USE_LLVM_COMPILER }}" == "true" ]] - && echo "--ar ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ar" - "--ranlib ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ranlib" ) + $( + [[ "${{ env.USE_LLVM_COMPILER }}" == "true" ]] + && echo + "--ar ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ar" + "--ranlib ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ranlib" + ) # See https://github.com/actions/setup-python/issues/577 - name: "Remove preinstalled binaries which conflict with brew's installs" From 379c20662d58d54e2e0128164513fafabd20e90e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 09:09:15 -0500 Subject: [PATCH 26/27] formalize --- .github/workflows/clp-core-build-macos.yaml | 41 +++++++++++---------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index d58efd6c1..ab3dea7dc 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -41,7 +41,10 @@ jobs: build-macos: strategy: matrix: - os: ["macos-13", "macos-14"] + os: + - "macos-13" + - "macos-14" + - "macos-15" use_shared_libs: [true, false] runs-on: "${{matrix.os}}" steps: @@ -49,6 +52,24 @@ jobs: with: submodules: "recursive" + # See https://github.com/actions/setup-python/issues/577 + - name: "Remove preinstalled binaries which conflict with brew's installs" + run: | + rm -f /usr/local/bin/2to3* + rm -f /usr/local/bin/go* + rm -f /usr/local/bin/idle3* + rm -f /usr/local/bin/pydoc3* + rm -f /usr/local/bin/python3* + + - name: "Install dependencies" + run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" + + - run: "./tools/scripts/deps-download/init.sh" + shell: "bash" + + - run: "task deps:core" + shell: "bash" + - if: "'macos-13' == matrix.os || 'macos-14' == matrix.os" name: "Upgrade the default AppleClang to LLVM Clang for C++20 features" shell: "bash" @@ -81,21 +102,3 @@ jobs: "--ar ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ar" "--ranlib ${{ env.LLVM_INSTALL_BINDIR }}/llvm-ranlib" ) - - # See https://github.com/actions/setup-python/issues/577 - - name: "Remove preinstalled binaries which conflict with brew's installs" - run: | - rm -f /usr/local/bin/2to3* - rm -f /usr/local/bin/go* - rm -f /usr/local/bin/idle3* - rm -f /usr/local/bin/pydoc3* - rm -f /usr/local/bin/python3* - - - name: "Install dependencies" - run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" - - - run: "./tools/scripts/deps-download/init.sh" - shell: "bash" - - - run: "task deps:core" - shell: "bash" From f62ad4591ca8868645d7809aeba0e5aa71eed3b7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 6 Mar 2025 09:59:31 -0500 Subject: [PATCH 27/27] Remove debugging statements --- components/core/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index a995cbca3..d8b0352a4 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -25,9 +25,6 @@ else() message(SEND_ERROR "GENERAL_COMPRESSOR=${GENERAL_COMPRESSOR} is unimplemented.") endif() -message(STATUS "CMAKE_AR: ${CMAKE_AR}") -message(STATUS "CMAKE_RANLIB: ${CMAKE_RANLIB}") - # Add local CMake module directory to CMake's modules path set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")