Merge pull request #2793 from tomjakubowski/arrow-conda-patch

Port Conda patches, use vendored boost
finos · Oct 18, 2024 · 4f1142a · 4f1142a
2 parents 83bcdfa + f9ca424
commit 4f1142a
Show file tree

Hide file tree

Showing 33 changed files with 226 additions and 282 deletions.
diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml
@@ -111,7 +111,6 @@ runs:
         #   with:
         #       key: ${{ github.job }}-${{ matrix.os }}
 
-        # TODO this should move to `install_tools.mjs`
         - name: Windows init steps (vc143)
           id: vcpkg-step
           shell: pwsh
@@ -220,17 +219,6 @@ runs:
               fi
           if: ${{ runner.os == 'Linux' && inputs.cpp == 'true' && inputs.javascript == 'false' }}
 
-        # TODO do this earlier?
-        - name: Linux init steps
-          shell: bash
-          run: node tools/perspective-scripts/install_tools.mjs
-          if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux == 'true' }}
-
-        - name: Linux init steps
-          shell: bash
-          run: sudo node tools/perspective-scripts/install_tools.mjs
-          if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux  == 'false' }}
-
         - name: Install Python Pyodide dependencies
           shell: bash
           run: python -m pip install -r rust/perspective-python/requirements-pyodide.txt

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -250,6 +250,7 @@ jobs:
               if: ${{ contains(matrix.os, 'windows') }}
               env:
                   CARGO_TARGET_DIR: D:\psp-rust
+                  PSP_CPP_BUILD_DIR: D:\psp-build
                   PSP_ROOT_DIR: ${{ github.workspace }}
                   PACKAGE: "perspective-python"
                   PSP_ARCH: ${{ matrix.arch }}

diff --git a/.prettierignore b/.prettierignore
@@ -3,11 +3,12 @@ build/
 node_modules/
 target/
 .emsdk/
+.venv*/
 boost*/
 py_modules/
 ts-rs/
 rust/perspective-python/perspective/labextension/
 rust/perspective-python/perspective.data/
 rust/perspective-python/*/data
 expression_gen.md
-rust/perspective-viewer/docs/exprtk.md
+rust/perspective-viewer/docs/exprtk.md
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -23,9 +23,6 @@ automatically call the correct build and test tools.
 dependencies to be installed:
 
 -   [CMake](https://cmake.org/) (version 3.29.5 or higher)
--   [Boost](https://www.boost.org/) (version 1.83 or highery). This can be
-    installed from tarball with the included script
-    `node tools/perspective-scripts/install_tools.mjs`.
 -   [pnpm](https://pnpm.io/).
 
 **_This list may be non-exhaustive depending on your OS/environment; please open

diff --git a/cmake/Boost.txt.in b/cmake/Boost.txt.in
@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.7.2)
+
+project(Boost-download NONE)
+
+# Makes GIT_SUBMODULES "" in ExternalProject_Add skip initializing submodules
+cmake_policy(SET CMP0097 NEW)
+
+include(ExternalProject)
+ExternalProject_Add(Boost
+  URL               "https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz"
+  SOURCE_DIR        "${CMAKE_BINARY_DIR}/Boost-src"
+  BINARY_DIR        "${CMAKE_BINARY_DIR}/Boost-build"
+  SOURCE_SUBDIR     ""
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND     ""
+  INSTALL_COMMAND   ""
+  TEST_COMMAND      ""
+)
diff --git a/cmake/Pybind.txt.in b/cmake/Pybind.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(pybind11
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/arrow.txt.in b/cmake/arrow.txt.in
@@ -2,21 +2,23 @@ cmake_minimum_required(VERSION 3.7.2)
 
 project(arrow-download NONE)
 
-# *******************************************************************
-# NOTE: this depedency's CMakeLists.txt is overloaded, make sure to
-# adjust if you update the tag
-# *******************************************************************
+# Makes GIT_SUBMODULES "" in ExternalProject_Add skip initializing submodules
+cmake_policy(SET CMP0097 NEW)
 
 include(ExternalProject)
 ExternalProject_Add(apachearrow
   GIT_REPOSITORY    https://github.com/apache/arrow.git
   GIT_TAG           apache-arrow-17.0.0
+  GIT_SUBMODULES    ""
+  GIT_SHALLOW       TRUE
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/arrow-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/arrow-build"
   SOURCE_SUBDIR     "cpp"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        "-DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+  # This patch is to work around https://github.com/apache/arrow/issues/44384
+  # It can be removed when a version of Arrow is released with https://github.com/apache/arrow/pull/44385
+  PATCH_COMMAND     ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> git apply ${CMAKE_SOURCE_DIR}/patches/fix_arrow_libtool.patch
 )
diff --git a/cmake/date.txt.in b/cmake/date.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(date
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/double-conversion.txt.in b/cmake/double-conversion.txt.in
diff --git a/cmake/exprtk.txt.in b/cmake/exprtk.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(exprtk
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/flatbuffers.txt.in b/cmake/flatbuffers.txt.in
diff --git a/cmake/hopscotch.txt.in b/cmake/hopscotch.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(hopscotch
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/lz4.txt.in b/cmake/lz4.txt.in
diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake
@@ -7,8 +7,10 @@ set(D "$")
 # Helper to grab dependencies from remote sources #
 # ##################################################
 function(psp_build_dep name cmake_file)
-    if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build AND NOT name STREQUAL "lz4")
+    if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build)
         psp_build_message("${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
+    elseif(NAME STREQUAL "Boost")
+        psp_build_message("${Cyan}Not building boost${ColorReset}")
     else()
         configure_file(${cmake_file} ${CMAKE_BINARY_DIR}/${name}-download/CMakeLists.txt)
         set(_cwd ${CMAKE_BINARY_DIR}/${name}-download)
@@ -35,7 +37,6 @@ function(psp_build_dep name cmake_file)
             message(FATAL_ERROR "Build step for ${name} failed: ${result}")
         endif()
     endif()
-
     if(${name} STREQUAL arrow)
         set(ARROW_SIMD_LEVEL "NONE")
         set(ARROW_DEFINE_OPTIONS ON)
@@ -49,59 +50,45 @@ function(psp_build_dep name cmake_file)
         set(ARROW_WITH_ZSTD ON)
         set(ARROW_WITH_LZ4 ON)
         set(ARROW_NO_EXPORT ON)
+        set(ARROW_DEPENDENCY_SOURCE "BUNDLED" CACHE STRING "override arrow's dependency source" FORCE)
         if(PSP_WASM_BUILD)
             set(ARROW_ENABLE_THREADING OFF)
         else()
             set(ARROW_ENABLE_THREADING ON)
-            if(WIN32)
-                set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
-            endif()
         endif()
 
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-build/src)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL
         )
-
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
-    elseif(${name} STREQUAL exprtk)
-        # no cmakelists - just include the header
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
+        set(${name}_INCLUDE_DIRS
+            "${CMAKE_BINARY_DIR}/${name}-build/src" # needed for generated headers, e.g. <arrow/util/config.h>
+            "${CMAKE_BINARY_DIR}/${name}-src/cpp/src"
+            PARENT_SCOPE)
     elseif(${name} STREQUAL re2)
         # Overwrite re2's CMakeLists with our custom CMakeLists.
         configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/ COPYONLY)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
 
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
-    elseif(${name} STREQUAL lz4)
-        # lz4's CMakeLists.txt is in a subdir, build/cmake
-        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/build/cmake
-            ${CMAKE_BINARY_DIR}/${name}-build
-            EXCLUDE_FROM_ALL)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/lib)
+        set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src PARENT_SCOPE)
     elseif(${name} STREQUAL protobuf)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/src)
-    else()
+        set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src/src PARENT_SCOPE)
+    elseif(${name} STREQUAL rapidjson)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
-
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/extras/${name}/include)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/include)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
-    endif()
-
-    if(NOT PSP_WASM_BUILD AND (MACOS OR NOT MANYLINUX))
-        if(${name} STREQUAL arrow_static OR ${name} STREQUAL flatbuffers OR ${name} STREQUAL double-conversion OR ${name} STREQUAL re2)
-            target_compile_options(${name} PRIVATE -fvisibility=hidden)
-        endif()
+        set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src/include" PARENT_SCOPE)
+    # Header-only dependencies without a build step - no add_subdirectory()
+    elseif(${name} MATCHES "^(Boost|exprtk)")
+        set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src" PARENT_SCOPE)
+    elseif(${name} MATCHES "^(hopscotch|date|ordered-map)$")
+        set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src/include" PARENT_SCOPE)
+    else()
+        message(FATAL_ERROR "Unknown dependency `${name}`")
     endif()
 endfunction()
-
-# #############################
diff --git a/cmake/modules/LLVMToolchain.cmake b/cmake/modules/LLVMToolchain.cmake
@@ -6,4 +6,4 @@ set(CMAKE_AR "${LLVM_ROOT}/bin/llvm-ar")
 set(CMAKE_RANLIB "${LLVM_ROOT}/bin/llvm-ranlib")
 
 # include_directories(BEFORE SYSTEM "${LLVM_ROOT}/include")
-# link_directories(BEFORE "${LLVM_ROOT}/lib")
+# link_directories(BEFORE "${LLVM_ROOT}/lib")
diff --git a/cmake/modules/SetupClangd.cmake b/cmake/modules/SetupClangd.cmake
@@ -36,7 +36,3 @@ if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/.clangd.in)
 else()
     message("${Yellow}No .clangd.in found, skipping IDE setup${ColorReset}")
 endif()
-
-if(NOT WIN32)
-    include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-endif()
diff --git a/cmake/ordered-map.txt.in b/cmake/ordered-map.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(ordered-map
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/protobuf.txt.in b/cmake/protobuf.txt.in
@@ -20,5 +20,4 @@ ExternalProject_Add(protobuf
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/rapidjson.txt.in b/cmake/rapidjson.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(rapidjson
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DRAPIDJSON_BUILD_DOC=OFF -DRAPIDJSON_BUILD_EXAMPLES=OFF -DRAPIDJSON_BUILD_TESTS=OFF -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/re2.txt.in b/cmake/re2.txt.in
@@ -17,5 +17,4 @@ ExternalProject_Add(re2
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )