mega commit

WIP not working ``` /bin/sh: /opt/homebrew/lib/python3.11/site-packages/cmake/data/bin/cmake -E chdir /Users/tom/perspective/perspective/rust/target/debug/build/perspective-server-639831860500edae/out/build/arrow-src git apply /Users/tom/perspective/perspective/rust/perspective-server/cpp/perspective/patches/fix_arrow_libtool.patch: No such file or directory ``` switching off to arrow PR will also add the fix_flags.patch to this branch I think Signed-off-by: Tom Jakubowski <[email protected]> WIP, switch to BUNDLED arrow build failing because despite ARROW_JEMALLOC assuredly being OFF, the .so is ending up with missing arrow symbols referencing jemalloc ``` ImportError: dlopen(/Users/tom/perspective/perspective/rust/perspective-python/perspective/perspective.abi3.so, 0x0002): symbol not found in flat namespace '__ZN5arrow11memory_pool8internal17JemallocAllocator13ReleaseUnusedEv' ``` Signed-off-by: Tom Jakubowski <[email protected]> reorder arrow Signed-off-by: Tom Jakubowski <[email protected]> fix patch Signed-off-by: Tom Jakubowski <[email protected]> remove zstd dep Signed-off-by: Tom Jakubowski <[email protected]> remove traces of arrow deps moved into bundle Signed-off-by: Tom Jakubowski <[email protected]> annotate PATCH_COMMAND Signed-off-by: Tom Jakubowski <[email protected]> ignore .venv for prettier Signed-off-by: Tom Jakubowski <[email protected]> Better control over include directory ordering `psp_build_dep(name)` now exports in its parent scope a `${name}_INCLUDE_DIRS` variable which contains a list of include directories containing that dependency's header files. These variables are then joined into a list in CMakeLists.txt, which is then passed to `target_include_directories(psp)` There are no longer any calls to `include_directories()`. Some extra include paths were also removed, like `/usr/local/include` for Boost, which is better covered by `Boost_INCLUDE_DIRS`. Also removes `boost_system` from the list of Boost requirements; we only need Boost headers. closes #2792 Signed-off-by: Tom Jakubowski <[email protected]> more careful with Boost hoping won't need to restore the WIN32-sensitive ordering of the find_package(Boost) call Signed-off-by: Tom Jakubowski <[email protected]> remove unused CMAKE_ARGS Signed-off-by: Tom Jakubowski <[email protected]> perspective-server: parse CMAKE_ARGS from environ supports the conda build Signed-off-by: Tom Jakubowski <[email protected]> fix emscripten build Signed-off-by: Tom Jakubowski <[email protected]> make VCPKG_ROOT optional, but warn prevents need for fix-windows.diff patch Signed-off-by: Tom Jakubowski <[email protected]> oops fix typo Signed-off-by: Tom Jakubowski <[email protected]> replace find_package(Boost) with psp_build_dep This may make local builds from clean take slightly longer (since it won't use any latent Boost on the system), but should make CI builds faster since the "build boost" step of install-tools.mjs can be skipped. The headers in Boost's release tarballs can be used directly. Was also possible to update `find_package(Boost)` call to use `PATHS` to request the Emscripten build to look in system trees, which I tried first, but getting the paths to where FindBoost.cmake might be right on every environment was like whack-a-mole. This way is the same on every environment and makes Boost dependency consistent with the others Signed-off-by: Tom Jakubowski <[email protected]> conda-build: detect conda, use , add feature flag for abi3 subsumes fix_maturin.diff and disable_abi3.diff Signed-off-by: Tom Jakubowski <[email protected]> use explicit PSP_CPP_BUILD_DIR completes subsumption of fix_windows.diff Signed-off-by: Tom Jakubowski <[email protected]> use non-system includes for psp_INCLUDE_DIRS Signed-off-by: Tom Jakubowski <[email protected]> Remove install_tools.mjs Signed-off-by: Tom Jakubowski <[email protected]> fix re2 include_dirs export how was psp able to build like this? Signed-off-by: Tom Jakubowski <[email protected]> fixup install_tools Signed-off-by: Tom Jakubowski <[email protected]> pass include dirs to perspective_esm target also Signed-off-by: Tom Jakubowski <[email protected]>
finos · Oct 17, 2024 · 493b915 · 493b915
1 parent 13b26d7
commit 493b915
Show file tree

Hide file tree

Showing 28 changed files with 189 additions and 271 deletions.
diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml
@@ -108,7 +108,6 @@ runs:
         #   with:
         #       key: ${{ github.job }}-${{ matrix.os }}
 
-        # TODO this should move to `install_tools.mjs`
         - name: Windows init steps (vc143)
           id: vcpkg-step
           shell: pwsh
@@ -205,17 +204,6 @@ runs:
               fi
           if: ${{ runner.os == 'Linux' && inputs.cpp == 'true' && inputs.javascript == 'false' }}
 
-        # TODO do this earlier?
-        - name: Linux init steps
-          shell: bash
-          run: node tools/perspective-scripts/install_tools.mjs
-          if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux == 'true' }}
-
-        - name: Linux init steps
-          shell: bash
-          run: sudo node tools/perspective-scripts/install_tools.mjs
-          if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux  == 'false' }}
-
         - name: Install Python Pyodide dependencies
           shell: bash
           run: python -m pip install -r rust/perspective-python/requirements-pyodide.txt

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -204,6 +204,7 @@ jobs:
               if: ${{ contains(matrix.os, 'windows') }}
               env:
                   CARGO_TARGET_DIR: D:\psp-rust
+                  PSP_CPP_BUILD_DIR: D:\psp-build
                   PSP_ROOT_DIR: ${{ github.workspace }}
                   VCPKG_ROOT: ${{ steps.init-step.outputs.VCPKG_INSTALLATION_ROOT }}
                   PACKAGE: "perspective-python"

diff --git a/.prettierignore b/.prettierignore
@@ -9,3 +9,4 @@ ts-rs/
 rust/perspective-python/perspective/labextension/
 rust/perspective-python/perspective.data/
 rust/perspective-python/*/data
+.venv*/
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/cmake/Pybind.txt.in b/cmake/Pybind.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(pybind11
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/arrow.txt.in b/cmake/arrow.txt.in
@@ -2,21 +2,23 @@ cmake_minimum_required(VERSION 3.7.2)
 
 project(arrow-download NONE)
 
-# *******************************************************************
-# NOTE: this depedency's CMakeLists.txt is overloaded, make sure to
-# adjust if you update the tag
-# *******************************************************************
+# Makes GIT_SUBMODULES "" in ExternalProject_Add skip initializing submodules
+cmake_policy(SET CMP0097 NEW)
 
 include(ExternalProject)
 ExternalProject_Add(apachearrow
   GIT_REPOSITORY    https://github.com/apache/arrow.git
   GIT_TAG           apache-arrow-17.0.0
+  GIT_SUBMODULES    ""
+  GIT_SHALLOW       TRUE
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/arrow-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/arrow-build"
   SOURCE_SUBDIR     "cpp"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        "-DFLATBUFFERS_BUILD_TESTS=OFF -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+  # This patch is to work around https://github.com/apache/arrow/issues/44384
+  # It can be removed when a version of Arrow is released with https://github.com/apache/arrow/pull/44385
+  PATCH_COMMAND     ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> git apply ${CMAKE_SOURCE_DIR}/patches/fix_arrow_libtool.patch
 )
diff --git a/cmake/boost.txt.in b/cmake/boost.txt.in
@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.7.2)
+
+project(boost-download NONE)
+
+# Makes GIT_SUBMODULES "" in ExternalProject_Add skip initializing submodules
+cmake_policy(SET CMP0097 NEW)
+
+include(ExternalProject)
+ExternalProject_Add(apachearrow
+  URL               "https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz"
+  SOURCE_DIR        "${CMAKE_BINARY_DIR}/boost-src"
+  BINARY_DIR        "${CMAKE_BINARY_DIR}/boost-build"
+  SOURCE_SUBDIR     ""
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND     ""
+  INSTALL_COMMAND   ""
+  TEST_COMMAND      ""
+)
diff --git a/cmake/date.txt.in b/cmake/date.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(date
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/double-conversion.txt.in b/cmake/double-conversion.txt.in
diff --git a/cmake/exprtk.txt.in b/cmake/exprtk.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(exprtk
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/flatbuffers.txt.in b/cmake/flatbuffers.txt.in
diff --git a/cmake/hopscotch.txt.in b/cmake/hopscotch.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(hopscotch
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/lz4.txt.in b/cmake/lz4.txt.in
diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake
@@ -7,8 +7,10 @@ set(D "$")
 # Helper to grab dependencies from remote sources #
 # ##################################################
 function(psp_build_dep name cmake_file)
-    if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build AND NOT name STREQUAL "lz4")
+    if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build)
         psp_build_message("${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
+    elseif(NAME STREQUAL "boost")
+        psp_build_message("${Cyan}Not building boost${ColorReset}")
     else()
         configure_file(${cmake_file} ${CMAKE_BINARY_DIR}/${name}-download/CMakeLists.txt)
         set(_cwd ${CMAKE_BINARY_DIR}/${name}-download)
@@ -35,7 +37,6 @@ function(psp_build_dep name cmake_file)
             message(FATAL_ERROR "Build step for ${name} failed: ${result}")
         endif()
     endif()
-
     if(${name} STREQUAL arrow)
         set(ARROW_SIMD_LEVEL "NONE")
         set(ARROW_DEFINE_OPTIONS ON)
@@ -49,59 +50,45 @@ function(psp_build_dep name cmake_file)
         set(ARROW_WITH_ZSTD ON)
         set(ARROW_WITH_LZ4 ON)
         set(ARROW_NO_EXPORT ON)
+        set(ARROW_DEPENDENCY_SOURCE "BUNDLED" CACHE STRING "override arrow's dependency source" FORCE)
         if(PSP_WASM_BUILD)
             set(ARROW_ENABLE_THREADING OFF)
         else()
             set(ARROW_ENABLE_THREADING ON)
-            if(WIN32)
-                set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
-            endif()
         endif()
 
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-build/src)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL
         )
-
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
-    elseif(${name} STREQUAL exprtk)
-        # no cmakelists - just include the header
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
+        set(${name}_INCLUDE_DIRS
+            "${CMAKE_BINARY_DIR}/${name}-build/src" # needed for generated headers, e.g. <arrow/util/config.h>
+            "${CMAKE_BINARY_DIR}/${name}-src/cpp/src"
+            PARENT_SCOPE)
     elseif(${name} STREQUAL re2)
         # Overwrite re2's CMakeLists with our custom CMakeLists.
         configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/ COPYONLY)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
 
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
-    elseif(${name} STREQUAL lz4)
-        # lz4's CMakeLists.txt is in a subdir, build/cmake
-        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/build/cmake
-            ${CMAKE_BINARY_DIR}/${name}-build
-            EXCLUDE_FROM_ALL)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/lib)
+        set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src PARENT_SCOPE)
     elseif(${name} STREQUAL protobuf)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/src)
-    else()
+        set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src/src PARENT_SCOPE)
+    elseif(${name} STREQUAL rapidjson)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
-
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/extras/${name}/include)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/include)
-        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src)
-    endif()
-
-    if(NOT PSP_WASM_BUILD AND (MACOS OR NOT MANYLINUX))
-        if(${name} STREQUAL arrow_static OR ${name} STREQUAL flatbuffers OR ${name} STREQUAL double-conversion OR ${name} STREQUAL re2)
-            target_compile_options(${name} PRIVATE -fvisibility=hidden)
-        endif()
+        set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src/include" PARENT_SCOPE)
+    # Header-only dependencies without a build step - no add_subdirectory()
+    elseif(${name} MATCHES "^(boost|exprtk)")
+        set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src" PARENT_SCOPE)
+    elseif(${name} MATCHES "^(hopscotch|date|ordered-map)$")
+        set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src/include" PARENT_SCOPE)
+    else()
+        message(FATAL_ERROR "Unknown dependency `${name}`")
     endif()
 endfunction()
-
-# #############################
diff --git a/cmake/modules/LLVMToolchain.cmake b/cmake/modules/LLVMToolchain.cmake
@@ -6,4 +6,4 @@ set(CMAKE_AR "${LLVM_ROOT}/bin/llvm-ar")
 set(CMAKE_RANLIB "${LLVM_ROOT}/bin/llvm-ranlib")
 
 # include_directories(BEFORE SYSTEM "${LLVM_ROOT}/include")
-# link_directories(BEFORE "${LLVM_ROOT}/lib")
+# link_directories(BEFORE "${LLVM_ROOT}/lib")
diff --git a/cmake/modules/SetupClangd.cmake b/cmake/modules/SetupClangd.cmake
@@ -36,7 +36,3 @@ if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/.clangd.in)
 else()
     message("${Yellow}No .clangd.in found, skipping IDE setup${ColorReset}")
 endif()
-
-if(NOT WIN32)
-    include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-endif()
diff --git a/cmake/ordered-map.txt.in b/cmake/ordered-map.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(ordered-map
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/protobuf.txt.in b/cmake/protobuf.txt.in
@@ -20,5 +20,4 @@ ExternalProject_Add(protobuf
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/rapidjson.txt.in b/cmake/rapidjson.txt.in
@@ -12,5 +12,4 @@ ExternalProject_Add(rapidjson
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DRAPIDJSON_BUILD_DOC=OFF -DRAPIDJSON_BUILD_EXAMPLES=OFF -DRAPIDJSON_BUILD_TESTS=OFF -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )
diff --git a/cmake/re2.txt.in b/cmake/re2.txt.in
@@ -17,5 +17,4 @@ ExternalProject_Add(re2
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        " -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
 )