Merge branch 'main' of github.com:facebookincubator/velox into circle…

…ci-add-cuda
facebookincubator · Mar 7, 2024 · 33235e5 · 33235e5
2 parents dbfc5a4 + 6f189b0
commit 33235e5
Show file tree

Hide file tree

Showing 543 changed files with 21,493 additions and 25,455 deletions.
diff --git a/.circleci/dist_compile.yml b/.circleci/dist_compile.yml
@@ -299,12 +299,6 @@ jobs:
       DuckDB_SOURCE: SYSTEM
     steps:
       - pre-steps
-      - run:
-          name: "Install Adapter Dependencies"
-          command: |
-            source /opt/rh/gcc-toolset-9/enable
-            set -xu
-            PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh abfs
       - run:
           name: "Install Java for Hadoop"
           command: |
@@ -431,43 +425,6 @@ jobs:
           fuzzer_exe: "_build/debug/velox/exec/tests/velox_join_fuzzer_test"
           fuzzer_args: " --seed ${RANDOM} --duration_sec 3600 --logtostderr=1 --minloglevel=0"
 
-  doc-gen-job:
-    executor: build
-    steps:
-      - checkout
-      - update-submodules
-      - add_ssh_keys:
-          fingerprints:
-            - "7b:24:f3:1a:b1:15:97:c6:fe:06:46:27:3e:b7:6b:96"
-      - run:
-          name: "Build docs and update gh-pages"
-          command: |
-            for i in {1..3}; do
-              make clean
-              git config --global user.email "[email protected]"
-              git config --global user.name "velox"
-              git checkout main
-              conda init bash
-              source ~/.bashrc
-              conda create -y --name docgenenv python=3.7
-              conda activate docgenenv
-              pip install sphinx sphinx-tabs breathe sphinx_rtd_theme chardet
-              source /opt/rh/gcc-toolset-9/enable
-              ./scripts/gen-docs.sh docgenenv
-              git checkout gh-pages
-              cp -R velox/docs/_build/html/* docs
-              git add docs
-              if [ -n "$(git status --porcelain --untracked-files=no)" ]
-              then
-                git commit -m "Update documentation"
-                git push
-                if [ $? -eq 0 ]; then
-                  break;
-                fi
-              fi
-            done
-
-
   linux-pr-fuzzer-run:
     executor: build
     steps:
@@ -572,11 +529,6 @@ workflows:
       - linux-build-options
       - linux-adapters
       - linux-presto-fuzzer-run
-      - doc-gen-job:
-          filters:
-            branches:
-              only:
-                - main
 
   shorter-fuzzer:
     unless: << pipeline.parameters.run-longer-expression-fuzzer >>
@@ -585,8 +537,3 @@ workflows:
       - linux-pr-fuzzer-run
       - linux-build-options
       - linux-adapters
-      - doc-gen-job:
-          filters:
-            branches:
-              only:
-                - main
diff --git a/velox/experimental/codegen/proto/protobuf.sh → .github/dependabot.yml b/velox/experimental/codegen/proto/protobuf.sh → .github/dependabot.yml
@@ -1,4 +1,3 @@
-#!/bin/bash
 # Copyright (c) Facebook, Inc. and its affiliates.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,4 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"${PROTOC}" "${PROTO_FILE}" "--${LANG}_out=$INSTALL_DIR"
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -35,7 +35,7 @@ concurrency:
 defaults:
   run:
     shell: bash
-#TODO  concurrency groups?
+
 jobs:
   benchmark:
     if: github.repository == 'facebookincubator/velox'
@@ -198,7 +198,7 @@ jobs:
           echo "::endgroup::"
 
       - name: "Save PR number"
-        run: echo "${{ github.event.pull_request.number }}" > pr_number.txt
+        run: echo "${{ github.event.pull_request.number || 0 }}" > pr_number.txt
 
       - name: "Upload PR number"
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/conbench_upload.yml b/.github/workflows/conbench_upload.yml
@@ -94,7 +94,18 @@ jobs:
       run: |
         unzip benchmark-results.zip -d benchmark-results
         unzip pr_number.zip
-        echo "pr_number=$(cat pr_number.txt)" >> $GITHUB_OUTPUT
+
+        pr_number=$(grep -ox '[[:digit:]]*' pr_number.txt | head -1)
+
+        if [ "$pr_number" -ge 0 ]; then
+          echo "Found PR number: $pr_number"
+        else
+          echo '::error :: Malformed input, aborting!'
+          exit 1
+        fi
+
+        echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
+
     - uses: actions/checkout@v3
       with:
         path: velox
@@ -134,7 +145,7 @@ jobs:
 
     - name: "Create a GitHub Status on the contender commit (whether the upload was successful)"
       uses: actions/github-script@v6
-      if: always()
+      if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
       with:
         script: |
           let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}'

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -0,0 +1,85 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Update Documentation
+
+on:
+  push:
+    paths:
+      - "velox/docs/**"
+      - ".github/workflows/docs.yml"
+
+  pull_request:
+    paths:
+      - "velox/docs/**"
+      - ".github/workflows/docs.yml"
+
+permissions:
+  contents: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  build_docs:
+    name: Build and Push
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          fetch-depth: 0
+
+      - name: Setup Git User
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "velox"
+
+      - name: Install Dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y pandoc
+          pip install sphinx sphinx-tabs breathe sphinx_rtd_theme chardet
+
+      - name: Build Documentation
+        run: |
+          cd velox/docs
+          make clean
+          # pyvelox
+          mkdir -p bindings/python
+          pandoc ../../pyvelox/README.md --from markdown --to rst -s -o bindings/python/README_generated_pyvelox.rst
+          # velox
+          make html
+
+      - name: Push Documentation
+        if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}}
+        run: |
+          git checkout gh-pages
+          cp -R velox/docs/_build/html/* docs
+          git add docs
+
+          if [ -n "$(git status --porcelain --untracked-files=no)" ]
+          then
+            git commit -m "Update documentation"
+            git push
+          fi
+
+      - name: Upload Documentation
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          path: velox/docs/_build/html
+          retention-days: 3
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -58,6 +58,8 @@ jobs:
         with:
           submodules: recursive
       - name: Install Dependencies
+        env:
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE"
         run: |
           brew install \
             bison boost ccache double-conversion flex fmt gflags glog \

diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml
@@ -18,6 +18,7 @@ on:
   pull_request:
     paths:
       - ".github/workflows/scheduled.yml"
+      - scripts/*
 
   schedule:
     - cron: '0 3 * * *'
@@ -75,7 +76,7 @@ jobs:
           ref: "${{ inputs.ref || 'main' }}"
 
       - name: "Install dependencies"
-        run: cd  velox && source ./scripts/setup-ubuntu.sh
+        run: cd  velox && sudo ./scripts/setup-ubuntu.sh 
 
       - name: "Build"
         run: |
@@ -125,7 +126,7 @@ jobs:
           ref: "${{ inputs.ref || 'main' }}"
 
       - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
+        run: sudo ./scripts/setup-ubuntu.sh
 
       - name: Download presto fuzzer
         uses: actions/download-artifact@v3
@@ -173,7 +174,7 @@ jobs:
           ref: "${{ inputs.ref || 'main' }}"
 
       - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
+        run: sudo ./scripts/setup-ubuntu.sh
 
       - name: Download spark expression fuzzer
         uses: actions/download-artifact@v3
@@ -247,7 +248,7 @@ jobs:
           ref: "${{ inputs.ref || 'main' }}"
 
       - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
+        run: sudo ./scripts/setup-ubuntu.sh
 
       - name: Download aggregation fuzzer
         uses: actions/download-artifact@v3
@@ -288,7 +289,7 @@ jobs:
           ref: "${{ inputs.ref || 'main' }}"
 
       - name: "Install dependencies"
-        run: source ./scripts/setup-ubuntu.sh
+        run:  sudo ./scripts/setup-ubuntu.sh
 
       - name: Download join fuzzer
         uses: actions/download-artifact@v3

diff --git a/CMake/resolve_dependency_modules/README.md b/CMake/resolve_dependency_modules/README.md
@@ -31,6 +31,7 @@ by Velox. See details on bundling below.
 | xsimd             | 10.0.0          | Yes      |
 | re2               | 2021-04-01      | Yes      |
 | fmt               | 10.1.1          | Yes      |
+| simdjson          | 3.2.0           | Yes      |
 | folly             | v2023.12.04.00  | Yes      |
 | fizz              | v2023.12.04.00  | No       |
 | wangle            | v2023.12.04.00  | No       |

diff --git a/CMake/resolve_dependency_modules/boost/CMakeLists.txt b/CMake/resolve_dependency_modules/boost/CMakeLists.txt
@@ -21,14 +21,14 @@ add_compile_options(-w)
 # We need to use boost > 1.70 to build it with CMake 1.81 was the first to be
 # released as a github release INCLUDING the cmake files (which are not in the
 # officale releases for some reason)
-set(VELOX_BOOST_BUILD_VERSION 1.81.0)
+set(VELOX_BOOST_BUILD_VERSION 1.84.0)
 string(
   CONCAT VELOX_BOOST_SOURCE_URL
          "https://github.com/boostorg/boost/releases/download/"
          "boost-${VELOX_BOOST_BUILD_VERSION}/"
          "boost-${VELOX_BOOST_BUILD_VERSION}.tar.gz")
 set(VELOX_BOOST_BUILD_SHA256_CHECKSUM
-    121da556b718fd7bd700b5f2e734f8004f1cfa78b7d30145471c526ba75a151c)
+    4d27e9efed0f6f152dc28db6430b9d3dfb40c0345da7342eaa5a987dde57bd95)
 
 resolve_dependency_url(BOOST)
 message(STATUS "Building boost from source")
@@ -65,8 +65,5 @@ list(REMOVE_ITEM BOOST_INCLUDE_LIBRARIES headers)
 set(BUILD_SHARED_LIBS ON)
 FetchContent_MakeAvailable(Boost)
 
-# To aling with Boost system install we create Boost::headers target
-add_library(boost_headers INTERFACE)
-add_library(Boost::headers ALIAS boost_headers)
 list(TRANSFORM BOOST_HEADER_ONLY PREPEND Boost::)
 target_link_libraries(boost_headers INTERFACE ${BOOST_HEADER_ONLY})
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -456,7 +456,7 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
 endif()
 
 set_source(simdjson)
-resolve_dependency(simdjson)
+resolve_dependency(simdjson 3.2.0)
 
 # Locate or build folly.
 add_compile_definitions(FOLLY_HAVE_INT128_T=1)

diff --git a/scripts/check.py b/scripts/check.py
@@ -189,9 +189,12 @@ def get_files(commit, path):
         )
         filelist = stdout.splitlines()
     else:
-        for root, dirs, files in os.walk(path):
-            for name in files:
-                filelist.append(os.path.join(root, name))
+        if os.path.isfile(path):
+            filelist.append(path)
+        else:
+            for root, dirs, files in os.walk(path):
+                for name in files:
+                    filelist.append(os.path.join(root, name))
 
     return [
         file

diff --git a/scripts/setup-centos8.sh b/scripts/setup-centos8.sh
@@ -90,7 +90,7 @@ function install_lzo {
 }
 
 function install_boost {
-  wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost
+  wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
   (
    cd boost
    ./bootstrap.sh --prefix=/usr/local

diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh
@@ -158,6 +158,7 @@ function cmake_install {
   mkdir -p "${BINARY_DIR}"
   CPU_TARGET="${CPU_TARGET:-unknown}"
   COMPILER_FLAGS=$(get_cxx_flags $CPU_TARGET)
+  SUDO="${SUDO:-""}"
 
   # CMAKE_POSITION_INDEPENDENT_CODE is required so that Velox can be built into dynamic libraries \
   cmake -Wno-dev -B"${BINARY_DIR}" \
@@ -169,6 +170,6 @@ function cmake_install {
     -DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" \
     -DBUILD_TESTING=OFF \
     "$@"
-  ninja -C "${BINARY_DIR}" install
+  ${SUDO} ninja -C "${BINARY_DIR}" install
 }