Refactor setup scripts

Use a common file to download and install external dependencies.
Extract versions for each library.

.github/workflows/linux-build.yml

@@ -25,6 +25,8 @@ on:
       - "CMake/**"
       - "third_party/**"
       - "scripts/setup-ubuntu.sh"
+      - "scripts/setup-common.sh"
+      - "scripts/setup-versions.sh"
       - "scripts/setup-helper-functions.sh"
       - ".github/workflows/linux-build.yml"
 
@@ -36,6 +38,8 @@ on:
       - "CMake/**"
       - "third_party/**"
       - "scripts/setup-ubuntu.sh"
+      - "scripts/setup-common.sh"
+      - "scripts/setup-versions.sh"
       - "scripts/setup-helper-functions.sh"
       - ".github/workflows/linux-build.yml"
 
@@ -171,7 +175,7 @@ jobs:
           MAKEFLAGS: "NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3"
           EXTRA_CMAKE_FLAGS: "-DVELOX_ENABLE_ARROW=ON -DVELOX_ENABLE_PARQUET=ON"
         run: |
-          make debug 
+          make debug
 
       - name: CCache after
         run: |

.github/workflows/macos.yml

@@ -63,9 +63,7 @@ jobs:
         env:
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE"
         run: |
-          source scripts/setup-macos.sh
-          brew install $MACOS_BUILD_DEPS $MACOS_VELOX_DEPS
-
+          scripts/setup-macos.sh
           echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
           brew unlink protobuf || echo "protobuf not installed"
           brew link --force protobuf@21
@@ -78,7 +76,7 @@ jobs:
 
       - name: Configure Build
         env:
-          folly_SOURCE: BUNDLED #brew folly does not have int128
+          fmt_SOURCE: BUNDLED #picked up brew fmt (from ccache) is too new
         run: |
             ccache -sz -M 5Gi
             cmake \

.github/workflows/scheduled.yml

@@ -23,6 +23,8 @@ on:
       - "CMake/**"
       - "third_party/**"
       - "scripts/setup-ubuntu.sh"
+      - "scripts/setup-common.sh"
+      - "scripts/setup-versions.sh"
       - "scripts/setup-helper-functions.sh"
       - ".github/workflows/scheduled.yml"
 
@@ -36,6 +38,8 @@ on:
       - "CMake/**"
       - "third_party/**"
       - "scripts/setup-ubuntu.sh"
+      - "scripts/setup-common.sh"
+      - "scripts/setup-versions.sh"
       - "scripts/setup-helper-functions.sh"
       - ".github/workflows/scheduled.yml"
 

README.md

@@ -132,11 +132,13 @@ libraries. Once you have checked out Velox, you can setup and build like so:
 
 ```shell
 $ ./scripts/setup-centos9.sh
-$ ./scripts/setup-adapters.sh
+$ ./scripts/setup-centos9.sh install_adapters
 $ make
 ```
 
-Note that `setup-adapters.sh` supports MacOS and Ubuntu 20.04 or later.
+Note that the `install_adapters` command is available for the supported MacOS and
+Ubuntu (20.04 or later) scripts. Individual adapters can be installed by specifying
+the individual install command, e.g. `setup-centos9.sh install_aws`.
 
 ### Using Clang on Linux
 

scripts/adapters.dockerfile

@@ -15,15 +15,18 @@
 ARG image=ghcr.io/facebookincubator/velox-dev:centos9
 FROM $image
 
-COPY scripts/setup-adapters.sh /
+COPY scripts/setup-centos9.sh /
+COPY scripts/setup-common.sh /
+COPY scripts/setup-versions.sh /
+COPY scripts/setup-helper-functions.sh /
 RUN mkdir build && ( cd build &&  source /opt/rh/gcc-toolset-12/enable && \
-    bash /setup-adapters.sh ) && rm -rf build && dnf remove -y conda && dnf clean all
+    bash /setup-centos9.sh install_adapters ) && rm -rf build && dnf remove -y conda && dnf clean all
 
 # install miniforge
 RUN curl -L -o /tmp/miniforge.sh https://github.com/conda-forge/miniforge/releases/download/23.11.0-0/Mambaforge-23.11.0-0-Linux-x86_64.sh && \
     bash /tmp/miniforge.sh -b -p /opt/miniforge && \
     rm /tmp/miniforge.sh
-ENV PATH=/opt/miniforge/condabin:${PATH} 
+ENV PATH=/opt/miniforge/condabin:${PATH}
 
 # install test dependencies
 RUN mamba create -y --name adapters python=3.8

scripts/centos.dockerfile

@@ -16,6 +16,8 @@ ARG image=quay.io/centos/centos:stream9
 FROM $image
 
 COPY scripts/setup-helper-functions.sh /
+COPY scripts/setup-versions.sh /
+COPY scripts/setup-common.sh /
 COPY scripts/setup-centos9.sh /
 # The removal of the build dir has to happen in the same layer as the build
 # to minimize the image size. gh & jq are required for CI