Merge branch 'main' of https://github.com/mohsaka/velox into ipaddr

.clang-format

@@ -1,20 +1,26 @@
 ---
 AccessModifierOffset: -1
 AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveMacros: false
 AlignConsecutiveAssignments: false
+AlignConsecutiveBitFields: false
 AlignConsecutiveDeclarations: false
-AlignEscapedNewlinesLeft: true
-AlignOperands:   false
+AlignEscapedNewlines: Left
+AlignOperands: DontAlign
 AlignTrailingComments: false
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
 AllowAllParametersOfDeclarationOnNextLine: false
-AllowShortBlocksOnASingleLine: false
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
 AllowShortCaseLabelsOnASingleLine: false
 AllowShortFunctionsOnASingleLine: Empty
-AllowShortIfStatementsOnASingleLine: false
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
 AllowShortLoopsOnASingleLine: false
 AlwaysBreakAfterReturnType: None
 AlwaysBreakBeforeMultilineStrings: true
-AlwaysBreakTemplateDeclarations: true
+AlwaysBreakTemplateDeclarations: Yes
 BinPackArguments: false
 BinPackParameters: false
 BraceWrapping:
@@ -31,19 +37,27 @@ BraceWrapping:
   IndentBraces:    false
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
+BreakInheritanceList: BeforeColon
 BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
 BreakAfterJavaFieldAnnotations: false
 BreakStringLiterals: false
-ColumnLimit:     80
-CommentPragmas:  '^ IWYU pragma:'
+ColumnLimit: 80
+CommentPragmas: '^ IWYU pragma:'
+CompactNamespaces: false
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
+DeriveLineEnding: true
 DerivePointerAlignment: false
-DisableFormat:   false
-ForEachMacros:   [ FOR_EACH, FOR_EACH_R, FOR_EACH_RANGE, ]
+DisableFormat: false
+FixNamespaceComments: true
+ForEachMacros:
+  - FOR_EACH
+  - FOR_EACH_R
+  - FOR_EACH_RANGE
+IncludeBlocks: Preserve
 IncludeCategories:
   - Regex:           '^<.*\.h(pp)?>'
     Priority:        1
@@ -52,36 +66,59 @@ IncludeCategories:
   - Regex:           '.*'
     Priority:        3
 IndentCaseLabels: true
-IndentWidth:     2
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentExternBlock: AfterExternBlock
+IndentWidth: 2
 IndentWrappedFunctionNames: false
+InsertNewlineAtEOF: true
+InsertTrailingCommas: None
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
 KeepEmptyLinesAtTheStartOfBlocks: false
 MacroBlockBegin: ''
-MacroBlockEnd:   ''
+MacroBlockEnd: ''
 MaxEmptyLinesToKeep: 1
 NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
 ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
 ObjCSpaceAfterProperty: false
 ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 2
 PenaltyBreakBeforeFirstCallParameter: 1
 PenaltyBreakComment: 300
 PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
-ReflowComments:  true
-SortIncludes:    true
+ReflowComments: true
+SortIncludes: true
+SortUsingDeclarations: true
 SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
 SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
 SpaceInEmptyParentheses: false
 SpacesBeforeTrailingComments: 1
-SpacesInAngles:  false
+SpacesInAngles: false
+SpacesInConditionalStatement: false
 SpacesInContainerLiterals: true
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
-Standard:        Cpp11
-TabWidth:        8
-UseTab:          Never
+SpaceBeforeSquareBrackets: false
+Standard: Cpp11
+TabWidth: 8
+UseCRLF: false
+UseTab: Never
 ...

.github/workflows/docker.yml

@@ -20,6 +20,7 @@ on:
       - scripts/*.dockerfile
       - scripts/setup-*.sh
       - .github/workflows/docker.yml
+      - docker-compose.yml
   push:
     branches: [main]
     paths:
@@ -46,16 +47,14 @@ jobs:
         include:
           - name: Check
             file: "scripts/check-container.dockfile"
-            args: "cpu_target=avx"
-            tags: "ghcr.io/facebookincubator/velox-dev:check-avx"
+            tags: "ghcr.io/facebookincubator/velox-dev:check"
           - name: Centos 9
             file: "scripts/centos.dockerfile"
-            args: "cpu_target=avx"
             tags: "ghcr.io/facebookincubator/velox-dev:centos9"
           - name: Dev
             file: "scripts/ubuntu-22.04-cpp.dockerfile"
             args: ""
-            tags: "ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx"
+            tags: "ghcr.io/facebookincubator/velox-dev:ubuntu-22.04"
 
     steps:
       - name: Login to GitHub Container Registry
@@ -89,11 +88,10 @@ jobs:
         include:
           - name: Adapters
             file: "scripts/adapters.dockerfile"
-            args: "cpu_target=avx"
             tags: "ghcr.io/facebookincubator/velox-dev:adapters"
           - name: Presto Java
             file: "scripts/prestojava-container.dockerfile"
-            args: "PRESTO_VERSION=0.284"
+            args: "PRESTO_VERSION=0.288"
             tags: "ghcr.io/facebookincubator/velox-dev:presto-java"
           - name: Spark server
             file: "scripts/spark-container.dockerfile"

.github/workflows/linux-build.yml

@@ -59,8 +59,10 @@ jobs:
     env:
       CCACHE_DIR: "/__w/velox/velox/.ccache"
       VELOX_DEPENDENCY_SOURCE: SYSTEM
+      Protobuf_SOURCE: BUNDLED # can be removed after #10134 is merged
       simdjson_SOURCE: BUNDLED
       xsimd_SOURCE: BUNDLED
+      Arrow_SOURCE: AUTO
       CUDA_VERSION: "12.4"
     steps:
       - uses: actions/checkout@v4
@@ -108,7 +110,11 @@ jobs:
             "-DVELOX_ENABLE_GPU=ON"
           )
           make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}"
-    
+
+      #Temporarily rename Minio binary to include version until CI image is updated with the same.
+      - name: Rename Minio
+        run: [ -f /usr/local/bin/minio ] && mv /usr/local/bin/minio /usr/local/bin/minio-2022-05-26
+
       - name: Ccache after
         run: ccache -s
 

.github/workflows/macos.yml

@@ -63,17 +63,18 @@ jobs:
         run: |
           brew install \
             bison boost ccache double-conversion flex fmt gflags glog \
-            icu4c libevent libsodium lz4 lzo ninja openssl range-v3 simdjson \
-            snappy thrift xz xsimd zstd
+            icu4c libevent libsodium lz4 lzo ninja openssl protobuf@21 \
+            range-v3 simdjson snappy thrift xz xsimd zstd
 
           echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
+          brew unlink protobuf || echo "protobuf not installed"
+          brew link --force protobuf@21
 
       - name: Cache ccache
-        uses: actions/cache@v4
+        uses: assignUser/stash/restore@v1
         with:
           path: '${{ env.CCACHE_DIR }}'
-          key: ccache-macos-${{ matrix.os }}-${{ hashFiles('velox/*') }}
-          restore-keys: ccache-macos-${{ matrix.os }}
+          key: ccache-macos-${{ matrix.os }}
 
       - name: Configure Build
         env:
@@ -92,6 +93,12 @@ jobs:
         run: |
             cmake --build _build/debug -j $NJOBS
             ccache -s
+
+      - uses: assignUser/stash/save@v1
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-macos-${{ matrix.os }}
+
       - name: Run Tests
         if: false
         run: ctest -j $NJOBS --test-dir _build/debug --output-on-failure

.github/workflows/preliminary_checks.yml

@@ -27,34 +27,31 @@ jobs:
   check-matrix:
     name: ${{ matrix.config.name }}
     runs-on: ubuntu-latest
+    container: ghcr.io/facebookincubator/velox-dev:check-avx
     strategy:
       fail-fast: false
       matrix:
         config:
           - { name: "License Header", 
               command: "header-fix",
               message: "Found missing License Header(s)",
-              reqs: "regex" 
             }
           - { name: "Code Format",
               command: "format-fix",
               message: "Found format issues",
-              reqs: "regex cmake-format black" 
             }
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Install Dependencies
-        run: |
-          python -m venv check_env
-          source check_env/bin/activate
-          pip install ${{ matrix.config.reqs }}
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory /__w/velox/velox
 
       - name: Check ${{ matrix.config.name }} 
         run: |
-          source check_env/bin/activate
           make ${{ matrix.config.command }}
 
           if ! git diff --quiet; then

.github/workflows/scheduled.yml

@@ -319,6 +319,13 @@ jobs:
           path: velox/_build/debug//velox/exec/tests/velox_row_number_fuzzer_test
           retention-days: "${{ env.RETENTION }}"
 
+      - name: Upload writer fuzzer
+        uses: actions/upload-artifact@v4
+        with:
+          name: writer
+          path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_writer_fuzzer_test
+          retention-days: "${{ env.RETENTION }}"
+
   presto-fuzzer-run:
     name: Presto Fuzzer
     if: ${{ needs.compile.outputs.presto_bias  != 'true' }}
@@ -892,3 +899,69 @@ jobs:
           path: |
             /tmp/window_fuzzer_repro
             /tmp/server.log
+
+  presto-java-writer-fuzzer-run:
+    name: Writer Fuzzer with Presto as source of truth
+    needs: compile
+    runs-on: ubuntu-latest
+    container: ghcr.io/facebookincubator/velox-dev:presto-java
+    timeout-minutes: 120
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/.ccache/"
+      LINUX_DISTRO: "centos"
+    steps:
+
+      - name: Download writer fuzzer
+        uses: actions/download-artifact@v4
+        with:
+          name: writer
+
+      - name: "Checkout Repo"
+        uses: actions/checkout@v4
+        with:
+          path: velox
+          submodules: 'recursive'
+          ref: "${{ inputs.ref }}"
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory /__w/velox/velox/velox
+
+
+      - name: "Run Writer Fuzzer"
+        run: |
+          cd  velox
+          cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
+          ls -lR $PRESTO_HOME/etc
+          echo "jvm config content:"
+          cat $PRESTO_HOME/etc/jvm.config
+          $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
+          ls -lR /var/log
+          # Sleep for 60 seconds to allow Presto server to start.
+          sleep 60
+          /opt/presto-cli --version
+          /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
+          cd -
+          mkdir -p /tmp/writer_fuzzer_repro/logs/
+          chmod -R 777 /tmp/writer_fuzzer_repro
+          chmod +x velox_writer_fuzzer_test
+          ./velox_writer_fuzzer_test \
+                --seed ${RANDOM} \
+                --duration_sec $DURATION \
+                --minloglevel=0 \
+                --stderrthreshold=2 \
+                --req_timeout_ms 60000 \
+                --log_dir=/tmp/writer_fuzzer_repro/logs \
+                --presto_url=http://127.0.0.1:8080 \
+          && echo -e "\n\Writer fuzzer run finished successfully."
+
+      - name: Archive writer production artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: presto-sot-writer-fuzzer-failure-artifacts
+          path: |
+            /tmp/writer_fuzzer_repro
+            /tmp/server.log
+            /var/log