Merge remote-tracking branch 'origin/main' into deleter

Signed-off-by: Xuanwo <[email protected]>
apache · Dec 5, 2024 · bd58401 · bd58401
2 parents f3a1b0e + f3bf1d4
commit bd58401
Show file tree

Hide file tree

Showing 177 changed files with 2,490 additions and 894 deletions.
diff --git a/.github/actions/setup-hadoop/action.yaml b/.github/actions/setup-hadoop/action.yaml
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Hadoop
+description: 'Prepare hadoop binary and env'
+
+inputs:
+  need-hadoop:
+    description: "This setup needs hadoop or not"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+
+    - name: Cache hadoop
+      id: cache-hadoop
+      uses: actions/cache@v4
+      if: inputs.need-hadoop == 'true'
+      with:
+        path: /home/runner/hadoop-3.3.5
+        key: cache-hadoop-3.3.5
+
+    - name: Build hadoop if not cached
+      if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true'
+      shell: bash
+      run: |
+        set -e
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+
+    - name: Setup hadoop env
+      shell: bash
+      run: |
+        export HADOOP_HOME=/home/runner/hadoop-3.3.5
+        echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV
+        echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV 
+        echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV
+        cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+
+
+
+
diff --git a/.github/scripts/test_go_binding/matrix.yaml b/.github/scripts/test_go_binding/matrix.yaml
@@ -15,12 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
-os: ["ubuntu-latest"]
 build:
   - target: "x86_64-unknown-linux-gnu"
     cc: "gcc"
     goos: "linux"
     goarch: "amd64"
+    os: "ubuntu-latest"
+  - target: "aarch64-apple-darwin"
+    cc: "clang"
+    goos: "darwin"
+    goarch: "arm64"
+    os: "macos-latest"
 service:
   - "fs"
 
diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml
@@ -25,20 +25,10 @@ runs:
       shell: bash
       working-directory: fixtures/hdfs
       run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait
-    - name: Setup java env
-      uses: actions/setup-java@v4
+    - name: Setup hadoop
+      uses: ./.github/actions/setup-hadoop
       with:
-        distribution: temurin
-        java-version: "11"
-    - name: Setup hadoop env
-      shell: bash
-      run: |
-        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
-        export HADOOP_HOME=/home/runner/hadoop-3.3.5
-        echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV
-        echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV 
-        echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV
-        cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+        need-hadoop: true
     - name: Setup opendal env
       shell: bash
       run: |

diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
@@ -25,20 +25,10 @@ runs:
       shell: bash
       working-directory: fixtures/hdfs
       run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait
-    - name: Setup java env
-      uses: actions/setup-java@v4
+    - name: Setup hadoop
+      uses: ./.github/actions/setup-hadoop
       with:
-        distribution: temurin
-        java-version: "11"
-    - name: Setup hadoop env
-      shell: bash
-      run: |
-        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
-        export HADOOP_HOME=/home/runner/hadoop-3.3.5
-        echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV
-        echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV 
-        echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV
-        cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+        need-hadoop: true
     - name: Setup opendal env
       shell: bash
       run: |

diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml
@@ -21,25 +21,14 @@ description: 'Behavior test for hdfs default'
 runs:
   using: "composite"
   steps:
-    - name: Setup java env
-      uses: actions/setup-java@v4
+    - name: Setup hadoop
+      uses: ./.github/actions/setup-hadoop
       with:
-        distribution: temurin
-        java-version: "11"
-    - name: Setup
+        need-hadoop: true
+    - name: Setup OpenDAL env
       shell: bash
       run: |
-        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
-
-        export HADOOP_HOME="/home/runner/hadoop-3.3.5"
-        export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
-
-        cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
-
         cat << EOF >> $GITHUB_ENV
-        HADOOP_HOME=${HADOOP_HOME}
-        CLASSPATH=${CLASSPATH}
-        LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
         OPENDAL_HDFS_ROOT=/tmp/opendal/
         OPENDAL_HDFS_NAME_NODE=default
         OPENDAL_HDFS_ENABLE_APPEND=false

diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
@@ -21,25 +21,14 @@ description: 'Behavior test for hdfs default with atomic write dir'
 runs:
   using: "composite"
   steps:
-    - name: Setup java env
-      uses: actions/setup-java@v4
+    - name: Setup hadoop
+      uses: ./.github/actions/setup-hadoop
       with:
-        distribution: temurin
-        java-version: "11"
-    - name: Setup
+        need-hadoop: true
+    - name: Setup OpenDAL env
       shell: bash
       run: |
-        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
-
-        export HADOOP_HOME="/home/runner/hadoop-3.3.5"
-        export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
-
-        cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
-
         cat << EOF >> $GITHUB_ENV
-        HADOOP_HOME=${HADOOP_HOME}
-        CLASSPATH=${CLASSPATH}
-        LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
         OPENDAL_HDFS_ROOT=/tmp/opendal/
         OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
         OPENDAL_HDFS_NAME_NODE=default

diff --git a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml
@@ -44,4 +44,5 @@ runs:
         OPENDAL_S3_SECRET_ACCESS_KEY=demo
         OPENDAL_S3_REGION=us-east-1
         OPENDAL_S3_ENABLE_VERSIONING=true
+        OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on
         EOF
diff --git a/.github/services/s3/ceph_rados_s3/action.yml b/.github/services/s3/ceph_rados_s3/action.yml
@@ -41,4 +41,5 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID=demo
         OPENDAL_S3_SECRET_ACCESS_KEY=demo
         OPENDAL_S3_REGION=us-east-1
+        OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on
         EOF
diff --git a/.github/workflows/ci_bin_oli.yml b/.github/workflows/ci_bin_oli.yml
@@ -25,7 +25,7 @@ on:
     branches:
       - main
     paths:
-      - "bin/oay/**"
+      - "bin/oli/**"
       - "core/**"
       - ".github/workflows/ci_bin_oli.yml"
 

diff --git a/.github/workflows/ci_bindings_cpp.yml b/.github/workflows/ci_bindings_cpp.yml
@@ -41,13 +41,13 @@ permissions:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install libgtest-dev ninja-build libboost-all-dev valgrind doxygen
+          sudo apt-get install ninja-build valgrind doxygen
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup
@@ -69,3 +69,12 @@ jobs:
           cmake -GNinja -DOPENDAL_ENABLE_TESTING=ON ..
           ninja
           valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose ./opendal_cpp_test
+
+      - name: Build Cpp binding with async && Run tests
+        working-directory: "bindings/cpp"
+        run: |
+          mkdir build-async
+          cd build-async
+          cmake -GNinja -DOPENDAL_DEV=ON -DOPENDAL_ENABLE_ASYNC=ON -DCMAKE_CXX_COMPILER=clang++-18 ..
+          ninja
+          ./opendal_cpp_test
diff --git a/.github/workflows/ci_bindings_go.yml b/.github/workflows/ci_bindings_go.yml
@@ -70,7 +70,7 @@ jobs:
           echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
   test:
     needs: [ matrix ]
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.build.os }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
@@ -98,23 +98,34 @@ jobs:
         run: |
           python -m pip install toml
           python tools/.github/scripts/setup_features.py
+      - name: Install dependencies (Linux)
+        if: ${{ matrix.build.os == 'ubuntu-latest' }}
+        run: sudo apt install zstd
+      - name: Install dependencies (macOS)
+        if: ${{ matrix.build.os == 'macos-latest' }}
+        run: brew install zstd libffi
       - name: Build C Binding
         working-directory: bindings/c
         env:
           VERSION: "latest"
           SERVICE: ${{ matrix.service }}
           TARGET: ${{ matrix.build.target }}
           CC: ${{ matrix.build.cc }}
+          OS: ${{ matrix.build.os }}
         run: |
           cargo build --target $TARGET  --release
-          sudo apt install zstd
           DIR=$GITHUB_WORKSPACE/libopendal_c_${VERSION}_${SERVICE}_$TARGET
           mkdir $DIR
-          zstd -22 ./target/$TARGET/release/libopendal_c.so -o $DIR/libopendal_c.$TARGET.so.zst
+          if [ ${OS} == 'ubuntu-latest' ]; then
+            SO=so
+          else
+            SO=dylib
+          fi
+          zstd -19 ./target/$TARGET/release/libopendal_c.$SO -o $DIR/libopendal_c.$TARGET.$SO.zst
       - name: Build Go Artifact
         working-directory: tools/internal/generate
         env:
-          MATRIX: ${{ needs.matrix.outputs.matrix }}
+          MATRIX: '{"build": [${{ toJson(matrix.build) }}], "service": ["${{ matrix.service }}"]}'
           VERSION: "latest"
         run: |
           go run generate.go
@@ -132,4 +143,8 @@ jobs:
           OPENDAL_TEST: ${{ matrix.service }}
           OPENDAL_FS_ROOT: "/tmp/opendal/"
         working-directory: bindings/go/tests/behavior_tests
-        run: CGO_ENABLE=0 go test -v -run TestBehavior
+        run: |
+          if [ ${{ matrix.build.os }} == 'macos-latest' ]; then
+            export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/opt/libffi/lib
+          fi
+          CGO_ENABLE=0 go test -v -run TestBehavior
diff --git a/.github/workflows/ci_bindings_nodejs.yml b/.github/workflows/ci_bindings_nodejs.yml
@@ -153,7 +153,7 @@ jobs:
         shell: bash
         working-directory: .
         run: ${{ matrix.settings.build }}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: bindings-linux-${{ matrix.settings.target }}
           path: bindings/nodejs/*.node
@@ -201,7 +201,7 @@ jobs:
       - name: Build
         shell: bash
         run: ${{ matrix.settings.build }}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: bindings-windows-${{ matrix.settings.target }}
           path: bindings/nodejs/*.node
@@ -223,7 +223,7 @@ jobs:
             build: |
               rustup target add x86_64-apple-darwin;
               export NAPI_TARGET=x86_64-apple-darwin;
-              
+
               pnpm build
               strip -x *.node
           - target: aarch64-apple-darwin
@@ -291,7 +291,7 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: bindings/nodejs/artifacts
       - name: Move artifacts