ci: Migrate hdfs default setup (#4140)

* ci: Migrate hdfs default setup Signed-off-by: Xuanwo <[email protected]> * Fix build Signed-off-by: Xuanwo <[email protected]> * try fix classpath Signed-off-by: Xuanwo <[email protected]> * Revert "try fix classpath" This reverts commit 7c7b85a. * trick for mvn Signed-off-by: Xuanwo <[email protected]> * try Signed-off-by: Xuanwo <[email protected]> * Don't run hdfs on java Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
apache · Feb 5, 2024 · 1b3c95f · 1b3c95f
1 parent 7d94296
commit 1b3c95f
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 36 deletions.
diff --git a/.github/scripts/test_behavior/plan.py b/.github/scripts/test_behavior/plan.py
@@ -198,6 +198,10 @@ def generate_language_binding_cases(
 ) -> list[dict[str, str]]:
     cases = unique_cases(cases)
 
+    # Remove hdfs cases for java.
+    if language == "java":
+        cases = [v for v in cases if v["service"] != "hdfs"]
+
     if os.getenv("GITHUB_IS_PUSH") == "true":
         return cases
 

diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_default
+description: 'Behavior test for hdfs default'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+    - name: Setup
+      shell: bash
+      run: |
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+
+        export HADOOP_HOME="/home/runner/hadoop-3.3.5"
+        export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
+
+        cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+
+        cat << EOF >> $GITHUB_ENV
+        HADOOP_HOME=${HADOOP_HOME}
+        CLASSPATH=${CLASSPATH}
+        LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
+        OPENDAL_HDFS_ROOT=/tmp/opendal/
+        OPENDAL_HDFS_NAME_NODE=default
+        OPENDAL_HDFS_ENABLE_APPEND=false
+        EOF
diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml
@@ -37,42 +37,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  hdfs-default:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup
-        with:
-          need-nextest: true
-
-      - name: Setup java env
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: "11"
-      - name: Setup hadoop env
-        shell: bash
-        run: |
-          curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
-
-      - name: Test
-        shell: bash
-        working-directory: core
-        run: |
-          export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
-          export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native
-          cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${{ env.HADOOP_HOME }}/etc/hadoop/hdfs-site.xml
-
-          cargo test behavior --features tests,services-hdfs
-        env:
-          HADOOP_HOME: "/home/runner/hadoop-3.3.5"
-          OPENDAL_TEST: hdfs
-          OPENDAL_HDFS_ROOT: /tmp/opendal/
-          OPENDAL_HDFS_NAME_NODE: default
-          OPENDAL_HDFS_ENABLE_APPEND: false
-
   hdfs-cluster:
     runs-on: ubuntu-latest
     steps: