ci: Add hdfs test case for s3 and azure (#4168)

apache · Feb 14, 2024 · eb661b2 · eb661b2
1 parent c27e35f
commit eb661b2
Show file tree

Hide file tree

Showing 4 changed files with 208 additions and 0 deletions.
diff --git a/.github/services/hdfs/hdfs_default_azblob/action.yml b/.github/services/hdfs/hdfs_default_azblob/action.yml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_default_azblob
+description: 'Behavior test for hdfs default over azure blob storage'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+    - name: Load secrets
+      uses: 1password/load-secrets-action@v1
+      with:
+        export-env: true
+      env:
+        # Need a environment variable with this name to securely encrypt them - otherwise there will be warning
+        # See https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html#Keystore_Passwords
+        HADOOP_CREDSTORE_PASSWORD: op://services/hdfs/credstore_password
+        # TODO1: Do we have this as a secret
+        # TODO2: Does this start with / and ends with /
+        OPENDAL_AZBLOB_ROOT: op://services/azblob/root
+        OPENDAL_AZBLOB_CONTAINER: op://services/azblob/container
+        OPENDAL_AZBLOB_ACCOUNT_NAME: op://services/azblob/account_name
+        OPENDAL_AZBLOB_ACCOUNT_KEY: op://services/azblob/account_key
+    - name: Setup
+      shell: bash
+      run: |
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+
+        export HADOOP_HOME="/home/runner/hadoop-3.3.5"
+
+        curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-azure-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.5/hadoop-azure-3.3.5.jar
+        curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/azure-storage-7.0.1.jar https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/7.0.1/azure-storage-7.0.1.jar
+
+        export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
+
+        mkdir -p /tmp/hdfs
+
+        pushd ${HADOOP_HOME}
+        ./bin/hadoop credential create fs.azure.account.key.${OPENDAL_AZBLOB_ACCOUNT_NAME}.blob.core.windows.net -value ${OPENDAL_AZBLOB_ACCOUNT_KEY} -provider localjceks://file/tmp/hdfs/wasb.jceks
+        popd
+
+        cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+        cp ./fixtures/hdfs/azblob-core-site.xml ${HADOOP_HOME}/etc/hadoop/core-site.xml
+
+        cat << EOF >> $GITHUB_ENV
+        HADOOP_HOME=${HADOOP_HOME}
+        CLASSPATH=${CLASSPATH}
+        LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
+        OPENDAL_HDFS_ROOT=${OPENDAL_AZBLOB_ROOT}
+        OPENDAL_HDFS_NAME_NODE=wasbs://${OPENDAL_AZBLOB_CONTAINER}
+        OPENDAL_HDFS_ENABLE_APPEND=false
+        EOF
diff --git a/.github/services/hdfs/hdfs_default_s3/action.yml b/.github/services/hdfs/hdfs_default_s3/action.yml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_default_s3
+description: 'Behavior test for hdfs default over aws s3'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup java env
+      uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "11"
+    - name: Load secrets
+      uses: 1password/load-secrets-action@v1
+      with:
+        export-env: true
+      env:
+        # Need a secret with this name to securely encrypt them - otherwise there will be warning
+        # See https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html#Keystore_Passwords
+        HADOOP_CREDSTORE_PASSWORD: op://services/hdfs/credstore_password
+        # TODO: Does this start with / and ends with /
+        OPENDAL_S3_ROOT: op://services/s3/root
+        OPENDAL_S3_BUCKET: op://services/s3/bucket
+        OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+        OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+    - name: Setup
+      shell: bash
+      run: |
+        curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+
+        export HADOOP_HOME="/home/runner/hadoop-3.3.5"
+
+        curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-aws-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.5/hadoop-aws-3.3.5.jar
+        curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/aws-java-sdk-bundle-1.12.653.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.653/aws-java-sdk-bundle-1.12.653.jar
+
+        export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
+
+        mkdir -p /tmp/hdfs
+
+        pushd ${HADOOP_HOME}
+        ./bin/hadoop credential create fs.s3a.access.key -value ${OPENDAL_S3_ACCESS_KEY_ID} -provider localjceks://file/tmp/hdfs/s3.jceks
+        ./bin/hadoop credential create fs.s3a.secret.key -value ${OPENDAL_S3_SECRET_ACCESS_KEY} -provider localjceks://file/tmp/hdfs/s3.jceks
+        popd
+
+        cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+        cp ./fixtures/hdfs/s3-core-site.xml ${HADOOP_HOME}/etc/hadoop/core-site.xml
+
+        cat << EOF >> $GITHUB_ENV
+        HADOOP_HOME=${HADOOP_HOME}
+        CLASSPATH=${CLASSPATH}
+        LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
+        OPENDAL_HDFS_ROOT=${OPENDAL_S3_ROOT}
+        OPENDAL_HDFS_NAME_NODE=s3a://${OPENDAL_S3_BUCKET}
+        OPENDAL_HDFS_ENABLE_APPEND=false
+        EOF
diff --git a/fixtures/hdfs/azblob-core-site.xml b/fixtures/hdfs/azblob-core-site.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>hadoop.security.credential.provider.path</name>
+    <value>localjceks://file/tmp/hdfs/wasb.jceks</value>
+    <description>Path to interrogate for protected azure blob storage credentials.</description>
+  </property>
+</configuration>
diff --git a/fixtures/hdfs/s3-core-site.xml b/fixtures/hdfs/s3-core-site.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>hadoop.security.credential.provider.path</name>
+    <value>localjceks://file/tmp/hdfs/s3.jceks</value>
+    <description>Path to interrogate for protected aws s3 storage credentials.</description>
+  </property>
+</configuration>