refactor(ci/behavior_test): Migrate to 1password instead (#3338)

* Save work Signed-off-by: Xuanwo <[email protected]> * Renaming Signed-off-by: Xuanwo <[email protected]> * Refactor python Signed-off-by: Xuanwo <[email protected]> * refactor Signed-off-by: Xuanwo <[email protected]> * Refactor Signed-off-by: Xuanwo <[email protected]> * Fix Signed-off-by: Xuanwo <[email protected]> * remove debug log Signed-off-by: Xuanwo <[email protected]> * Fix file changed Signed-off-by: Xuanwo <[email protected]> * compare with pr base Signed-off-by: Xuanwo <[email protected]> * Migrate Signed-off-by: Xuanwo <[email protected]> * migrate existing s3 tests Signed-off-by: Xuanwo <[email protected]> * fix secrets Signed-off-by: Xuanwo <[email protected]> * Fix typo Signed-off-by: Xuanwo <[email protected]> * Fix endpoint Signed-off-by: Xuanwo <[email protected]> * Test directly Signed-off-by: Xuanwo <[email protected]> * Fix plan Signed-off-by: Xuanwo <[email protected]> * remove needs Signed-off-by: Xuanwo <[email protected]> * Remove features Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
apache · Oct 18, 2023 · 0631953 · 0631953
1 parent 625b7f7
commit 0631953
Show file tree

Hide file tree

Showing 15 changed files with 334 additions and 232 deletions.
diff --git a/.github/actions/test-core/action.yaml → ...ub/actions/behavior_test_core/action.yaml b/.github/actions/test-core/action.yaml → ...ub/actions/behavior_test_core/action.yaml
@@ -22,9 +22,8 @@ inputs:
     description: "The setup action for test"
   service:
     description: "The service to test"
-  secrets:
-    description: "The secrets for test"
-    default: "{}"
+  feature:
+    description: "The feature to test"
 
 runs:
   using: "composite"
@@ -39,12 +38,10 @@ runs:
               steps:
               - name: Setup Test Core
                 uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
-                with:
-                  secrets: ${{ toJson(inputs.secrets) }}
               - name: Run Test Core
                 shell: bash
                 working-directory: core
-                run: cargo nextest run behavior --archive-file=./core-test.tar.zst
+                run: cargo nextest run behavior --features ${{ inputs.feature }}
                 env:
                   OPENDAL_TEST: ${{ inputs.service }}
             EOF

diff --git a/.github/scripts/behavior_test/__init__.py b/.github/scripts/behavior_test/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/scripts/workflow_planner.py → .github/scripts/behavior_test/plan.py b/scripts/workflow_planner.py → .github/scripts/behavior_test/plan.py
@@ -21,9 +21,16 @@
 import os
 from pathlib import Path
 
+# The path for current script.
+SCRIPT_PATH = Path(__file__).parent.absolute()
+# The path for `.github` dir.
+GITHUB_DIR = SCRIPT_PATH.parent.parent
+# The project dir for opendal.
+PROJECT_DIR = GITHUB_DIR.parent
+
 
 def get_provided_cases():
-    root_dir = ".github/services"
+    root_dir = f"{GITHUB_DIR}/services"
 
     cases = [
         {
@@ -62,7 +69,9 @@ def calculate_core_cases(cases, changed_files):
 
     # If any of the core files changed, we will run all cases.
     if any(
-        p.startswith("core/src/") and not p.startswith("core/src/services")
+        p.startswith("core/")
+        and not p.startswith("core/src/services/")
+        and not p.endswith(".md")
         for p in changed_files
     ):
         return cases
@@ -89,11 +98,13 @@ def plan(changed_files):
         jobs["core"] = [
             {
                 "os": "ubuntu-latest",
-                "features": ",".join(
-                    set([f"services-{v['service']}" for v in core_cases])
-                ),
                 "cases": [
-                    {"setup": v["setup"], "service": v["service"]} for v in core_cases
+                    {
+                        "setup": v["setup"],
+                        "service": v["service"],
+                        "feature": "services-{}".format(v["service"].replace("_", "-")),
+                    }
+                    for v in core_cases
                 ],
             },
         ]
@@ -103,12 +114,13 @@ def plan(changed_files):
             jobs["core"].append(
                 {
                     "os": "windows-latest",
-                    "features": "services-fs",
-                    "cases": [{"setup": "local-fs", "service": "fs"}],
+                    "cases": [
+                        {"setup": "local_fs", "service": "fs", "feature": "services-fs"}
+                    ],
                 }
             )
 
-    return json.dumps(jobs)
+    return jobs
 
 
 # For quick test:
@@ -117,4 +129,4 @@ def plan(changed_files):
 if __name__ == "__main__":
     changed_files = sys.argv[1:]
     result = plan(changed_files)
-    print(result)
+    print(json.dumps(result))
diff --git a/.github/scripts/behavior_test/test_plan.py b/.github/scripts/behavior_test/test_plan.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from plan import plan
+
+
+class BehaviorTestPlan(unittest.TestCase):
+    def test_empty(self):
+        result = plan([])
+        self.assertEqual(result, {})
+
+    def test_core_cargo_toml(self):
+        result = plan(["core/Cargo.toml"])
+        self.assertTrue(result["components"]["core"])
+
+    def test_core_services_fs(self):
+        result = plan(["core/src/services/fs/mod.rs"])
+        self.assertTrue(result["components"]["core"])
+        self.assertTrue(len(result["core"]) > 0)
+
+        cases = [v["service"] for v in result["core"][0]["cases"]]
+        # Should not contain fs
+        self.assertTrue("fs" in cases)
+        # Should not contain s3
+        self.assertFalse("s3" in cases)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/.github/services/fs/local-fs/action.yml → .github/services/fs/local_fs/action.yml b/.github/services/fs/local-fs/action.yml → .github/services/fs/local_fs/action.yml
@@ -16,7 +16,7 @@
 # under the License.
 
 name: local_fs
-description: "Setup local fs"
+description: 'Behavior test for local fs'
 
 runs:
     using: "composite"

diff --git a/.github/services/s3/aws_s3/action.yml b/.github/services/s3/aws_s3/action.yml
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: aws_s3
+description: 'Behavior test for AWS S3. This service is sponsored by @datafuse_labs.'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup
+      uses: 1password/load-secrets-action@v1
+      with:
+        export-env: true
+      env:
+        OPENDAL_S3_ROOT: op://services/s3/root
+        OPENDAL_S3_BUCKET: op://services/s3/bucket
+        OPENDAL_S3_ENDPOINT: op://services/s3/endpoint
+        OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+        OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+        OPENDAL_S3_REGION: op://services/s3/region
diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: aws_s3_with_sse_c
+description: 'Behavior test for AWS S3 with SSE-C. This service is sponsored by @datafuse_labs.'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup
+      uses: 1password/load-secrets-action@v1
+      with:
+        export-env: true
+      env:
+        OPENDAL_S3_ROOT: op://services/s3/root
+        OPENDAL_S3_BUCKET: op://services/s3/bucket
+        OPENDAL_S3_ENDPOINT: op://services/s3/endpoint
+        OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+        OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+        OPENDAL_S3_REGION: op://services/s3/region
+
+    - name: Add extra settings
+      shell: bash
+      run: |
+        cat << EOF >> $GITHUB_ENV
+        OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256
+        OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=
+        OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw==
+        EOF
diff --git a/.github/services/s3/aws-s3/action.yml → ...es/s3/aws_s3_with_virtual_host/action.yml b/.github/services/s3/aws-s3/action.yml → ...es/s3/aws_s3_with_virtual_host/action.yml
@@ -15,24 +15,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: aws_s3
-description: "Setup AWS S3"
+name: aws_s3_with_virtual_host
+description: 'Behavior test for AWS S3 with virtual host. This service is sponsored by @datafuse_labs.'
 
-inputs:
-  secrets:
-    description: "The secrets for test"
-    default: "{}"
-
-# TODO: we should migrate to 1password instead.
 runs:
   using: "composite"
   steps:
     - name: Setup
+      uses: 1password/load-secrets-action@v1
+      with:
+        export-env: true
+      env:
+        OPENDAL_S3_ROOT: op://services/s3/root
+        OPENDAL_S3_BUCKET: op://services/s3/bucket
+        OPENDAL_S3_ENDPOINT: op://services/s3/endpoint
+        OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+        OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+        OPENDAL_S3_REGION: op://services/s3/region
+
+    - name: Add extra settings
       shell: bash
       run: |
-        echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV
-        echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV
-        echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV
-        echo "OPENDAL_S3_ACCESS_KEY_ID=${{ fromJson(inputs.secrets).OPENDAL_S3_ACCESS_KEY_ID }}" >> $GITHUB_ENV
-        echo "OPENDAL_S3_SECRET_ACCESS_KEY=${{ fromJson(inputs.secrets).OPENDAL_S3_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
-        echo "OPENDAL_S3_REGION=ap-northeast-1" >> $GITHUB_ENV
+        echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV
diff --git a/.github/services/s3/minio_s3/action.yml b/.github/services/s3/minio_s3/action.yml
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: minio_s3
+description: 'Behavior test for Minio S3.'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup MinIO Server
+      shell: bash
+      working-directory: fixtures/s3
+      run: docker-compose -f docker-compose-minio.yml up -d
+    - name: Setup test bucket
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: "minioadmin"
+        AWS_SECRET_ACCESS_KEY: "minioadmin"
+        AWS_EC2_METADATA_DISABLED: "true"
+      run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
+    - name: Setup
+      shell: bash
+      run: |
+        cat << EOF >> $GITHUB_ENV
+        OPENDAL_S3_BUCKET=test
+        OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
+        OPENDAL_S3_ACCESS_KEY_ID=minioadmin
+        OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
+        OPENDAL_S3_REGION=us-east-1
+        EOF
diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: minio_s3_with_anonymous
+description: 'Behavior test for Minio S3 with anonymous access.'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup MinIO Server
+      shell: bash
+      working-directory: fixtures/s3
+      run: |
+        docker-compose -f docker-compose-minio.yml up -d
+    - name: Setup test bucket
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: "minioadmin"
+        AWS_SECRET_ACCESS_KEY: "minioadmin"
+        AWS_EC2_METADATA_DISABLED: "true"
+      run: |
+        aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
+
+        curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
+        chmod +x mc
+        ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin
+        ./mc anonymous set public local/test
+    - name: Setup
+      shell: bash
+      run: |
+        cat << EOF >> $GITHUB_ENV
+        OPENDAL_S3_BUCKET=test
+        OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
+        OPENDAL_S3_REGION=us-east-1
+        OPENDAL_S3_ALLOW_ANONYMOUS=on
+        EOF