From c0e9640ed4a6083dc03c815bfb2e882097b0c81b Mon Sep 17 00:00:00 2001
From: Nadeshiko Manju <me@manjusaka.me>
Date: Mon, 30 Oct 2023 23:44:35 +0800
Subject: [PATCH] feat(test): Enable new test workflow planner for python
 binding (#3397)

* feat(test): Enable new test workflow planner for python binding

Signed-off-by: Manjusaka <me@manjusaka.me>

* Update code

Signed-off-by: Manjusaka <me@manjusaka.me>

* Update code

Signed-off-by: Manjusaka <me@manjusaka.me>

* Update code

Signed-off-by: Manjusaka <me@manjusaka.me>

* Update code

Signed-off-by: Manjusaka <me@manjusaka.me>

* update code

Signed-off-by: Manjusaka <me@manjusaka.me>

* Update code

Signed-off-by: Manjusaka <me@manjusaka.me>

* update code

Signed-off-by: Manjusaka <me@manjusaka.me>

---------

Signed-off-by: Manjusaka <me@manjusaka.me>
---
 .../behavior_test_binding_java/action.yaml    |   2 +-
 .../behavior_test_binding_python/action.yaml  |  51 ++++
 .github/scripts/behavior_test/plan.py         |  51 +++-
 .github/services/mysql/mysql/action.yml       |   8 +-
 .github/workflows/behavior_test.yml           |  12 +
 .../behavior_test_binding_python.yml          |  71 ++++++
 .github/workflows/bindings_python.yml         |  19 --
 .github/workflows/service_test_memory.yml     |  61 -----
 .github/workflows/service_test_s3.yml         |  42 +---
 bindings/python/Cargo.toml                    | 100 ++++++++
 bindings/python/tests/conftest.py             |  60 +++++
 bindings/python/tests/test_capability.py      |  31 +++
 bindings/python/tests/test_read.py            |  97 ++++++++
 bindings/python/tests/test_services.py        | 224 ------------------
 bindings/python/tests/test_write.py           | 115 +++++++++
 15 files changed, 592 insertions(+), 352 deletions(-)
 create mode 100644 .github/actions/behavior_test_binding_python/action.yaml
 create mode 100644 .github/workflows/behavior_test_binding_python.yml
 delete mode 100644 .github/workflows/service_test_memory.yml
 create mode 100644 bindings/python/tests/test_capability.py
 create mode 100644 bindings/python/tests/test_read.py
 delete mode 100644 bindings/python/tests/test_services.py
 create mode 100644 bindings/python/tests/test_write.py

diff --git a/.github/actions/behavior_test_binding_java/action.yaml b/.github/actions/behavior_test_binding_java/action.yaml
index cb3724a81d2f..d9252b1c324f 100644
--- a/.github/actions/behavior_test_binding_java/action.yaml
+++ b/.github/actions/behavior_test_binding_java/action.yaml
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Test Core
+name: Test Binding Java
 description: 'Test Core with given setup and service'
 inputs:
   setup:
diff --git a/.github/actions/behavior_test_binding_python/action.yaml b/.github/actions/behavior_test_binding_python/action.yaml
new file mode 100644
index 000000000000..72437b08fb34
--- /dev/null
+++ b/.github/actions/behavior_test_binding_python/action.yaml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Test Binding Python
+description: 'Test Core with given setup and service'
+inputs:
+  setup:
+    description: "The setup action for test"
+  service:
+    description: "The service to test"
+  feature:
+    description: "The feature to test"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup
+      shell: bash
+      run: |
+        mkdir -p ./dynamic_test_binding_python &&
+        cat <<EOF >./dynamic_test_binding_python/action.yml
+        runs:
+          using: composite
+          steps:
+          - name: Setup Test
+            uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
+          - name: Run Test Binding Python
+            shell: bash
+            working-directory: bindings/python
+            run: |
+              source venv/bin/activate
+              pytest -v tests
+            env:
+              OPENDAL_TEST: ${{ inputs.service }}
+        EOF
+    - name: Run
+      uses: ./dynamic_test_binding_python
diff --git a/.github/scripts/behavior_test/plan.py b/.github/scripts/behavior_test/plan.py
index 69397f67f817..3e0e5bdd66fd 100755
--- a/.github/scripts/behavior_test/plan.py
+++ b/.github/scripts/behavior_test/plan.py
@@ -22,6 +22,7 @@
 import re
 from pathlib import Path
 from dataclasses import dataclass, field
+from typing import Any
 
 # The path for current script.
 SCRIPT_PATH = Path(__file__).parent.absolute()
@@ -31,7 +32,7 @@
 PROJECT_DIR = GITHUB_DIR.parent
 
 
-def provided_cases():
+def provided_cases() -> list[dict[str, str]]:
     root_dir = f"{GITHUB_DIR}/services"
 
     cases = [
@@ -74,6 +75,8 @@ class Hint:
     core: bool = field(default=False, init=False)
     # Is binding java affected?
     binding_java: bool = field(default=False, init=False)
+    # Is binding python affected?
+    binding_python: bool = field(default=False, init=False)
 
     # Should we run all services test?
     all_service: bool = field(default=False, init=False)
@@ -81,7 +84,7 @@ class Hint:
     services: set = field(default_factory=set, init=False)
 
 
-def calculate_hint(changed_files):
+def calculate_hint(changed_files: list[str]) -> Hint:
     hint = Hint()
 
     # Remove all files that ends with `.md`
@@ -95,12 +98,14 @@ def calculate_hint(changed_files):
         if p == ".github/workflows/behavior_test.yml":
             hint.core = True
             hint.binding_java = True
+            hint.binding_python = True
             hint.all_service = True
         if p == ".github/workflows/behavior_test_core.yml":
             hint.core = True
             hint.all_service = True
         if p == ".github/workflows/behavior_test_binding_java.yml":
             hint.binding_java = True
+            hint.binding_python = True
             hint.all_service = True
 
         # core affected
@@ -113,11 +118,13 @@ def calculate_hint(changed_files):
         ):
             hint.core = True
             hint.binding_java = True
+            hint.binding_python = True
             hint.all_service = True
 
         # binding java affected.
         if p.startswith("bindings/java/"):
             hint.binding_java = True
+            hint.binding_python = True
             hint.all_service = True
 
         # core service affected
@@ -125,6 +132,7 @@ def calculate_hint(changed_files):
         if match:
             hint.core = True
             hint.binding_java = True
+            hint.binding_python = True
             hint.services.add(match.group(1))
 
         # core test affected
@@ -132,6 +140,7 @@ def calculate_hint(changed_files):
         if match:
             hint.core = True
             hint.binding_java = True
+            hint.binding_python = True
             hint.services.add(match.group(1))
 
     return hint
@@ -154,7 +163,9 @@ def unique_cases(cases):
     return list(ucases.values())
 
 
-def generate_core_cases(cases, hint):
+def generate_core_cases(
+    cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
     # Always run all tests if it is a push event.
     if os.getenv("GITHUB_IS_PUSH") == "true":
         return cases
@@ -172,7 +183,9 @@ def generate_core_cases(cases, hint):
     return cases
 
 
-def generate_binding_java_cases(cases, hint):
+def generate_binding_java_cases(
+    cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
     cases = unique_cases(cases)
 
     # Always run all tests if it is a push event.
@@ -192,20 +205,43 @@ def generate_binding_java_cases(cases, hint):
     return cases
 
 
-def plan(changed_files):
+def generate_binding_python_cases(
+    cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
+    cases = unique_cases(cases)
+    if os.getenv("GITHUB_IS_PUSH") == "true":
+        return cases
+
+    # Return empty if core is False
+    if not hint.binding_python:
+        return []
+
+    # Return all services if all_service is True
+    if hint.all_service:
+        return cases
+
+    # Filter all cases that not shown un in changed files
+    cases = [v for v in cases if v["service"] in hint.services]
+    return cases
+
+
+def plan(changed_files: list[str]) -> dict[str, Any]:
     cases = provided_cases()
     hint = calculate_hint(changed_files)
 
     core_cases = generate_core_cases(cases, hint)
     binding_java_cases = generate_binding_java_cases(cases, hint)
+    binding_python_cases = generate_binding_python_cases(cases, hint)
 
     jobs = {
         "components": {
             "core": False,
             "binding_java": False,
+            "binding_python": False,
         },
         "core": [],
         "binding_java": [],
+        "binding_python": [],
     }
 
     if len(core_cases) > 0:
@@ -228,6 +264,11 @@ def plan(changed_files):
         jobs["binding_java"].append(
             {"os": "ubuntu-latest", "cases": binding_java_cases}
         )
+    if len(binding_python_cases) > 0:
+        jobs["components"]["binding_python"] = True
+        jobs["binding_python"].append(
+            {"os": "ubuntu-latest", "cases": binding_python_cases}
+        )
 
     return jobs
 
diff --git a/.github/services/mysql/mysql/action.yml b/.github/services/mysql/mysql/action.yml
index 2f2b332e5ee9..a41e7d2be761 100644
--- a/.github/services/mysql/mysql/action.yml
+++ b/.github/services/mysql/mysql/action.yml
@@ -24,7 +24,13 @@ runs:
     - name: Setup MySQL Server
       shell: bash
       working-directory: fixtures/mysql
-      run: docker-compose -f docker-compose.yml up -d
+      run: |
+        apt update && apt install -y mysql-client
+        docker-compose -f docker-compose.yml up -d
+        while ! mysql -h localhost -P 3306 --protocol=tcp -u root -p'password' -e "SELECT 1"; do
+            echo "Waiting for MySQL..."
+            sleep 1
+        done
     - name: Setup
       shell: bash
       run: |
diff --git a/.github/workflows/behavior_test.yml b/.github/workflows/behavior_test.yml
index 364eba63b67a..861a7f981900 100644
--- a/.github/workflows/behavior_test.yml
+++ b/.github/workflows/behavior_test.yml
@@ -102,3 +102,15 @@ jobs:
     with:
       os: ${{ matrix.os }}
       cases: ${{ toJson(matrix.cases) }}
+  test_binding_python:
+    name: binding_python / ${{ matrix.os }}
+    needs: [plan]
+    if: fromJson(needs.plan.outputs.plan).components.binding_python
+    secrets: inherit
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.plan.outputs.plan).binding_python }}
+    uses: ./.github/workflows/behavior_test_binding_python.yml
+    with:
+      os: ${{ matrix.os }}
+      cases: ${{ toJson(matrix.cases) }}
diff --git a/.github/workflows/behavior_test_binding_python.yml b/.github/workflows/behavior_test_binding_python.yml
new file mode 100644
index 000000000000..ed6b099d20f2
--- /dev/null
+++ b/.github/workflows/behavior_test_binding_python.yml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Behavior Test Binding Python
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      cases:
+        required: true
+        type: string
+
+jobs:
+  test:
+    name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
+    runs-on: ${{ inputs.os }}
+    strategy:
+      matrix:
+        cases: ${{ fromJson(inputs.cases) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          need-nextest: true
+          need-protoc: true
+          need-rocksdb: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # TODO: 1password is only supported on linux
+      #
+      # Waiting for https://github.com/1Password/load-secrets-action/issues/46
+      - name: Setup service account token for 1password
+        if: runner.os == 'Linux'
+        shell: bash
+        run: echo "OP_SERVICE_ACCOUNT_TOKEN=${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" >> $GITHUB_ENV
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Build with maturin
+        shell: bash
+        working-directory: "bindings/python"
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install maturin[patchelf]
+          maturin develop -E test -F ${{ matrix.cases.feature }}
+
+      - name: Test Core
+        uses: ./.github/actions/behavior_test_binding_python
+        with:
+          setup: ${{ matrix.cases.setup }}
+          service: ${{ matrix.cases.service }}
+          feature: ${{ matrix.cases.feature }}
diff --git a/.github/workflows/bindings_python.yml b/.github/workflows/bindings_python.yml
index 56bfbea68ea4..d3ba0bcd2616 100644
--- a/.github/workflows/bindings_python.yml
+++ b/.github/workflows/bindings_python.yml
@@ -39,25 +39,6 @@ permissions:
   contents: read
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup
-      - name: Build with maturin
-        working-directory: "bindings/python"
-        run: |
-          python -m pip install -e .[test]
-      - name: Run pytest
-        working-directory: "bindings/python"
-        env:
-          OPENDAL_MEMORY_TEST: on
-        run: |
-          pytest -vk TestMemory
 
   sdist:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/service_test_memory.yml b/.github/workflows/service_test_memory.yml
deleted file mode 100644
index 3277a4a67799..000000000000
--- a/.github/workflows/service_test_memory.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Service Test Memory
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "core/src/**"
-      - "core/tests/**"
-      - "!core/src/docs/**"
-      - "!core/src/services/**"
-      - "core/src/services/memory/**"
-      - ".github/workflows/service_test_memory.yml"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-jobs:
-  python:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup
-
-      - name: Build with maturin
-        working-directory: "bindings/python"
-        run: |
-          python -m pip install -e .[test]
-
-      - name: Test
-        shell: bash
-        working-directory: bindings/python
-        run: pytest -vk TestMemory
-        env:
-          OPENDAL_MEMORY_TEST: on
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
index ed2f5a387165..1a255585a1c7 100644
--- a/.github/workflows/service_test_s3.yml
+++ b/.github/workflows/service_test_s3.yml
@@ -69,44 +69,4 @@ jobs:
           OPENDAL_S3_ROOT: CI/
           OPENDAL_S3_BUCKET: opendal-testing
           OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
-          OPENDAL_S3_REGION: ap-northeast-1
-
-  python:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: Setup MinIO Server
-        shell: bash
-        working-directory: fixtures/s3
-        run: docker-compose -f docker-compose-minio.yml up -d
-
-      - name: Setup test bucket
-        env:
-          AWS_ACCESS_KEY_ID: "minioadmin"
-          AWS_SECRET_ACCESS_KEY: "minioadmin"
-          AWS_EC2_METADATA_DISABLED: "true"
-        run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
-
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup
-
-      - name: Build with maturin
-        working-directory: "bindings/python"
-        run: |
-          python -m pip install -e .[test]
-
-      - name: Test
-        shell: bash
-        working-directory: bindings/python
-        run: pytest -vk TestS3
-        env:
-          OPENDAL_S3_TEST: on
-          OPENDAL_S3_BUCKET: test
-          OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
-          OPENDAL_S3_ACCESS_KEY_ID: minioadmin
-          OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
-          OPENDAL_S3_REGION: us-east-1
+          OPENDAL_S3_REGION: ap-northeast-1
\ No newline at end of file
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index 4bf502e8e187..a2ef50b5bf81 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -27,6 +27,106 @@ repository.workspace = true
 rust-version.workspace = true
 version.workspace = true
 
+
+[features]
+# Enable all opendal default feature by default.
+default = [
+  "services-azblob",
+  "services-azdls",
+  "services-cos",
+  "services-fs",
+  "services-gcs",
+  "services-ghac",
+  "services-http",
+  "services-ipmfs",
+  "services-memory",
+  "services-obs",
+  "services-oss",
+  "services-s3",
+  "services-webdav",
+  "services-webhdfs",
+]
+
+services-all = [
+  "default",
+
+  "services-cacache",
+  "services-dashmap",
+  "services-dropbox",
+  "services-etcd",
+  # FIXME this requires a preinstalled fdb library
+  # "services-foundationdb",
+  "services-ftp",
+  "services-gdrive",
+  # FIXME how to support HDFS services in bindings?
+  # "services-hdfs",
+  "services-ipfs",
+  "services-memcached",
+  "services-mini-moka",
+  "services-moka",
+  "services-onedrive",
+  "services-persy",
+  "services-postgresql",
+  "services-mysql",
+  "services-redb",
+  "services-redis",
+  "services-redis-rustls",
+  "services-rocksdb",
+  "services-sled",
+  "services-supabase",
+  "services-tikv",
+  "services-vercel-artifacts",
+  "services-wasabi",
+  "services-mongodb",
+  "services-sqlite",
+]
+
+# Default services provided by opendal.
+services-azblob = [ "opendal/services-azblob" ]
+services-azdls = [ "opendal/services-azdls" ]
+services-cos = [ "opendal/services-cos" ]
+services-fs = [ "opendal/services-fs" ]
+services-gcs = [ "opendal/services-gcs" ]
+services-ghac = [ "opendal/services-ghac" ]
+services-http = [ "opendal/services-http" ]
+services-ipmfs = [ "opendal/services-ipmfs" ]
+services-memory = [ "opendal/services-memory" ]
+services-obs = [ "opendal/services-obs" ]
+services-oss = [ "opendal/services-oss" ]
+services-s3 = [ "opendal/services-s3" ]
+services-webdav = [ "opendal/services-webdav" ]
+services-webhdfs = [ "opendal/services-webhdfs" ]
+
+# Optional services provided by opendal.
+services-cacache = ["opendal/services-cacache"]
+services-dashmap = ["opendal/services-dashmap"]
+services-dropbox = ["opendal/services-dropbox"]
+services-etcd = ["opendal/services-etcd"]
+services-foundationdb = ["opendal/services-foundationdb"]
+services-ftp = ["opendal/services-ftp"]
+services-gdrive = ["opendal/services-gdrive"]
+services-hdfs = ["opendal/services-hdfs"]
+services-ipfs = ["opendal/services-ipfs"]
+services-memcached = ["opendal/services-memcached"]
+services-mini-moka = ["opendal/services-mini-moka"]
+services-moka = ["opendal/services-moka"]
+services-onedrive = ["opendal/services-onedrive"]
+services-persy = ["opendal/services-persy"]
+services-postgresql = ["opendal/services-postgresql"]
+services-redb = ["opendal/services-redb"]
+services-redis = ["opendal/services-redis"]
+services-redis-rustls = ["opendal/services-redis-rustls"]
+services-rocksdb = ["opendal/services-rocksdb"]
+services-sftp = ["opendal/services-sftp"]
+services-sled = ["opendal/services-sled"]
+services-supabase = ["opendal/services-supabase"]
+services-tikv = ["opendal/services-tikv"]
+services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
+services-wasabi = ["opendal/services-wasabi"]
+services-mysql = ["opendal/services-mysql"]
+services-mongodb = ["opendal/services-mongodb"]
+services-sqlite = ["opendal/services-sqlite"]
+
 [lib]
 crate-type = ["cdylib"]
 doc = false
diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py
index 9db7dbe92b8f..6c1a518dab57 100644
--- a/bindings/python/tests/conftest.py
+++ b/bindings/python/tests/conftest.py
@@ -15,9 +15,69 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
+
 from dotenv import load_dotenv
 import pytest
+import opendal
 
 
 load_dotenv()
 pytest_plugins = ("pytest_asyncio",)
+
+
+def pytest_configure(config):
+    # register an additional marker
+    config.addinivalue_line(
+        "markers",
+        "need_capability(*capability): mark test to run only on named capability",
+    )
+
+
+@pytest.fixture()
+def service_name():
+    service_name = os.environ.get("OPENDAL_TEST")
+    if service_name is None:
+        pytest.skip("OPENDAL_TEST not set")
+    return service_name
+
+
+@pytest.fixture()
+def setup_config(service_name):
+    # Read arguments from envs.
+    prefix = f"opendal_{service_name}_"
+    config = {}
+    for key in os.environ.keys():
+        if key.lower().startswith(prefix):
+            config[key[len(prefix) :].lower()] = os.environ.get(key)
+
+    return config
+
+
+@pytest.fixture()
+def operator(service_name, setup_config):
+    return opendal.Operator(service_name, **setup_config)
+
+
+@pytest.fixture()
+def async_operator(service_name, setup_config):
+    return opendal.AsyncOperator(service_name, **setup_config)
+
+
+@pytest.fixture(autouse=True)
+def check_capability(request, operator, async_operator):
+    if request.node.get_closest_marker("need_capability"):
+        if request.node.get_closest_marker("need_capability").args:
+            if not all(
+                [
+                    getattr(operator.capability(), x)
+                    for x in request.node.get_closest_marker("need_capability").args
+                ]
+                + [
+                    getattr(async_operator.capability(), x)
+                    for x in request.node.get_closest_marker("need_capability").args
+                ]
+            ):
+                pytest.skip(
+                    f"skip because {request.node.get_closest_marker('need_capability').args} not supported"
+                )
diff --git a/bindings/python/tests/test_capability.py b/bindings/python/tests/test_capability.py
new file mode 100644
index 000000000000..da2e9c14f854
--- /dev/null
+++ b/bindings/python/tests/test_capability.py
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+
+def test_capability(service_name, operator):
+    cap = operator.capability()
+    assert cap is not None
+    assert cap.read is not None
+
+
+def test_capability_exception(service_name, operator):
+    cap = operator.capability()
+    assert cap is not None
+    with pytest.raises(AttributeError) as e_info:
+        cap.read_demo
diff --git a/bindings/python/tests/test_read.py b/bindings/python/tests/test_read.py
new file mode 100644
index 000000000000..7c89ac9fdb38
--- /dev/null
+++ b/bindings/python/tests/test_read.py
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from uuid import uuid4
+from random import randint
+
+import pytest
+
+
+@pytest.mark.need_capability("read", "write", "delete")
+def test_sync_read(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"random_file_{str(uuid4())}"
+    content = os.urandom(size)
+    operator.write(filename, content)
+
+    read_content = operator.read(filename)
+    assert read_content is not None
+    assert read_content == content
+
+    operator.delete(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "delete")
+async def test_async_read(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"random_file_{str(uuid4())}"
+    content = os.urandom(size)
+    await async_operator.write(filename, content)
+
+    read_content = await async_operator.read(filename)
+    assert read_content is not None
+    assert read_content == content
+
+    await async_operator.delete(filename)
+
+
+@pytest.mark.need_capability("read", "write", "delete", "stat")
+def test_sync_read_stat(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"random_file_{str(uuid4())}"
+    content = os.urandom(size)
+    operator.write(filename, content)
+
+    metadata = operator.stat(filename)
+    assert metadata is not None
+    assert metadata.content_length == len(content)
+    assert metadata.mode.is_file()
+
+    operator.delete(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read", "write", "delete", "stat")
+async def test_async_read_stat(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"random_file_{str(uuid4())}"
+    content = os.urandom(size)
+    await async_operator.write(filename, content)
+
+    metadata = await async_operator.stat(filename)
+    assert metadata is not None
+    assert metadata.content_length == len(content)
+    assert metadata.mode.is_file()
+
+    await async_operator.delete(filename)
+
+    operator.delete(filename)
+
+
+@pytest.mark.need_capability("read")
+def test_sync_read_not_exists(service_name, operator, async_operator):
+    with pytest.raises(FileNotFoundError):
+        operator.read(str(uuid4()))
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("read")
+async def test_async_read_not_exists(service_name, operator, async_operator):
+    with pytest.raises(FileNotFoundError):
+        await async_operator.read(str(uuid4()))
diff --git a/bindings/python/tests/test_services.py b/bindings/python/tests/test_services.py
deleted file mode 100644
index 1d23cefc2e93..000000000000
--- a/bindings/python/tests/test_services.py
+++ /dev/null
@@ -1,224 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-from abc import ABC
-from uuid import uuid4
-from random import randint
-
-import opendal
-import pytest
-
-
-class AbstractTestSuite(ABC):
-    service_name = ""
-
-    def setup_method(self):
-        # Read arguments from envs.
-        prefix = f"opendal_{self.service_name}_"
-        self.config = {}
-        for key in os.environ.keys():
-            if key.lower().startswith(prefix):
-                self.config[key[len(prefix) :].lower()] = os.environ.get(key)
-
-        # Check if current test be enabled.
-        test_flag = self.config.get("test", "")
-        if test_flag != "on" and test_flag != "true":
-            raise ValueError(f"Service {self.service_name} test is not enabled.")
-
-        self.operator = opendal.Operator(self.service_name, **self.config)
-        self.async_operator = opendal.AsyncOperator(self.service_name, **self.config)
-
-    def test_sync_read(self):
-        size = randint(1, 1024)
-        filename = f"random_file_{str(uuid4())}"
-        content = os.urandom(size)
-        self.operator.write(filename, content)
-
-        read_content = self.operator.read(filename)
-        assert read_content is not None
-        assert read_content == content
-
-        self.operator.delete(filename)
-
-    @pytest.mark.asyncio
-    async def test_async_read(self):
-        size = randint(1, 1024)
-        filename = f"random_file_{str(uuid4())}"
-        content = os.urandom(size)
-        await self.async_operator.write(filename, content)
-
-        read_content = await self.async_operator.read(filename)
-        assert read_content is not None
-        assert read_content == content
-
-        await self.async_operator.delete(filename)
-
-    def test_sync_read_stat(self):
-        size = randint(1, 1024)
-        filename = f"random_file_{str(uuid4())}"
-        content = os.urandom(size)
-        self.operator.write(filename, content)
-
-        metadata = self.operator.stat(filename)
-        assert metadata is not None
-        assert metadata.content_length == len(content)
-        assert metadata.mode.is_file()
-
-        self.operator.delete(filename)
-
-    @pytest.mark.asyncio
-    async def test_async_read_stat(self):
-        size = randint(1, 1024)
-        filename = f"random_file_{str(uuid4())}"
-        content = os.urandom(size)
-        await self.async_operator.write(filename, content)
-
-        metadata = await self.async_operator.stat(filename)
-        assert metadata is not None
-        assert metadata.content_length == len(content)
-        assert metadata.mode.is_file()
-
-        await self.async_operator.delete(filename)
-
-        self.operator.delete(filename)
-
-    def test_sync_read_not_exists(self):
-        with pytest.raises(FileNotFoundError):
-            self.operator.read(str(uuid4()))
-
-    @pytest.mark.asyncio
-    async def test_async_read_not_exists(self):
-        with pytest.raises(FileNotFoundError):
-            await self.async_operator.read(str(uuid4()))
-
-    def test_sync_write(self):
-        size = randint(1, 1024)
-        filename = f"test_file_{str(uuid4())}.txt"
-        content = os.urandom(size)
-        size = len(content)
-        self.operator.write(filename, content)
-        metadata = self.operator.stat(filename)
-        assert metadata is not None
-        assert metadata.mode.is_file()
-        assert metadata.content_length == size
-
-        self.operator.delete(filename)
-
-    @pytest.mark.asyncio
-    async def test_async_write(self):
-        size = randint(1, 1024)
-        filename = f"test_file_{str(uuid4())}.txt"
-        content = os.urandom(size)
-        size = len(content)
-        await self.async_operator.write(filename, content)
-        metadata = await self.async_operator.stat(filename)
-        assert metadata is not None
-        assert metadata.mode.is_file()
-        assert metadata.content_length == size
-
-        await self.async_operator.delete(filename)
-
-    def test_sync_write_with_non_ascii_name(self):
-        size = randint(1, 1024)
-        filename = f"βŒπŸ˜±δΈ­ζ–‡_{str(uuid4())}.test"
-        content = os.urandom(size)
-        size = len(content)
-        self.operator.write(filename, content)
-        metadata = self.operator.stat(filename)
-        assert metadata is not None
-        assert metadata.mode.is_file()
-        assert metadata.content_length == size
-
-        self.operator.delete(filename)
-
-    @pytest.mark.asyncio
-    async def test_async_write_with_non_ascii_name(self):
-        size = randint(1, 1024)
-        filename = f"βŒπŸ˜±δΈ­ζ–‡_{str(uuid4())}.test"
-        content = os.urandom(size)
-        size = len(content)
-        await self.async_operator.write(filename, content)
-        metadata = await self.async_operator.stat(filename)
-        assert metadata is not None
-        assert metadata.mode.is_file()
-        assert metadata.content_length == size
-
-        await self.async_operator.delete(filename)
-
-    def test_sync_create_dir(self):
-        path = f"test_dir_{str(uuid4())}/"
-        self.operator.create_dir(path)
-        metadata = self.operator.stat(path)
-        assert metadata is not None
-        assert metadata.mode.is_dir()
-
-        self.operator.delete(path)
-
-    @pytest.mark.asyncio
-    async def test_async_create_dir(self):
-        path = f"test_dir_{str(uuid4())}/"
-        await self.async_operator.create_dir(path)
-        metadata = await self.async_operator.stat(path)
-        assert metadata is not None
-        assert metadata.mode.is_dir()
-
-        await self.async_operator.delete(path)
-
-    def test_sync_delete(self):
-        size = randint(1, 1024)
-        filename = f"test_file_{str(uuid4())}.txt"
-        content = os.urandom(size)
-        size = len(content)
-        self.operator.write(filename, content)
-        self.operator.delete(filename)
-        with pytest.raises(FileNotFoundError):
-            self.operator.stat(filename)
-
-    @pytest.mark.asyncio
-    async def test_async_delete(self):
-        size = randint(1, 1024)
-        filename = f"test_file_{str(uuid4())}.txt"
-        content = os.urandom(size)
-        size = len(content)
-        await self.async_operator.write(filename, content)
-        await self.async_operator.delete(filename)
-        with pytest.raises(FileNotFoundError):
-            await self.operator.stat(filename)
-    
-    def test_capability(self):
-        cap = self.operator.capability()
-        assert cap is not None
-        assert cap.read is not None
-    
-    def test_capability_exception(self):
-        cap = self.operator.capability()
-        assert cap is not None
-        with pytest.raises(AttributeError) as e_info:
-            cap.read_demo
-
-
-class TestS3(AbstractTestSuite):
-    service_name = "s3"
-
-
-class TestFS(AbstractTestSuite):
-    service_name = "fs"
-
-
-class TestMemory(AbstractTestSuite):
-    service_name = "memory"
diff --git a/bindings/python/tests/test_write.py b/bindings/python/tests/test_write.py
new file mode 100644
index 000000000000..12e173ed98b6
--- /dev/null
+++ b/bindings/python/tests/test_write.py
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from uuid import uuid4
+from random import randint
+
+import pytest
+
+
+@pytest.mark.need_capability("write", "delete", "stat")
+def test_sync_write(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"test_file_{str(uuid4())}.txt"
+    content = os.urandom(size)
+    size = len(content)
+    operator.write(filename, content)
+    metadata = operator.stat(filename)
+    assert metadata is not None
+    assert metadata.mode.is_file()
+    assert metadata.content_length == size
+
+    operator.delete(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("write", "delete", "stat")
+async def test_async_write(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"test_file_{str(uuid4())}.txt"
+    content = os.urandom(size)
+    size = len(content)
+    await async_operator.write(filename, content)
+    metadata = await async_operator.stat(filename)
+    assert metadata is not None
+    assert metadata.mode.is_file()
+    assert metadata.content_length == size
+
+    await async_operator.delete(filename)
+
+
+@pytest.mark.need_capability("write", "delete", "stat")
+def test_sync_write_with_non_ascii_name(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"βŒπŸ˜±δΈ­ζ–‡_{str(uuid4())}.test"
+    content = os.urandom(size)
+    size = len(content)
+    operator.write(filename, content)
+    metadata = operator.stat(filename)
+    assert metadata is not None
+    assert metadata.mode.is_file()
+    assert metadata.content_length == size
+
+    operator.delete(filename)
+
+@pytest.mark.need_capability("create_dir", "stat")
+def test_sync_create_dir(service_name, operator, async_operator):
+    path = f"test_dir_{str(uuid4())}/"
+    operator.create_dir(path)
+    metadata = operator.stat(path)
+    assert metadata is not None
+    assert metadata.mode.is_dir()
+
+    operator.delete(path)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("create_dir", "stat")
+async def test_async_create_dir(service_name, operator, async_operator):
+    path = f"test_dir_{str(uuid4())}/"
+    await async_operator.create_dir(path)
+    metadata = await async_operator.stat(path)
+    assert metadata is not None
+    assert metadata.mode.is_dir()
+
+    await async_operator.delete(path)
+
+
+@pytest.mark.need_capability("delete", "stat")
+def test_sync_delete(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"test_file_{str(uuid4())}.txt"
+    content = os.urandom(size)
+    size = len(content)
+    operator.write(filename, content)
+    operator.delete(filename)
+    with pytest.raises(FileNotFoundError):
+        operator.stat(filename)
+
+
+@pytest.mark.asyncio
+@pytest.mark.need_capability("delete", "stat")
+async def test_async_delete(service_name, operator, async_operator):
+    size = randint(1, 1024)
+    filename = f"test_file_{str(uuid4())}.txt"
+    content = os.urandom(size)
+    size = len(content)
+    await async_operator.write(filename, content)
+    await async_operator.delete(filename)
+    with pytest.raises(FileNotFoundError):
+        await operator.stat(filename)