feat: add MinIO configuration (#23)

Signed-off-by: Pratiksha Sankhe <[email protected]>
Co-authored-by: Alex Kanitz <[email protected]>
Co-authored-by: Alex Kanitz <[email protected]>

.github/workflows/pr_validation.yaml

@@ -10,7 +10,7 @@ jobs:
   detect-unresolved-conflicts:
     name: Detect unresolved merge conflicts
     runs-on: ubuntu-latest
-    needs: semantic_pr
+    needs: semantic-pr
     steps:
       - uses: actions/checkout@v3
       - name: List files with merge conflict markers
@@ -33,8 +33,13 @@ jobs:
           poetry-install-options: "--only=misc --no-root"
           poetry-export-options: "--only=misc"
 
-      - name: Check all the pre-commit hooks passed
-        run: pre-commit run --all-files
+      - name: Install pre-commit
+        run: pip install pre-commit
+
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]
+        with:
+          config: ../../.pre-commit-config.yaml
 
   semantic-pr:
     name: Semantic PR title

.pre-commit-config.yaml

@@ -34,6 +34,7 @@ repos:
     rev: v1.35.1
     hooks:
       - id: yamllint
+        exclude: deployment/config.yaml
   - repo: https://github.com/pappasam/toml-sort
     rev: v0.23.1
     hooks:

cloud_storage_handler/api/elixircloud/csh/client.py

@@ -0,0 +1,60 @@
+"""MinIO client class and convenience functions."""
+
+import logging
+
+from connexion import FlaskApp
+from minio import Minio
+
+from cloud_storage_handler.api.elixircloud.csh.models import MinioConfig
+
+logger = logging.getLogger(__name__)
+
+
+class MinioClient:
+    """Client for MinIO operations.
+
+    Wraps ``minio.Minio`` and adds convenience methods.
+
+    Attributes:
+        config: MinIO configuration.
+        client: MinIO client instance.
+    """
+
+    def __init__(self, config: MinioConfig) -> None:
+        """Class constructor.
+
+        Args:
+            config: MinIO configuration.
+        """
+        self.config: MinioConfig = config
+        self.client: Minio = Minio(
+            endpoint=f"{config.hostname}:{config.port}",
+            access_key=config.access_key,
+            secret_key=config.secret_key,
+            secure=config.secure,
+        )
+
+    def create_bucket(self) -> None:
+        """Create bucket if it does not exist."""
+        if not self.client.bucket_exists(self.config.bucket_name):
+            self.client.make_bucket(self.config.bucket_name)
+            logger.info(f"Bucket '{self.config.bucket_name}' created.")
+        else:
+            logger.debug(f"Bucket '{self.config.bucket_name}' already exists.")
+
+
+def register_minio_client(app: FlaskApp) -> FlaskApp:
+    """Register MinIO client and create bucket.
+
+    Args:
+        app: Connexion Flask app instance.
+
+    Returns:
+        Connexion Flask app instance with a MinIO client instance added to
+            its config.
+    """
+    minio_client = MinioClient(config=app.app.config.foca.custom.minio)
+    minio_client.create_bucket()
+    app.app.config.foca.custom.minio.client = minio_client
+    logger.info("MinIO client registered.")
+    return app

cloud_storage_handler/api/elixircloud/csh/models.py

@@ -0,0 +1,45 @@
+"""Model for MinIO Configuration."""
+
+from typing import Annotated, Optional
+
+from minio import Minio
+from pydantic import BaseModel, conint, constr
+
+
+class MinioConfig(BaseModel):
+    """Configuration for MinIO.
+
+    Attributes:
+        hostname: The name of the host where the MinIO server is running.
+        port: The port on which the MinIO server is running. Must be between 1
+            and 65535.
+        access_key: The access key used for authentication with MinIO.
+        secret_key: The secret key used for authentication with MinIO.
+        bucket_name: The name of the bucket where files are stored. Must be at
+            least 1 character long.
+        secure: Whether to use TLS connection to storage service or not.
+        client: Client instance.
+
+    Examples:
+        MinioConfig(
+            hostname="localhost",
+            port=9000,
+            access_key="minioadmin",
+            secret_key="minioadmin",
+            bucket_name="files",
+            secure=False,
+        )
+    """
+
+    hostname: str = "localhost"
+    port: Annotated[int, conint(ge=1, le=65535)] = 9000
+    access_key: str = "minioadmin"
+    secret_key: str = "minioadmin"
+    bucket_name: Annotated[str, constr(min_length=1)] = "files"
+    secure: bool = False
+    client: Optional[Minio] = None
+
+    class Config:
+        """Model configuration."""
+
+        arbitrary_types_allowed = True

cloud_storage_handler/custom_config.py

@@ -0,0 +1,15 @@
+"""Custom configuration model for the FOCA app."""
+
+from pydantic import BaseModel
+
+from cloud_storage_handler.api.elixircloud.csh.models import MinioConfig
+
+
+class CustomConfig(BaseModel):
+    """Custom configuration model for the FOCA app.
+
+    Attributes:
+        minio: MinIO configuration parameters.
+    """
+
+    minio: MinioConfig

cloud_storage_handler/main.py

@@ -1,51 +1,44 @@
 """API server entry point."""
 
-import logging
 import os
 from pathlib import Path
 
 from connexion import FlaskApp
 from foca import Foca
 
-logger = logging.getLogger(__name__)
+from cloud_storage_handler.api.elixircloud.csh.client import (
+    register_minio_client,
+)
 
 
 def init_app() -> FlaskApp:
     """Initialize and return the FOCA app.
 
     This function initializes the FOCA app by loading the configuration
     from the environment variable `CSH_FOCA_CONFIG_PATH` if set, or from
-    the default path if not. It raises a `FileNotFoundError` if the
-    configuration file is not found.
+    the default path if not.
 
     Returns:
-        A Connexion application instance.
-
-    Raises:
-        FileNotFoundError: If the configuration file is not found.
+        A Connexion Flask application instance.
     """
-    # Determine the configuration path
-    if config_path_env := os.getenv("CSH_FOCA_CONFIG_PATH"):
-        print(config_path_env)
-        config_path = Path(config_path_env).resolve()
-    else:
-        config_path = (
-            Path(__file__).parents[1] / "deployment" / "config.yaml"
-        ).resolve()
-
-    # Check if the configuration file exists
-    if not config_path.exists():
-        raise FileNotFoundError(f"Config file not found at: {config_path}")
+    config_path = Path(
+        os.getenv(
+            "CSH_FOCA_CONFIG_PATH",
+            Path(__file__).parents[1] / "deployment" / "config.yaml",
+        )
+    ).resolve()
 
     foca = Foca(
         config_file=config_path,
+        custom_config_model="cloud_storage_handler.custom_config.CustomConfig",
     )
     return foca.create_app()
 
 
 def main() -> None:
     """Run FOCA application."""
     app = init_app()
+    app = register_minio_client(app)
     app.run(port=app.port)
 
 

deployment/config.yaml

@@ -1,42 +1,48 @@
 ---
 server:
-    host: '0.0.0.0'
-    port: 8080
-    debug: false
-    environment: development
-    testing: false
-    use_reloader: true
+  host: '0.0.0.0'
+  port: 8080
+  debug: false
+  environment: development
+  testing: false
+  use_reloader: true
 
 security:
-    auth:
-        required: true
-        add_key_to_claims: true
-        allow_expired: false
-        audience: null
-        claim_identity: sub
-        claim_issuer: iss
-        algorithms:
-            - RS256
-        validation_methods:
-            - userinfo
-            - public_key
-        validation_checks: all
+  auth:
+    required: true
+    add_key_to_claims: true
+    allow_expired: false
+    audience: null
+    claim_identity: sub
+    claim_issuer: iss
+    algorithms:
+      - RS256
+    validation_methods:
+      - userinfo
+      - public_key
+    validation_checks: all
 
 api:
-    specs:
-        - path: cloud_storage_handler/api/specs/specs.yaml
-          append: null
-          add_operation_fields:
-              x-openapi-router-controller: cloud_storage_handler.api.elixircloud.csh.controllers
-          connexion:
-              strict_validation: true
-              validate_responses: false
-              options:
-                  swagger_ui: true
-                  serve_spec: true
+  specs:
+    - path: cloud_storage_handler/api/specs/specs.yaml
+      append: null
+      add_operation_fields:
+        x-openapi-router-controller: cloud_storage_handler.api.elixircloud.csh.controllers
+      connexion:
+        strict_validation: true
+        validate_responses: false
+
+custom:
+  minio:
+    hostname: localhost
+    port: 9000
+    access_key: minioadmin
+    secret_key: minioadmin
+    bucket_name: 'files'
+    secure: false
 
 exceptions:
-    required_members: [['message'], ['code']]
-    status_member: ['code']
-    exceptions: cloud_storage_handler.exceptions.exceptions
+  required_members: [['message'], ['code']]
+  status_member: ['code']
+  exceptions: cloud_storage_handler.exceptions.exceptions
 ...

docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst

@@ -4,6 +4,14 @@ cloud\_storage\_handler.api.elixircloud.csh package
 Submodules
 ----------
 
+cloud\_storage\_handler.api.elixircloud.csh.client module
+---------------------------------------------------------
+
+.. automodule:: cloud_storage_handler.api.elixircloud.csh.client
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 cloud\_storage\_handler.api.elixircloud.csh.controllers module
 --------------------------------------------------------------
 
@@ -12,6 +20,14 @@ cloud\_storage\_handler.api.elixircloud.csh.controllers module
    :undoc-members:
    :show-inheritance:
 
+cloud\_storage\_handler.api.elixircloud.csh.models module
+---------------------------------------------------------
+
+.. automodule:: cloud_storage_handler.api.elixircloud.csh.models
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/source/pages/cloud_storage_handler.rst

@@ -12,6 +12,14 @@ Subpackages
 Submodules
 ----------
 
+cloud\_storage\_handler.custom\_config module
+---------------------------------------------
+
+.. automodule:: cloud_storage_handler.custom_config
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 cloud\_storage\_handler.exceptions module
 -----------------------------------------