Merge remote-tracking branch 'origin/main'

Author: ahsimb

doc/changes/unreleased.md

@@ -5,6 +5,8 @@
 * #66: Implemented a standard CLI options builder.
 * #70: Implemented a CLI callback function for the language container deployment.
 * #69: Added create_bucketfs_location function.
+* #75: Added create_bucketfs_location_from_conn_object function.
+* #74: Implemented a CLI callback function for creating a bucket-fs connection object.
 
 # Refactoring
 

exasol/python_extension_common/cli/bucketfs_conn_object_cli.py

@@ -0,0 +1,10 @@
+import exasol.python_extension_common.connections.bucketfs_location as bl
+
+
+class BucketfsConnObjectCli:
+    def __init__(self, conn_name_arg: str):
+        self._conn_name_arg = conn_name_arg
+
+    def __call__(self, **kwargs):
+        conn_name = kwargs.pop(self._conn_name_arg)
+        bl.create_bucketfs_conn_object(conn_name=conn_name, **kwargs)

exasol/python_extension_common/connections/bucketfs_location.py

@@ -1,58 +1,34 @@
+from typing import Any
+from enum import Enum, auto
+from dataclasses import dataclass
+import json
+import pyexasol     # type: ignore
 import exasol.bucketfs as bfs   # type: ignore
 from exasol.saas.client.api_access import get_database_id   # type: ignore
 
 from exasol.python_extension_common.cli.std_options import StdParams, check_params
+from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection
 
 
-def create_bucketfs_location(**kwargs) -> bfs.path.PathLike:
-    """
-    Creates a BucketFS PathLike object using the data provided in the kwargs. These
-    can be parameters for the BucketFS either On-Prem or SaaS database. The parameters
-    should correspond to the CLI options defined in the cli/std_options.py.
+class _Backend(Enum):
+    onprem = auto()
+    saas = auto()
 
-    Raises a ValueError if the provided parameters are insufficient for either
-    On-Prem or SaaS cases.
-    """
 
-    path_in_bucket = kwargs.get(StdParams.path_in_bucket.name, '')
+def _infer_backend(bfs_params: dict[str, Any]) -> _Backend:
+    """
+    Infers the backend from the provided dictionary of CLI parameters.
+    Raises a ValueError if the collection of CLI parameters is insufficient to access
+    the BucketFS on either of the backends.
+    """
 
-    # Infer where the database is - on-prem or SaaS.
     if check_params([StdParams.bucketfs_host, StdParams.bucketfs_port,
                      StdParams.bucket, StdParams.bucketfs_user, StdParams.bucketfs_password],
-                    kwargs):
-        net_service = ('https' if kwargs.get(StdParams.bucketfs_use_https.name, True)
-                       else 'http')
-        url = (f"{net_service}://"
-               f"{kwargs[StdParams.bucketfs_host.name]}:"
-               f"{kwargs[StdParams.bucketfs_port.name]}")
-        return bfs.path.build_path(
-            backend=bfs.path.StorageBackend.onprem,
-            url=url,
-            username=kwargs[StdParams.bucketfs_user.name],
-            password=kwargs[StdParams.bucketfs_password.name],
-            service_name=kwargs.get(StdParams.bucketfs_name.name),
-            bucket_name=kwargs[StdParams.bucket.name],
-            verify=kwargs.get(StdParams.use_ssl_cert_validation.name, True),
-            path=path_in_bucket
-        )
+                    bfs_params):
+        return _Backend.onprem
     elif check_params([StdParams.saas_url, StdParams.saas_account_id, StdParams.saas_token,
-                       [StdParams.saas_database_id, StdParams.saas_database_name]], kwargs):
-        saas_url = kwargs[StdParams.saas_url.name]
-        saas_account_id = kwargs[StdParams.saas_account_id.name]
-        saas_token = kwargs[StdParams.saas_token.name]
-        saas_database_id = (kwargs.get(StdParams.saas_database_id.name) or
-                            get_database_id(
-                                host=saas_url,
-                                account_id=saas_account_id,
-                                pat=saas_token,
-                                database_name=kwargs[StdParams.saas_database_name.name]
-                            ))
-        return bfs.path.build_path(backend=bfs.path.StorageBackend.saas,
-                                   url=saas_url,
-                                   account_id=saas_account_id,
-                                   database_id=saas_database_id,
-                                   pat=saas_token,
-                                   path=path_in_bucket)
+                       [StdParams.saas_database_id, StdParams.saas_database_name]], bfs_params):
+        return _Backend.saas
 
     raise ValueError(
         'Incomplete parameter list. Please either provide the parameters ['
@@ -64,3 +40,175 @@ def create_bucketfs_location(**kwargs) -> bfs.path.PathLike:
         f'{StdParams.saas_database_name.name}, {StdParams.saas_token.name}] for a '
         'SaaS database.'
     )
+
+
+def _convert_onprem_bfs_params(bfs_params: dict[str, Any]) -> dict[str, Any]:
+    """
+    Converts OnPrem BucketFS parameters from the CLI format to the format expected
+    by the exasol.bucketfs.path.build_path.
+    """
+
+    net_service = ('https' if bfs_params.get(StdParams.bucketfs_use_https.name, True)
+                   else 'http')
+    url = (f"{net_service}://"
+           f"{bfs_params[StdParams.bucketfs_host.name]}:"
+           f"{bfs_params[StdParams.bucketfs_port.name]}")
+    return {
+        'backend': bfs.path.StorageBackend.onprem.name,
+        'url': url,
+        'username': bfs_params[StdParams.bucketfs_user.name],
+        'password': bfs_params[StdParams.bucketfs_password.name],
+        'service_name': bfs_params.get(StdParams.bucketfs_name.name),
+        'bucket_name': bfs_params[StdParams.bucket.name],
+        'verify': bfs_params.get(StdParams.use_ssl_cert_validation.name, True),
+        'path': bfs_params.get(StdParams.path_in_bucket.name, '')
+    }
+
+
+def _convert_saas_bfs_params(bfs_params: dict[str, Any]) -> dict[str, Any]:
+    """
+    Converts SaaS BucketFS parameters from the CLI format to the format expected
+    by the exasol.bucketfs.path.build_path.
+    """
+
+    saas_url = bfs_params[StdParams.saas_url.name]
+    saas_account_id = bfs_params[StdParams.saas_account_id.name]
+    saas_token = bfs_params[StdParams.saas_token.name]
+    saas_database_id = (bfs_params.get(StdParams.saas_database_id.name) or
+                        get_database_id(
+                            host=saas_url,
+                            account_id=saas_account_id,
+                            pat=saas_token,
+                            database_name=bfs_params[StdParams.saas_database_name.name]
+                        ))
+    return {
+        'backend': bfs.path.StorageBackend.saas.name,
+        'url': saas_url,
+        'account_id': saas_account_id,
+        'database_id': saas_database_id,
+        'pat': saas_token,
+        'path': bfs_params.get(StdParams.path_in_bucket.name, '')
+    }
+
+
+def create_bucketfs_location(**kwargs) -> bfs.path.PathLike:
+    """
+    Creates a BucketFS PathLike object using the data provided in the kwargs. These
+    can be parameters for the BucketFS at either On-Prem or SaaS database. The input
+    parameters should correspond to the CLI options defined in the cli/std_options.py.
+
+    Raises a ValueError if the provided parameters are insufficient for either
+    On-Prem or SaaS cases.
+    """
+
+    db_type = _infer_backend(kwargs)
+    if db_type == _Backend.onprem:
+        return bfs.path.build_path(**_convert_onprem_bfs_params(kwargs))
+    else:
+        return bfs.path.build_path(**_convert_saas_bfs_params(kwargs))
+
+
+@dataclass
+class ConnectionInfo:
+    """
+    This is not a connection object. It's just a structure to keep together the data
+    required for creating a BucketFs connection object. Useful for testing.
+    """
+    address: str
+    user: str
+    password: str
+
+
+def _to_json_str(bucketfs_params: dict[str, Any], selected: list[str]) -> str:
+    filtered_kwargs = {k: v for k, v in bucketfs_params.items()
+                       if (k in selected) and (v is not None)}
+    return json.dumps(filtered_kwargs)
+
+
+def write_bucketfs_conn_object(pyexasol_connection: pyexasol.ExaConnection,
+                               conn_name: str,
+                               conn_obj: ConnectionInfo) -> None:
+
+    query = (f"CREATE OR REPLACE  CONNECTION {conn_name} "
+             f"TO '{conn_obj.address}' "
+             f"USER '{conn_obj.user}' "
+             f"IDENTIFIED BY '{conn_obj.password}'")
+    pyexasol_connection.execute(query)
+
+
+def create_bucketfs_conn_object_onprem(pyexasol_connection: pyexasol.ExaConnection,
+                                       conn_name: str,
+                                       bucketfs_params: dict[str, Any]) -> None:
+    """
+    Creates in the database a connection object encapsulating the BucketFS parameters
+    for an OnPrem backend.
+
+    Parameters:
+    pyexasol_connection:
+        DB connection.
+    conn_name:
+        Name for the connection object.
+    bucketfs_params:
+        OnPrem BucketFS parameters in the format of the exasol.bucketfs.path.build_path.
+    """
+    conn_to = _to_json_str(bucketfs_params, [
+        'backend', 'url', 'service_name', 'bucket_name', 'path', 'verify'])
+    conn_user = _to_json_str(bucketfs_params, ['username'])
+    conn_password = _to_json_str(bucketfs_params, ['password'])
+
+    write_bucketfs_conn_object(pyexasol_connection, conn_name,
+                               ConnectionInfo(conn_to, conn_user, conn_password))
+
+
+def create_bucketfs_conn_object_saas(pyexasol_connection: pyexasol.ExaConnection,
+                                     conn_name: str,
+                                     bucketfs_params: dict[str, Any]) -> None:
+    """
+    Creates in the database a connection object encapsulating the BucketFS parameters
+    for a SaaS backend.
+
+    Parameters:
+    pyexasol_connection:
+        DB connection.
+    conn_name:
+        Name for the connection object.
+    bucketfs_params:
+        SaaS BucketFS parameters in the format of the exasol.bucketfs.path.build_path.
+    """
+    conn_to = _to_json_str(bucketfs_params, ['backend', 'url', 'path'])
+    conn_user = _to_json_str(bucketfs_params, ['account_id', 'database_id'])
+    conn_password = _to_json_str(bucketfs_params, ['pat'])
+
+    write_bucketfs_conn_object(pyexasol_connection, conn_name,
+                               ConnectionInfo(conn_to, conn_user, conn_password))
+
+
+def create_bucketfs_conn_object(conn_name: str, **kwargs) -> None:
+    """
+    Creates in the database a connection object encapsulating the provided BucketFS
+    parameters. These can be parameters for either On-Prem or SaaS database. They
+    should correspond to the CLI options defined in the cli/std_options.py.
+
+    Raises a ValueError if the provided parameters are insufficient for either
+    On-Prem or SaaS cases.
+    """
+    with open_pyexasol_connection(**kwargs) as pyexasol_connection:
+        db_type = _infer_backend(kwargs)
+        if db_type == _Backend.onprem:
+            create_bucketfs_conn_object_onprem(pyexasol_connection, conn_name,
+                                               _convert_onprem_bfs_params(kwargs))
+        else:
+            create_bucketfs_conn_object_saas(pyexasol_connection, conn_name,
+                                             _convert_saas_bfs_params(kwargs))
+
+
+def create_bucketfs_location_from_conn_object(conn_obj) -> bfs.path.PathLike:
+    """
+    Creates a BucketFS PathLike object using data contained in the provided connection
+    object.
+    """
+
+    bfs_params = json.loads(conn_obj.address)
+    bfs_params.update(json.loads(conn_obj.user))
+    bfs_params.update(json.loads(conn_obj.password))
+    return bfs.path.build_path(**bfs_params)

test/integration/conftest.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
+from typing import Any
 import pytest
 import click
 import requests
+from urllib.parse import urlparse
 from contextlib import ExitStack, contextmanager
 import pyexasol
 import exasol.bucketfs as bfs
 
+from exasol.python_extension_common.cli.std_options import StdParams
 from exasol.python_extension_common.deployment.language_container_deployer import (
     LanguageContainerDeployer,
 )
@@ -95,3 +98,53 @@ def create_deployer(create_test_schema: bool = False, open_test_schema: bool = T
                 open_schema(pyexasol_connection, db_schema)
             yield LanguageContainerDeployer(pyexasol_connection, language_alias, bucketfs_path)
     return create_deployer
+
+
+@pytest.fixture(scope='session')
+def onprem_db_params(backend_aware_onprem_database,
+                     exasol_config) -> dict[str, Any]:
+    return {
+        StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}',
+        StdParams.db_user.name: exasol_config.username,
+        StdParams.db_password.name: exasol_config.password,
+        StdParams.use_ssl_cert_validation.name: False
+    }
+
+
+@pytest.fixture(scope='session')
+def onprem_bfs_params(backend_aware_onprem_database,
+                      bucketfs_config) -> dict[str, Any]:
+    parsed_url = urlparse(bucketfs_config.url)
+    host, port = parsed_url.netloc.split(":")
+    return {
+        StdParams.bucketfs_host.name: host,
+        StdParams.bucketfs_port.name: port,
+        StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https',
+        StdParams.bucketfs_user.name: bucketfs_config.username,
+        StdParams.bucketfs_password.name: bucketfs_config.password,
+        StdParams.bucketfs_name.name: 'bfsdefault',
+        StdParams.bucket.name: 'default',
+        StdParams.use_ssl_cert_validation.name: False
+    }
+
+
+@pytest.fixture(scope='session')
+def saas_params_id(saas_host,
+                   saas_pat,
+                   saas_account_id,
+                   backend_aware_saas_database_id) -> dict[str, Any]:
+    return {
+        StdParams.saas_url.name: saas_host,
+        StdParams.saas_account_id.name: saas_account_id,
+        StdParams.saas_database_id.name: backend_aware_saas_database_id,
+        StdParams.saas_token.name: saas_pat,
+    }
+
+
+@pytest.fixture(scope='session')
+def saas_params_name(saas_params_id,
+                     database_name) -> dict[str, Any]:
+    saas_params = dict(saas_params_id)
+    saas_params.pop(StdParams.saas_database_id.name)
+    saas_params[StdParams.saas_database_name.name] = database_name
+    return saas_params