Merge remote-tracking branch 'origin/main'

Author: ahsimb

doc/changes/unreleased.md

@@ -2,9 +2,10 @@
 
 ## Features
 
-* #66: Implement a standard CLI options builder.
-* #69: Add create_bucketfs_location function.
+* #66: Implemented a standard CLI options builder.
+* #70: Implemented a CLI callback function for the language container deployment.
+* #69: Added create_bucketfs_location function.
 
 # Refactoring
 
-* #68: Make open_pyexasol_connection(...) using kwargs derived from the cli options.
+* #68: Made open_pyexasol_connection(...) using kwargs derived from the cli options.

doc/user_guide/user-guide.md

@@ -15,10 +15,10 @@ similar to the one below.
   python -m <exasol_extension>.deploy language-container <options>
   ```
 
-The name of the script (```<exasol_extension>.deploy``` in the above command) can vary from one extension to another.
-Please check the user guide of a particular extension. The rest of the command line will have a common format. It
-will include the command - ```language-container``` - and selected options. The choice of options is primarily
-determined by the storage backend being used - On-Prem or SaaS.
+The name of the script (```<exasol_extension>.deploy``` in the above command) and the command name
+(e.g. ```language-container```) can vary from one extension to another. Please check the user guide of a particular
+extension. The rest of the command line will have a common format. It will include some of the options defined below.
+The choice of options is primarily determined by the storage backend being used - On-Prem or SaaS.
 
 ### List of options
 
@@ -55,13 +55,13 @@ another source.
 | version                      |   [x]   | [x]  | Optional, provide for downloading SLC from GitHub       |
 | container-file               |   [x]   | [x]  | Optional, provide for uploading SLC file                |
 | ssl-cert-path                |   [x]   | [x]  | Optional                                                |
-| [no_]use-ssl-cert-validation |   [x]   | [x]  | Optional boolean, defaults to True                      |
+| [no-]use-ssl-cert-validation |   [x]   | [x]  | Optional boolean, defaults to True                      |
 | ssl-client-cert-path         |   [x]   |      | Optional                                                |
 | ssl-client-private-key       |   [x]   |      | Optional                                                |
-| [no_]upload-container        |   [x]   | [x]  | Optional boolean, defaults to True                      |
-| [no_]alter-system            |   [x]   | [x]  | Optional boolean, defaults to True                      |
-| [dis]allow-override          |   [x]   | [x]  | Optional boolean, defaults to False                     |
-| [no_]wait_for_completion     |   [x]   | [x]  | Optional boolean, defaults to True                      |
+| [no-]upload-container        |   [x]   | [x]  | Optional boolean, defaults to True                      |
+| [no-]alter-system            |   [x]   | [x]  | Optional boolean, defaults to True                      |
+| [no-]allow-override          |   [x]   | [x]  | Optional boolean, defaults to False                     |
+| [no-]wait_for_completion     |   [x]   | [x]  | Optional boolean, defaults to True                      |
 
 ### Container selection
 

exasol/python_extension_common/cli/language_container_deployer_cli.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+
+from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer
+from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection
+from exasol.python_extension_common.connections.bucketfs_location import create_bucketfs_location
+from exasol.python_extension_common.cli.std_options import StdParams
+
+
+class LanguageContainerDeployerCli:
+    """
+    The class provides a CLI callback function that creates and runs the
+    LangaugeContainerDeployer.
+
+    At first glance, it may look a bit over-designed. The reason for wrapping a
+    callback function in a class is to let the user define the option names for the
+    container URL and container name. These options are not defined in the StdParams
+    but rather generated by formatters. The user can give them arbitrary names.
+    Hence, we don't want to assume any particular names in the callback function.
+    """
+
+    def __init__(self,
+                 container_url_arg: str | None = None,
+                 container_name_arg: str | None = None) -> None:
+        self._container_url_arg = container_url_arg
+        self._container_name_arg = container_name_arg
+
+    def __call__(self, **kwargs):
+
+        pyexasol_connection = open_pyexasol_connection(**kwargs)
+        bucketfs_location = create_bucketfs_location(**kwargs)
+
+        language_alias = kwargs[StdParams.language_alias.name]
+        container_file = kwargs[StdParams.container_file.name]
+        upload_container = kwargs[StdParams.upload_container.name]
+        alter_system = kwargs[StdParams.alter_system.name]
+        allow_override = kwargs[StdParams.allow_override.name]
+        wait_for_completion = kwargs[StdParams.wait_for_completion.name]
+
+        deployer = LanguageContainerDeployer(pyexasol_connection,
+                                             language_alias,
+                                             bucketfs_location)
+        if not upload_container:
+            deployer.run(alter_system=alter_system,
+                         allow_override=allow_override,
+                         wait_for_completion=wait_for_completion)
+        elif container_file:
+            deployer.run(container_file=Path(container_file),
+                         alter_system=alter_system,
+                         allow_override=allow_override,
+                         wait_for_completion=wait_for_completion)
+        elif kwargs.get(self._container_url_arg) and kwargs.get(self._container_name_arg):
+            deployer.download_and_run(kwargs[self._container_url_arg],
+                                      kwargs[self._container_name_arg],
+                                      alter_system=alter_system,
+                                      allow_override=allow_override,
+                                      wait_for_completion=wait_for_completion)
+        else:
+            raise ValueError("To upload a language container either its release version "
+                             f"(--{StdParams.version.name}) or a path of the already "
+                             f"downloaded container file (--{StdParams.container_file.name}) "
+                             "must be provided.")

exasol/python_extension_common/deployment/language_container_deployer.py

@@ -7,6 +7,7 @@
 import tempfile
 import ssl
 import requests     # type: ignore
+import warnings
 import pyexasol     # type: ignore
 import exasol.bucketfs as bfs   # type: ignore
 from exasol.saas.client.api_access import (get_connection_params, get_database_id)      # type: ignore
@@ -330,7 +331,12 @@ def create(cls,
                use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None,
                ssl_client_certificate: Optional[str] = None,
                ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer":
-
+        warnings.warn(
+            "create() function is deprecated and will be removed in a future version. "
+            "For CLI use the LanguageContainerDeployerCli class instead.",
+            DeprecationWarning,
+            stacklevel=2
+        )
         # Infer where the database is - on-prem or SaaS.
         if all((dsn, db_user, db_password, bucketfs_host, bucketfs_port,
                 bucketfs_name, bucket, bucketfs_user, bucketfs_password)):

exasol/python_extension_common/deployment/language_container_deployer_cli.py

@@ -4,6 +4,7 @@
 from enum import Enum
 from pathlib import Path
 import click
+import warnings
 from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer
 
 
@@ -183,6 +184,12 @@ def language_container_deployer_main(
         wait_for_completion: bool,
         container_url: Optional[str] = None,
         container_name: Optional[str] = None):
+    warnings.warn(
+        "language_container_deployer_main() function is deprecated and will be removed "
+        "in a future version. For CLI use the LanguageContainerDeployerCli class instead.",
+        DeprecationWarning,
+        stacklevel=2
+    )
 
     deployer = LanguageContainerDeployer.create(
         bucketfs_name=bucketfs_name,

test/integration/cli/test_language_container_deployer_cli.py

@@ -0,0 +1,192 @@
+from typing import Any
+from contextlib import ExitStack
+from urllib.parse import urlparse
+import pytest
+import click
+from click.testing import CliRunner
+
+from exasol.python_extension_common.cli.std_options import (
+    StdTags, StdParams, ParameterFormatters, select_std_options)
+from exasol.python_extension_common.connections.pyexasol_connection import (
+    open_pyexasol_connection)
+from exasol.python_extension_common.cli.language_container_deployer_cli import (
+    LanguageContainerDeployerCli)
+from test.utils.db_utils import (assert_udf_running, create_schema)
+from test.utils.revert_language_settings import revert_language_settings
+
+CONTAINER_URL_ARG = 'container_url'
+CONTAINER_NAME_ARG = 'container_name'
+
+
+@pytest.fixture(scope='session')
+def onprem_cli_args(backend_aware_onprem_database,
+                    exasol_config,
+                    bucketfs_config,
+                    language_alias) -> dict[str, Any]:
+
+    parsed_url = urlparse(bucketfs_config.url)
+    host, port = parsed_url.netloc.split(":")
+    return {
+        StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}',
+        StdParams.db_user.name: exasol_config.username,
+        StdParams.db_password.name: exasol_config.password,
+        StdParams.bucketfs_host.name: host,
+        StdParams.bucketfs_port.name: port,
+        StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https',
+        StdParams.bucketfs_user.name: bucketfs_config.username,
+        StdParams.bucketfs_password.name: bucketfs_config.password,
+        StdParams.bucketfs_name.name: 'bfsdefault',
+        StdParams.bucket.name: 'default',
+        StdParams.use_ssl_cert_validation.name: False,
+    }
+
+
+@pytest.fixture(scope='session')
+def saas_cli_args(saas_host,
+                  saas_pat,
+                  saas_account_id,
+                  backend_aware_saas_database_id,
+                  ) -> dict[str, Any]:
+    return {
+        StdParams.saas_url.name: saas_host,
+        StdParams.saas_account_id.name: saas_account_id,
+        StdParams.saas_database_id.name: backend_aware_saas_database_id,
+        StdParams.saas_token.name: saas_pat
+    }
+
+
+@pytest.fixture(scope='session')
+def slc_cli_args(language_alias) -> dict[str, Any]:
+    return {
+        StdParams.alter_system.name: True,
+        StdParams.allow_override.name: True,
+        StdParams.wait_for_completion.name: True,
+        StdParams.path_in_bucket.name: 'container',
+        StdParams.language_alias.name: language_alias
+    }
+
+
+def create_deploy_command(backend_tag: StdTags,
+                          container_name: str | None = None,
+                          container_url_formatter: str | None = None) -> click.Command:
+    """
+    This is a blueprint for creating an isolated click Command
+    for the language container deployment.
+
+    backend_tag should be either StdTags.ONPREM or StdTags.SAAS.
+    """
+    if container_name and container_url_formatter:
+        ver_formatter = ParameterFormatters()
+        ver_formatter.set_formatter(CONTAINER_URL_ARG, container_url_formatter)
+        ver_formatter.set_formatter(CONTAINER_NAME_ARG, container_name)
+        formatters = {StdParams.version: ver_formatter}
+    else:
+        formatters = None
+
+    opts = select_std_options(
+        [StdTags.DB | backend_tag, StdTags.BFS | backend_tag, StdTags.SLC],
+        formatters=formatters)
+    cli_callback = LanguageContainerDeployerCli(
+        container_url_arg=CONTAINER_URL_ARG,
+        container_name_arg=CONTAINER_NAME_ARG)
+
+    return click.Command('deploy_slc', params=opts, callback=cli_callback)
+
+
+def make_args_string(**kwargs) -> str:
+    def arg_string(k: str, v: Any):
+        k = k.replace("_", "-")
+        if isinstance(v, bool):
+            return f'--{k}' if v else f'--no-{k}'
+        return f'--{k} "{v}"'
+
+    return ' '.join(arg_string(k, v) for k, v in kwargs.items())
+
+
+def run_deploy_command(deploy_command: click.Command,
+                       arg_string: str,
+                       language_alias: str,
+                       db_schema: str,
+                       db_params: dict[str, Any]):
+
+    with ExitStack() as stack:
+        conn_before = stack.enter_context(open_pyexasol_connection(**db_params))
+        stack.enter_context(revert_language_settings(conn_before))
+
+        runner = CliRunner()
+        runner.invoke(deploy_command, args=arg_string,
+                      catch_exceptions=False, standalone_mode=False)
+
+        # We have to open another connection because the language settings on
+        # the previously opened connection are unaffected by the slc deployment.
+        conn_after = stack.enter_context(open_pyexasol_connection(**db_params))
+        create_schema(conn_after, db_schema)
+        assert_udf_running(conn_after, language_alias, db_schema)
+
+
+def test_slc_deployer_cli_onprem_url(use_onprem,
+                                     container_version,
+                                     container_name,
+                                     container_url_formatter,
+                                     language_alias,
+                                     db_schema,
+                                     onprem_cli_args,
+                                     slc_cli_args):
+    if not use_onprem:
+        pytest.skip("The test is not configured to use ITDE.")
+
+    deploy_command = create_deploy_command(StdTags.ONPREM,
+                                           container_name=container_name,
+                                           container_url_formatter=container_url_formatter)
+    extra_cli_args = {StdParams.version.name: container_version}
+    arg_string = make_args_string(**onprem_cli_args, **slc_cli_args, **extra_cli_args)
+    run_deploy_command(deploy_command, arg_string, language_alias, db_schema, onprem_cli_args)
+
+
+def test_slc_deployer_cli_onprem_file(use_onprem,
+                                      container_path,
+                                      language_alias,
+                                      db_schema,
+                                      onprem_cli_args,
+                                      slc_cli_args):
+    if not use_onprem:
+        pytest.skip("The test is not configured to use ITDE.")
+
+    deploy_command = create_deploy_command(StdTags.ONPREM)
+    extra_cli_args = {StdParams.container_file.name: container_path}
+    arg_string = make_args_string(**onprem_cli_args, **slc_cli_args, **extra_cli_args)
+    run_deploy_command(deploy_command, arg_string, language_alias, db_schema, onprem_cli_args)
+
+
+def test_slc_deployer_cli_saas_url(use_saas,
+                                   container_version,
+                                   container_name,
+                                   container_url_formatter,
+                                   language_alias,
+                                   db_schema,
+                                   saas_cli_args,
+                                   slc_cli_args):
+    if not use_saas:
+        pytest.skip("The test is not configured to run in SaaS.")
+
+    deploy_command = create_deploy_command(StdTags.SAAS,
+                                           container_name=container_name,
+                                           container_url_formatter=container_url_formatter)
+    extra_cli_args = {StdParams.version.name: container_version}
+    arg_string = make_args_string(**saas_cli_args, **slc_cli_args, **extra_cli_args)
+    run_deploy_command(deploy_command, arg_string, language_alias, db_schema, saas_cli_args)
+
+
+def test_slc_deployer_cli_saas_file(use_saas,
+                                    container_path,
+                                    language_alias,
+                                    db_schema,
+                                    saas_cli_args,
+                                    slc_cli_args):
+    if not use_saas:
+        pytest.skip("The test is not configured to run in SaaS.")
+
+    deploy_command = create_deploy_command(StdTags.SAAS)
+    extra_cli_args = {StdParams.container_file.name: container_path}
+    arg_string = make_args_string(**saas_cli_args, **slc_cli_args, **extra_cli_args)
+    run_deploy_command(deploy_command, arg_string, language_alias, db_schema, saas_cli_args)