#239: Changed return type for deploy API method (#240)

fixes #239

Author: tomuben

exasol/slc/api/deploy.py

@@ -1,7 +1,6 @@
 import getpass
-from typing import Optional, Tuple
+from typing import Dict, Optional, Tuple
 
-import luigi
 from exasol_integration_test_docker_environment.lib.api.common import (
     cli_function,
     generate_root_task,
@@ -14,7 +13,9 @@
     DependencyLoggerBaseTask,
 )
 
-from exasol.slc.internal.tasks.upload.upload_containers import UploadContainers
+from exasol.slc.internal.tasks.upload.deploy_containers import DeployContainers
+from exasol.slc.internal.tasks.upload.deploy_info import toDeployResult
+from exasol.slc.models.deploy_result import DeployResult
 
 
 @cli_function
@@ -52,13 +53,14 @@ def deploy(
     task_dependencies_dot_file: Optional[str] = None,
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
-) -> luigi.LocalTarget:
+) -> Dict[str, Dict[str, DeployResult]]:
     """
     This command uploads the whole script-language-container package of the flavor to the database.
-    If the stages or the packaged container do not exists locally, the system will build, pull or
+    If the stages or the packaged container do not exist locally, the system will build, pull or
     export them before the upload.
     :raises api_errors.TaskFailureError: if operation is not successful.
-    :return: Path to resulting report file.
+    :return: A dictionary with an instance of class DeployResult for each release for each deployed flavor.
+             For example { "flavors/standard-flavor" : {"release" : DeployResult(...) } }
     """
     import_build_steps(flavor_path)
     set_build_config(
@@ -94,7 +96,7 @@ def deploy(
 
     def root_task_generator() -> DependencyLoggerBaseTask:
         return generate_root_task(
-            task_class=UploadContainers,
+            task_class=DeployContainers,
             flavor_paths=list(flavor_path),
             release_goals=list(release_goal),
             database_host=bucketfs_host,
@@ -110,10 +112,30 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             use_ssl_cert_validation=use_ssl_cert_validation,
         )
 
-    return run_task(
+    deploy_infos = run_task(
         root_task_generator,
         workers=workers,
         task_dependencies_dot_file=task_dependencies_dot_file,
         log_level=log_level,
         use_job_specific_log_file=use_job_specific_log_file,
     )
+
+    return {
+        flavor: {
+            release: toDeployResult(
+                deploy_info=deploy_info,
+                bucketfs_use_https=bucketfs_use_https,
+                bucketfs_host=bucketfs_host,
+                bucketfs_port=bucketfs_port,
+                bucket_name=bucket,
+                bucketfs_name=bucketfs_name,
+                bucketfs_username=bucketfs_user,
+                bucketfs_password=bucketfs_password,
+                ssl_cert_path=ssl_cert_path,
+                use_ssl_cert_validation=use_ssl_cert_validation,
+                path_in_bucket=path_in_bucket,
+            )
+            for release, deploy_info in deploy_info_per_release.items()
+        }
+        for flavor, deploy_info_per_release in deploy_infos.items()
+    }

exasol/slc/internal/tasks/upload/deploy_container_base_task.py

@@ -0,0 +1,130 @@
+from pathlib import Path
+
+import exasol.bucketfs as bfs  # type: ignore
+import luigi
+from exasol_integration_test_docker_environment.abstract_method_exception import (
+    AbstractMethodException,
+)
+from exasol_integration_test_docker_environment.lib.base.flavor_task import (
+    FlavorBaseTask,
+)
+
+from exasol.slc.internal.tasks.upload.deploy_info import DeployInfo
+from exasol.slc.internal.tasks.upload.language_def_parser import (
+    parse_language_definition,
+)
+from exasol.slc.internal.tasks.upload.language_definition import LanguageDefinition
+from exasol.slc.internal.tasks.upload.upload_container_parameter import (
+    UploadContainerParameter,
+)
+from exasol.slc.models.export_info import ExportInfo
+from exasol.slc.models.language_definition_components import (
+    LanguageDefinitionComponents,
+)
+from exasol.slc.models.language_definitions_builder import LanguageDefinitionsBuilder
+
+
+class DeployContainerBaseTask(FlavorBaseTask, UploadContainerParameter):
+    # TODO check if upload was successfull by requesting the file
+    # TODO add error checks and propose reasons for the error
+    # TODO extract bucketfs interaction into own module
+
+    release_goal = luigi.Parameter()
+
+    def __init__(self, *args, **kwargs):
+        self.export_info_future = None
+        super().__init__(*args, **kwargs)
+
+    def register_required(self):
+        task = self.get_export_task()
+        self.export_info_future = self.register_dependency(task)
+
+    def get_export_task(self):
+        raise AbstractMethodException()
+
+    def run_task(self):
+        export_info = self.get_values_from_future(self.export_info_future)
+        path_in_bucket = self._upload_container(export_info)
+        language_definition = LanguageDefinition(
+            release_name=self._get_complete_release_name(export_info),
+            flavor_path=self.flavor_path,
+            bucketfs_name=self.bucketfs_name,
+            bucket_name=self.bucket_name,
+            path_in_bucket=self.path_in_bucket,
+        )
+        language_definitions = language_definition.generate_definition().split(" ")
+        language_def_components_list = list()
+        for lang_def in language_definitions:
+            alias, url = parse_language_definition(lang_def)
+            language_def_components_list.append(
+                LanguageDefinitionComponents(alias=alias, url=url)
+            )
+
+        lang_def_builder = LanguageDefinitionsBuilder(language_def_components_list)
+        try:
+            release_path = Path(export_info.cache_file).relative_to(Path("").absolute())
+        except ValueError:
+            release_path = Path(export_info.cache_file)
+
+        result = DeployInfo(
+            release_path=str(release_path),
+            complete_release_name=self._get_complete_release_name(export_info),
+            human_readable_location=self._complete_url(export_info),
+            language_definition_builder=lang_def_builder,
+        )
+        self.return_object(result)
+
+    def build_file_path_in_bucket(self, release_info: ExportInfo) -> bfs.path.PathLike:
+        backend = bfs.path.StorageBackend.onprem
+
+        complete_release_name = self._get_complete_release_name(release_info)
+        verify = self.ssl_cert_path or self.use_ssl_cert_validation
+        path_in_bucket_to_upload_path = bfs.path.build_path(
+            backend=backend,
+            url=self._url,
+            bucket_name=self.bucket_name,
+            service_name=self.bucketfs_name,
+            username=self.bucketfs_username,
+            password=self.bucketfs_password,
+            verify=verify,
+            path=self.path_in_bucket or "",
+        )
+        return path_in_bucket_to_upload_path / f"{complete_release_name}.tar.gz"
+
+    @property
+    def _url(self) -> str:
+        return f"{self._get_url_prefix()}{self.database_host}:{self.bucketfs_port}"
+
+    def _complete_url(self, export_info: ExportInfo):
+        path_in_bucket = (
+            f"{self.path_in_bucket}/" if self.path_in_bucket not in [None, ""] else ""
+        )
+        return f"{self._url}/{self.bucket_name}/{path_in_bucket}{self._get_complete_release_name(export_info)}.tar.gz"
+
+    def _upload_container(self, release_info: ExportInfo) -> bfs.path.PathLike:
+        bucket_path = self.build_file_path_in_bucket(release_info)
+        self.logger.info(
+            f"Upload {release_info.cache_file} to {self._complete_url(release_info)}"
+        )
+        with open(release_info.cache_file, "rb") as file:
+            bucket_path.write(file)
+        return bucket_path
+
+    def _get_complete_release_name(self, release_info: ExportInfo):
+        complete_release_name = f"""{release_info.name}-{release_info.release_goal}-{self._get_release_name(
+            release_info)}"""
+        return complete_release_name
+
+    def _get_release_name(self, release_info: ExportInfo):
+        if self.release_name is None:
+            release_name = release_info.hash
+        else:
+            release_name = self.release_name
+        return release_name
+
+    def _get_url_prefix(self):
+        if self.bucketfs_https:
+            url_prefix = "https://"
+        else:
+            url_prefix = "http://"
+        return url_prefix

exasol/slc/internal/tasks/upload/deploy_container_task.py

@@ -0,0 +1,38 @@
+import importlib
+
+import luigi
+from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import (
+    JsonPickleParameter,
+)
+from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import (
+    RequiredTaskInfo,
+)
+
+from exasol.slc.internal.tasks.upload.deploy_container_base_task import (
+    DeployContainerBaseTask,
+)
+
+
+class DeployContainerTask(DeployContainerBaseTask):
+    # We need to create the ExportContainerTask for UploadContainerTask dynamically,
+    # because we want to push as soon as possible after an image was build and
+    # don't want to wait for the push finishing before starting the build of depended images,
+    # but we also need to create a UploadContainerTask for each ExportContainerTask of a goal
+
+    required_task_info = JsonPickleParameter(
+        RequiredTaskInfo,
+        visibility=luigi.parameter.ParameterVisibility.HIDDEN,
+        significant=True,
+    )  # type: RequiredTaskInfo
+
+    def get_export_task(self):
+        module = importlib.import_module(
+            self.required_task_info.module_name  # pylint: disable=no-member
+        )
+        class_ = getattr(
+            module, self.required_task_info.class_name  # pylint: disable=no-member
+        )
+        instance = self.create_child_task(
+            class_, **self.required_task_info.params  # pylint: disable=no-member
+        )
+        return instance

exasol/slc/internal/tasks/upload/deploy_container_tasks_creator.py

@@ -0,0 +1,42 @@
+from typing import Dict
+
+from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_create_task import (
+    DockerCreateImageTask,
+)
+from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import (
+    RequiredTaskInfo,
+)
+
+from exasol.slc.internal.tasks.upload.deploy_container_task import DeployContainerTask
+from exasol.slc.internal.tasks.upload.upload_containers_parameter import (
+    UploadContainersParameter,
+)
+
+
+class DeployContainerTasksCreator:
+
+    def __init__(self, task: UploadContainersParameter):
+        self.task = task
+
+    def create_deploy_tasks(self, build_tasks: Dict[str, DockerCreateImageTask]):
+        return {
+            release_goal: self._create_deploy_task(release_goal, build_task)
+            for release_goal, build_task in build_tasks.items()
+        }
+
+    def _create_deploy_task(self, release_goal: str, build_task: DockerCreateImageTask):
+        required_task_info = self._create_required_task_info(build_task)
+        return self.task.create_child_task_with_common_params(  # type: ignore
+            DeployContainerTask,
+            required_task_info=required_task_info,
+            release_goal=release_goal,
+        )
+
+    @staticmethod
+    def _create_required_task_info(build_task):
+        required_task_info = RequiredTaskInfo(
+            module_name=build_task.__module__,
+            class_name=build_task.__class__.__name__,
+            params=build_task.param_kwargs,
+        )
+        return required_task_info

exasol/slc/internal/tasks/upload/deploy_containers.py

@@ -0,0 +1,63 @@
+from typing import Dict
+
+from exasol_integration_test_docker_environment.lib.base.flavor_task import (
+    FlavorsBaseTask,
+)
+
+from exasol.slc.internal.tasks.build.docker_flavor_build_base import (
+    DockerFlavorBuildBase,
+)
+from exasol.slc.internal.tasks.export.export_container_tasks_creator import (
+    ExportContainerTasksCreator,
+)
+from exasol.slc.internal.tasks.upload.deploy_container_tasks_creator import (
+    DeployContainerTasksCreator,
+)
+from exasol.slc.internal.tasks.upload.upload_containers_parameter import (
+    UploadContainersParameter,
+)
+
+
+class DeployContainers(FlavorsBaseTask, UploadContainersParameter):
+
+    def __init__(self, *args, **kwargs):
+        self.lang_def_builders_futures = None
+        super().__init__(*args, **kwargs)
+
+    def register_required(self):
+        tasks = self.create_tasks_for_flavors_with_common_params(  # type: ignore
+            DeployFlavorContainers
+        )  # type: Dict[str,DeployFlavorContainers]
+        self.lang_def_builders_futures = self.register_dependencies(tasks)
+
+    def run_task(self):
+        lang_definitionbuilders = self.get_values_from_futures(
+            self.lang_def_builders_futures
+        )
+        self.return_object(lang_definitionbuilders)
+
+
+class DeployFlavorContainers(DockerFlavorBuildBase, UploadContainersParameter):
+
+    def get_goals(self):
+        return set(self.release_goals)
+
+    def run_task(self):
+        build_tasks = self.create_build_tasks()
+
+        export_tasks = self.create_export_tasks(build_tasks)
+        deploy_tasks = self.create_deploy_tasks(export_tasks)
+
+        lang_definitions_futures = yield from self.run_dependencies(deploy_tasks)
+        language_definition = self.get_values_from_futures(lang_definitions_futures)
+        self.return_object(language_definition)
+
+    def create_deploy_tasks(self, export_tasks):
+        deploy_tasks_creator = DeployContainerTasksCreator(self)
+        deploy_tasks = deploy_tasks_creator.create_deploy_tasks(export_tasks)
+        return deploy_tasks
+
+    def create_export_tasks(self, build_tasks):
+        export_tasks_creator = ExportContainerTasksCreator(self, export_path=None)
+        export_tasks = export_tasks_creator.create_export_tasks(build_tasks)
+        return export_tasks