Merge remote-tracking branch 'origin/main'

Author: ahsimb

.github/workflows/report.yml

@@ -29,6 +29,7 @@ jobs:
           path: ./artifacts
 
       - name: Copy Artifacts into Root Folder
+        if: ${{ (hashFiles('./artifacts/.coverage/.coverage') != '') && (hashFiles('./artifacts/.lint.txt/.lint.txt') != '')}}
         working-directory: ./artifacts
         run: |
           cp .coverage/.coverage ../
@@ -44,6 +45,7 @@ jobs:
           path: metrics.json
 
       - name: Generate GitHub Summary
+        if: ${{ (hashFiles('./artifacts/.coverage/.coverage') != '') && (hashFiles('./artifacts/.lint.txt/.lint.txt') != '')}}
         run: |
           echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY
           poetry run nox -s report -- -- --format markdown >> $GITHUB_STEP_SUMMARY

doc/changes/changelog.md

@@ -1,6 +1,7 @@
 # 📝 Changes
 
 * [unreleased](unreleased.md)
+* [0.5.0](changes_0.5.0.md)
 * [0.4.0](changes_0.4.0.md)
 * [0.3.1](changes_0.3.1.md)
 * [0.3.0](changes_0.3.0.md)
@@ -12,6 +13,7 @@
 hidden:
 ---
 unreleased
+changes_0.5.0
 changes_0.4.0
 changes_0.3.1
 changes_0.3.0

doc/changes/changes_0.5.0.md

@@ -0,0 +1,29 @@
+# 0.5.0 - 2024-09-19
+
+This release prepares validation of Script Language Containers (SLC) uploaded to BucketFS as a file archive, e.g. with extension `tar.gz`.
+
+Validation is implemented by
+* activating the uploaded SLC,
+* running a UDF inside it,
+* and making the UDF check for a specific file having been extracted from the SLC archive to be available on each node of the databasecluster.
+
+Addtionally this release refactors the existing CLI tests for the `LanguageContainerDeployer` which were integration tests involving the whole chain from the CLI down to the API, starting a database and uploading and activating SLCs.
+
+The existing integration tests have been split into
+* either unit tests just verifiying that the CLI options are passed to the API
+* or ordinary integration tests not using the CLI.
+
+This enables faster and more robust tests for the pure CLI-related features, faster turnaounds during development, and separation of concerns.
+
+## Features
+
+* #50: Created new implementation `ExtractValidator` for validating extraction of
+* #49: Integrated new `ExtractValidator` into `LanguageContainerDeployer`
+
+# Refactoring
+
+* #51: Split CLI integration tests
+
+## Bug Fixing
+
+* #60: Fix handling pip requirements when creating an SLC

doc/changes/unreleased.md

@@ -1,25 +1 @@
 # Unreleased
-
-This release prepares validation of Script Language Containers (SLC) uploaded to BucketFS as a file archive, e.g. with extension `tar.gz`.
-
-Validation is implemented by
-* activating the uploaded SLC,
-* running a UDF inside it,
-* and making the UDF check for a specific file having been extracted from the SLC archive to be available on each node of the databasecluster.
-
-Addtionally this release refactors the existing CLI tests for the `LanguageContainerDeployer` which were integration tests involving the whole chain from the CLI down to the API, starting a database and uploading and activating SLCs.
-
-The existing integration tests have been split into
-* either unit tests just verifiying that the CLI options are passed to the API
-* or ordinary integration tests not using the CLI.
-
-This enables faster and more robust tests for the pure CLI-related features, faster turnaounds during development, and separation of concerns.
-
-## Features
-
-* #50: Created new implementation `ExtractValidator` for validating extraction of
-* #49: Integrated new `ExtractValidator` into `LanguageContainerDeployer`
-
-# Refactoring
-
-* #51: Split CLI integration tests

exasol/python_extension_common/deployment/language_container/flavor_base/build_steps.py

@@ -1,6 +1,4 @@
-from pathlib import Path
-from typing import Dict
-
+# pylint: disable=import-error
 from exasol_script_languages_container_tool.lib.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask    # type: ignore
 
 
@@ -14,6 +12,7 @@ def requires_tasks(self):
     def get_path_in_flavor(self):
         return "flavor_base"
 
+
 class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
     def get_build_step(self) -> str:
         return "release"

exasol/python_extension_common/deployment/language_container_builder.py

@@ -6,6 +6,7 @@
 from pathlib import Path
 from importlib import resources
 
+# pylint: disable=import-error
 from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo   # type: ignore
 from exasol_script_languages_container_tool.lib import api            # type: ignore
 from exasol_script_languages_container_tool.lib.tasks.export.export_containers import ExportContainerResult     # type: ignore
@@ -94,9 +95,17 @@ def write_file(self, file_name: str | Path, content: str) -> None:
         file_path.write_text(content)
 
     @property
-    def flavor_base(self):
+    def flavor_base(self) -> Path:
         return self.flavor_path / "flavor_base"
 
+    @property
+    def requirements_file(self) -> Path:
+        return self.flavor_base / "dependencies" / "requirements.txt"
+
+    @property
+    def wheel_target(self) -> Path:
+        return self.flavor_base / "release" / "dist"
+
     def prepare_flavor(self, project_directory: str | Path,
                        requirement_filter: Callable[[str], bool] | None = None):
         """
@@ -144,19 +153,22 @@ def _set_language_alias(self) -> None:
     def _add_requirements_to_flavor(self, project_directory: str | Path,
                                     requirement_filter: Callable[[str], bool] | None):
         """
-        Create the project's requirements.txt.
+        Adds project's requirements to the requirements.txt file. Creates this file
+        if it doesn't exist.
         """
         assert self._root_path is not None
-        dist_path = self._root_path / "requirements.txt"
         requirements_bytes = subprocess.check_output(["poetry", "export",
-                                                      "--without-hashes", "--without-urls",
-                                                      "--output", f'{dist_path}'],
+                                                      "--without-hashes", "--without-urls"],
                                                      cwd=str(project_directory))
         requirements = requirements_bytes.decode("UTF-8")
         if requirement_filter is not None:
             requirements = "\n".join(filter(requirement_filter, requirements.splitlines()))
-        requirements_file = self.flavor_base / "dependencies" / "requirements.txt"
-        requirements_file.write_text(requirements)
+        # Make sure the content ends with a new line, so that other requirements can be
+        # added at the end of it.
+        if not requirements.endswith('\n'):
+            requirements += '\n'
+        with self.requirements_file.open(mode='a') as f:
+            return f.write(requirements)
 
     def _add_wheel_to_flavor(self, project_directory: str | Path):
         """
@@ -176,6 +188,5 @@ def _add_wheel_to_flavor(self, project_directory: str | Path):
             raise RuntimeError(f"Did not find exactly one wheel file in dist directory {dist_path}. "
                                f"Found the following wheels: {wheels}")
         wheel = wheels[0]
-        wheel_target = self.flavor_base / "release" / "dist"
-        wheel_target.mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(wheel, wheel_target / wheel.name)
+        self.wheel_target.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(wheel, self.wheel_target / wheel.name)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-python-extension-common"
-version = "0.4.0"
+version = "0.5.0"
 description = "A collection of common utilities for Exasol extensions."
 packages = [ {include = "exasol"}, ]
 authors = ["Mikhail Beck <[email protected]>"]

test/integration/test_language_container_builder.py

@@ -16,6 +16,30 @@
 )
 
 
+def test_prepare_flavor(tmp_path):
+    project_directory = find_path_backwards("pyproject.toml", __file__).parent
+
+    with LanguageContainerBuilder('test_container', TEST_LANGUAGE_ALIAS) as container_builder:
+        container_builder.prepare_flavor(project_directory)
+        assert container_builder.requirements_file.exists()
+        assert container_builder.requirements_file.stat().st_size > 0
+        assert container_builder.wheel_target.exists()
+        assert container_builder.wheel_target.is_dir()
+        assert any(container_builder.wheel_target.iterdir())
+
+
+def test_prepare_flavor_extra(tmp_path):
+    """Tests that requirements from multiple projects can be added together"""
+    project_directory = find_path_backwards("pyproject.toml", __file__).parent
+    dummy_req = 'xyz\n'
+    with LanguageContainerBuilder('test_container', TEST_LANGUAGE_ALIAS) as container_builder:
+        container_builder.requirements_file.write_text(dummy_req)
+        container_builder.prepare_flavor(project_directory)
+        assert container_builder.requirements_file.exists()
+        assert container_builder.requirements_file.stat().st_size > len(dummy_req)
+        assert container_builder.requirements_file.read_text().startswith(dummy_req)
+
+
 def test_language_container_builder(itde: config.TestConfig,
                                     connection_factory: Callable[[config.Exasol], ExaConnection],
                                     tmp_path):

version.py

@@ -5,6 +5,6 @@
 # Do not edit this file manually!
 # If you need to change the version, do so in the project.toml, e.g. by using `poetry version X.Y.Z`.
 MAJOR = 0
-MINOR = 4
+MINOR = 5
 PATCH = 0
 VERSION = f"{MAJOR}.{MINOR}.{PATCH}"