Merge remote-tracking branch 'origin/main'

Author: ahsimb

doc/changes/unreleased.md

@@ -3,3 +3,4 @@
 ## Features
 
 * #42: Optionally wait until SLC is deployed to all nodes in the database cluster
+* #45: Moving the SCL build and export from the TE to PEC.

exasol/python_extension_common/connections/pyexasol_connection.py

@@ -53,7 +53,7 @@ def open_pyexasol_connection(
                                                            pat=saas_token)
     else:
         raise ValueError('Incomplete parameter list. '
-                         'Please either provide the parameters [dns, db_user, db_pass] '
+                         'Please either provide the parameters [dsn, db_user, db_pass] '
                          'for an On-Prem database or [saas_url, saas_account_id, '
                          'saas_database_id or saas_database_name, saas_token] '
                          'for a SaaS database.')

exasol/python_extension_common/deployment/__init__.py

@@ -3,15 +3,12 @@
 import pyexasol     # type: ignore
 
 from datetime import datetime, timedelta
+from textwrap import dedent
 from typing import Callable, List
 from tenacity import Retrying
 from tenacity.wait import wait_fixed
 from tenacity.stop import stop_after_delay
 
-from exasol.python_extension_common.deployment.language_container_validator import (
-    temp_schema
-)
-
 MANIFEST_FILE = "exasol-manifest.json"
 
 
@@ -74,8 +71,7 @@ def _create_manifest_udf(self, language_alias: str, udf_name: str):
         Much more a later statement "CREATE SCRIPT" will fail with an error
         message. Hence we need to use a retry here, as well.
         """
-        self._pyexasol_conn.execute(
-            f"""
+        self._pyexasol_conn.execute(dedent(f"""
             CREATE OR REPLACE {language_alias} SET SCRIPT
                 {udf_name}(my_path VARCHAR(256))
                 EMITS (node INTEGER, manifest BOOL) AS
@@ -84,7 +80,7 @@ def run(ctx):
                 ctx.emit(exa.meta.node_id, os.path.isfile(ctx.my_path))
             /
             """
-        )
+        ))
 
     def _check_all_nodes_with_retry(self, udf_name: str, nproc: int, manifest: str, timeout: timedelta):
         for attempt in Retrying(
@@ -97,7 +93,7 @@ def _check_all_nodes_with_retry(self, udf_name: str, nproc: int, manifest: str,
     def _check_all_nodes(self, udf_name: str, nproc: int, manifest: str):
         result = self._pyexasol_conn.execute(
             f"""
-            SELECT {udf_name}({manifest})
+            SELECT {udf_name}('{manifest}')
             FROM VALUES BETWEEN 1 AND {nproc} t(i) GROUP BY i
             """
         ).fetchall()
@@ -117,8 +113,9 @@ def verify_all_nodes(self, schema: str, language_alias: str, bfs_archive_path: b
         still nodes pending, for which the extraction could not be verified,
         yet.
         """
-        manifest = f"{bfs_archive_path.as_udf_path()}/{MANIFEST_FILE}"
-        nproc = self._pyexasol_conn.execute("SELECT nproc()").fetchone()
+        # manifest = f"{bfs_archive_path.as_udf_path()}/{MANIFEST_FILE}"
+        manifest = "/exaudf/exaudfclient_py3"
+        nproc = self._pyexasol_conn.execute("SELECT nproc()").fetchone()[0]
         udf_name = _udf_name(schema, language_alias)
         start = datetime.now()
         try:

exasol/python_extension_common/deployment/extract_validator.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+from typing import Dict
+
+from exasol_script_languages_container_tool.lib.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask    # type: ignore
+
+
+class AnalyzeDependencies(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "dependencies"
+
+    def requires_tasks(self):
+        return {}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "release"
+
+    def requires_tasks(self):
+        return {"dependencies":  AnalyzeDependencies}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"

exasol/python_extension_common/deployment/language_container/flavor_base/build_steps.py

@@ -0,0 +1 @@
+requirements.txt

exasol/python_extension_common/deployment/language_container/flavor_base/dependencies/.gitignore

@@ -0,0 +1,5 @@
+FROM exasol/script-language-container:template-Exasol-all-python-3.10-release_BFRSH344TDRPT7LK2FBOJK4KBIDW6A253FFPYEUYT4O2ERFMTCNA
+
+Run mkdir /project
+COPY dependencies/requirements.txt /project/requirements.txt
+RUN python3.10 -m pip install -r /project/requirements.txt

exasol/python_extension_common/deployment/language_container/flavor_base/dependencies/Dockerfile

@@ -0,0 +1 @@
+PYTHON3_FLAVOR=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3

exasol/python_extension_common/deployment/language_container/flavor_base/language_definition

@@ -0,0 +1 @@
+dist

exasol/python_extension_common/deployment/language_container/flavor_base/release/.gitignore

@@ -0,0 +1,8 @@
+FROM {{ dependencies }}
+
+COPY release/dist /project/dist
+RUN python3.10 -m pip install --no-deps /project/dist/*.whl
+
+RUN mkdir -p /build_info/actual_installed_packages/release && \
+    /scripts/list_installed_scripts/list_installed_apt.sh > /build_info/actual_installed_packages/release/apt_get_packages && \
+    /scripts/list_installed_scripts/list_installed_pip.sh python3.10 > /build_info/actual_installed_packages/release/python3_pip_packages