|
| 1 | +from __future__ import annotations |
| 2 | +from typing import Callable |
| 3 | +import shutil |
| 4 | +import subprocess |
| 5 | +import tempfile |
| 6 | +from pathlib import Path |
| 7 | +from importlib import resources |
| 8 | + |
| 9 | +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo # type: ignore |
| 10 | +from exasol_script_languages_container_tool.lib import api # type: ignore |
| 11 | +from exasol_script_languages_container_tool.lib.tasks.export.export_containers import ExportContainerResult # type: ignore |
| 12 | + |
| 13 | + |
| 14 | +def exclude_cuda(line: str) -> bool: |
| 15 | + return not line.startswith("nvidia") |
| 16 | + |
| 17 | + |
| 18 | +def find_path_backwards(target_path: str | Path, start_path: str | Path) -> Path: |
| 19 | + """ |
| 20 | + An utility searching for a specified path backwards. It begins with the given start |
| 21 | + path and checks if the target path is among its siblings. Then it moves to the parent |
| 22 | + path and so on, until it reaches the root of the file structure. Raises a FileNotFound |
| 23 | + error if the search is unsuccessful. |
| 24 | + """ |
| 25 | + current_path = Path(start_path).parent |
| 26 | + while current_path != current_path.root: |
| 27 | + result_path = Path(current_path, target_path) |
| 28 | + if result_path.exists(): |
| 29 | + return result_path |
| 30 | + current_path = current_path.parent |
| 31 | + raise FileNotFoundError(f"Could not find {target_path} when searching backwards from {start_path}") |
| 32 | + |
| 33 | + |
| 34 | +def copy_slc_flavor(dest_dir: str | Path) -> None: |
| 35 | + """ |
| 36 | + Copies the content of the language_container directory to the specified |
| 37 | + destination directory. |
| 38 | + """ |
| 39 | + files = resources.files(__package__).joinpath('language_container') |
| 40 | + with resources.as_file(files) as pkg_dir: |
| 41 | + shutil.copytree(pkg_dir, dest_dir, dirs_exist_ok=True) |
| 42 | + |
| 43 | + |
| 44 | +class LanguageContainerBuilder: |
| 45 | + |
| 46 | + def __init__(self, container_name: str, language_alias: str): |
| 47 | + self.container_name = container_name |
| 48 | + self.language_alias = language_alias |
| 49 | + self._root_path: Path | None = None |
| 50 | + self._output_path: Path | None = None |
| 51 | + |
| 52 | + def __enter__(self): |
| 53 | + |
| 54 | + # Create a temporary working directory |
| 55 | + self._root_path = Path(tempfile.mkdtemp()) |
| 56 | + self.flavor_path = self._root_path / self.container_name |
| 57 | + |
| 58 | + # Copy the flavor into the working directory |
| 59 | + copy_slc_flavor(self.flavor_path) |
| 60 | + |
| 61 | + # Write the language alias to the language definition |
| 62 | + self._set_language_alias() |
| 63 | + return self |
| 64 | + |
| 65 | + def __exit__(self, *exc_details): |
| 66 | + |
| 67 | + # Delete all local docker images. |
| 68 | + if self._output_path is not None: |
| 69 | + api.clean_all_images(output_directory=str(self._output_path)) |
| 70 | + self._output_path = None |
| 71 | + |
| 72 | + # Remove the temporary directory recursively |
| 73 | + if self._root_path is not None: |
| 74 | + shutil.rmtree(self._root_path, ignore_errors=True) |
| 75 | + self._root_path = None |
| 76 | + |
| 77 | + def read_file(self, file_name: str | Path) -> str: |
| 78 | + """ |
| 79 | + Reads the content of the specified file in the flavor directory. |
| 80 | + The provided file name should be relative to the flavor directory, e.g. |
| 81 | + flavor_base/dependencies/Dockerfile |
| 82 | + """ |
| 83 | + file_path = self.flavor_path.joinpath(file_name) |
| 84 | + return file_path.read_text() |
| 85 | + |
| 86 | + def write_file(self, file_name: str | Path, content: str) -> None: |
| 87 | + """ |
| 88 | + Replaces the content of the specified file in the flavor directory. |
| 89 | + This allows making modifications to the standard flavor. |
| 90 | + The provided file name should be relative to the flavor directory, e.g. |
| 91 | + flavor_base/dependencies/Dockerfile |
| 92 | + """ |
| 93 | + file_path = self.flavor_path.joinpath(file_name) |
| 94 | + file_path.write_text(content) |
| 95 | + |
| 96 | + @property |
| 97 | + def flavor_base(self): |
| 98 | + return self.flavor_path / "flavor_base" |
| 99 | + |
| 100 | + def prepare_flavor(self, project_directory: str | Path, |
| 101 | + requirement_filter: Callable[[str], bool] | None = None): |
| 102 | + """ |
| 103 | + Create the project's requirements.txt and the distribution wheel. |
| 104 | + """ |
| 105 | + self._add_requirements_to_flavor(project_directory, requirement_filter) |
| 106 | + self._add_wheel_to_flavor(project_directory) |
| 107 | + |
| 108 | + def build(self) -> dict[str, ImageInfo]: |
| 109 | + """ |
| 110 | + Builds the new script language container. |
| 111 | + """ |
| 112 | + image_info = api.build(flavor_path=(str(self.flavor_path),), goal=("release",)) |
| 113 | + return image_info |
| 114 | + |
| 115 | + def export(self, export_path: str | Path | None = None) -> ExportContainerResult: |
| 116 | + """ |
| 117 | + Exports the container into an archive. |
| 118 | + """ |
| 119 | + assert self._root_path is not None |
| 120 | + if not export_path: |
| 121 | + export_path = self._root_path / '.export' |
| 122 | + if not export_path.exists(): |
| 123 | + export_path.mkdir() |
| 124 | + if self._output_path is None: |
| 125 | + self._output_path = self._root_path / '.output' |
| 126 | + if not self._output_path.exists(): |
| 127 | + self._output_path.mkdir() |
| 128 | + |
| 129 | + export_result = api.export(flavor_path=(str(self.flavor_path),), |
| 130 | + output_directory=str(self._output_path), |
| 131 | + export_path=str(export_path)) |
| 132 | + return export_result |
| 133 | + |
| 134 | + def _set_language_alias(self) -> None: |
| 135 | + """ |
| 136 | + Sets the language alias provided in the constractor to the language definition. |
| 137 | + """ |
| 138 | + lang_def_path = self.flavor_base / 'language_definition' |
| 139 | + lang_def_template = lang_def_path.read_text() |
| 140 | + lang_def_text = lang_def_template.split("=", maxsplit=1)[1] |
| 141 | + lang_def = f'{self.language_alias}={lang_def_text}' |
| 142 | + lang_def_path.write_text(lang_def) |
| 143 | + |
| 144 | + def _add_requirements_to_flavor(self, project_directory: str | Path, |
| 145 | + requirement_filter: Callable[[str], bool] | None): |
| 146 | + """ |
| 147 | + Create the project's requirements.txt. |
| 148 | + """ |
| 149 | + assert self._root_path is not None |
| 150 | + dist_path = self._root_path / "requirements.txt" |
| 151 | + requirements_bytes = subprocess.check_output(["poetry", "export", |
| 152 | + "--without-hashes", "--without-urls", |
| 153 | + "--output", f'{dist_path}'], |
| 154 | + cwd=str(project_directory)) |
| 155 | + requirements = requirements_bytes.decode("UTF-8") |
| 156 | + if requirement_filter is not None: |
| 157 | + requirements = "\n".join(filter(requirement_filter, requirements.splitlines())) |
| 158 | + requirements_file = self.flavor_base / "dependencies" / "requirements.txt" |
| 159 | + requirements_file.write_text(requirements) |
| 160 | + |
| 161 | + def _add_wheel_to_flavor(self, project_directory: str | Path): |
| 162 | + """ |
| 163 | + Create the project's distribution wheel. |
| 164 | + """ |
| 165 | + assert self._root_path is not None |
| 166 | + # A newer version of poetry would allow using the --output parameter in |
| 167 | + # the build command. Then we could build the wheel in a temporary directory. |
| 168 | + # With the version currently used in the Python Toolbox we have to do this |
| 169 | + # inside the project. |
| 170 | + dist_path = Path(project_directory) / "dist" |
| 171 | + if dist_path.exists(): |
| 172 | + shutil.rmtree(dist_path) |
| 173 | + subprocess.call(["poetry", "build"], cwd=str(project_directory)) |
| 174 | + wheels = list(dist_path.glob("*.whl")) |
| 175 | + if len(wheels) != 1: |
| 176 | + raise RuntimeError(f"Did not find exactly one wheel file in dist directory {dist_path}. " |
| 177 | + f"Found the following wheels: {wheels}") |
| 178 | + wheel = wheels[0] |
| 179 | + wheel_target = self.flavor_base / "release" / "dist" |
| 180 | + wheel_target.mkdir(parents=True, exist_ok=True) |
| 181 | + shutil.copyfile(wheel, wheel_target / wheel.name) |
0 commit comments