From a20de09dcfb3083673f1b6a0d86806c88084610f Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 14 Aug 2024 23:41:59 +0200 Subject: [PATCH 1/3] kraken-build/: fix: Fix `dist()` function not using `TaskSet` correctly (regression introduced in 0.37.0, 8840ec5), add `TaskSet.build()` to compensate --- kraken-build/.changelog/_unreleased.toml | 5 ++ .../src/kraken/core/system/project_test.py | 64 +------------------ kraken-build/src/kraken/core/system/task.py | 47 +++++++++++++- .../src/kraken/core/system/task_test.py | 56 +++++++++++++++- kraken-build/src/kraken/std/dist.py | 2 +- kraken-build/src/kraken/std/dist_test.py | 34 ++++++++++ 6 files changed, 141 insertions(+), 67 deletions(-) create mode 100644 kraken-build/.changelog/_unreleased.toml create mode 100644 kraken-build/src/kraken/std/dist_test.py diff --git a/kraken-build/.changelog/_unreleased.toml b/kraken-build/.changelog/_unreleased.toml new file mode 100644 index 00000000..d8fa7a73 --- /dev/null +++ b/kraken-build/.changelog/_unreleased.toml @@ -0,0 +1,5 @@ +[[entries]] +id = "cf6a4820-5e32-47fe-beb1-206329e6bc11" +type = "fix" +description = "Fix `dist()` function not using `TaskSet` correctly (regression introduced in 0.37.0, 8840ec5), add `TaskSet.build()` to compensate" +author = "@NiklasRosenstein" diff --git a/kraken-build/src/kraken/core/system/project_test.py b/kraken-build/src/kraken/core/system/project_test.py index c705610b..63b4f308 100644 --- a/kraken-build/src/kraken/core/system/project_test.py +++ b/kraken-build/src/kraken/core/system/project_test.py @@ -1,59 +1,7 @@ -from dataclasses import dataclass - import pytest from kraken.core.system.project import Project -from kraken.core.system.property import Property -from kraken.core.system.task import Task, TaskSet, VoidTask - - -@dataclass -class MyDescriptor: - name: str - - -def test__Project__resolve_outputs__can_find_dataclass_in_metadata(kraken_project: Project) -> None: - kraken_project.task("carrier", VoidTask).outputs.append(MyDescriptor("foobar")) - assert list(TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).all()) == [ - MyDescriptor("foobar") - ] - - -def test__Project__resolve_outputs__can_find_dataclass_in_properties(kraken_project: Project) -> None: - class MyTask(Task): - out_prop: Property[MyDescriptor] = Property.output() - - def execute(self) -> None: ... - - task = kraken_project.task("carrier", MyTask) - task.out_prop = MyDescriptor("foobar") - assert list(TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).all()) == [ - MyDescriptor("foobar") - ] - - -def test__Project__resolve_outputs__can_not_find_input_property(kraken_project: Project) -> None: - class MyTask(Task): - out_prop: Property[MyDescriptor] - - def execute(self) -> None: ... - - task = kraken_project.task("carrier", MyTask) - task.out_prop = MyDescriptor("foobar") - assert list(TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).all()) == [] - - -def test__Project__resolve_outputs_supplier(kraken_project: Project) -> None: - class MyTask(Task): - out_prop: Property[MyDescriptor] = Property.output() - - def execute(self) -> None: ... - - task = kraken_project.task("carrier", MyTask) - task.out_prop = MyDescriptor("foobar") - assert TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).supplier().get() == [ - MyDescriptor("foobar") - ] +from kraken.core.system.task import VoidTask def test__Project__do_normalizes_taskname_backwards_compatibility_pre_0_12_0(kraken_project: Project) -> None: @@ -66,13 +14,3 @@ def test__Project__do_normalizes_taskname_backwards_compatibility_pre_0_12_0(kra "Starting with kraken-core 0.12.0, Task names must follow a stricter naming convention subject to the " "Address class' validation (must match /^[a-zA-Z0-9/_\\-\\.\\*]+$/)." ) - - -def test__Project__do__does_not_set_property_on_None_value(kraken_project: Project) -> None: - class MyTask(Task): - in_prop: Property[str] - - def execute(self) -> None: ... - - kraken_project.task("carrier", MyTask) - assert TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(str).supplier().get() == [] diff --git a/kraken-build/src/kraken/core/system/task.py b/kraken-build/src/kraken/core/system/task.py index d5605e2e..effa9811 100644 --- a/kraken-build/src/kraken/core/system/task.py +++ b/kraken-build/src/kraken/core/system/task.py @@ -23,6 +23,7 @@ from kraken.core.system.task_supplier import TaskSupplier if TYPE_CHECKING: + from kraken.core.system.context import Context from kraken.core.system.project import Project else: # Type hint evaluation in typeapi tries to fully resolve forward references to a type. In order to allow the @@ -567,6 +568,43 @@ def teardown(self) -> None: class TaskSet(Collection[Task]): """Represents a collection of tasks.""" + @staticmethod + def build( + context: Context | Project, + selector: str | Address | Task | Iterable[str | Address | Task], + project: Project | None = None, + ) -> TaskSet: + """ + For each item in *selector*, resolve tasks using [`Context.resolve_tasks()`]. If a selector is a string, + assign the resolved tasks to a partition by that selector value. + + Args: + context: A Kraken context or project to resolve the *selector* in. If it is a project, string selectors + are treated relative to the project. + selector: A single selector string or task, or a sequence thereof. Note that selectors of type [`Address`] + are converted to string partitions. + """ + + from kraken.core.system.project import Project + + if isinstance(context, Project): + project = context + context = context.context + else: + project = None + + if isinstance(selector, (str, Address, Task)): + selector = [selector] + + result = TaskSet() + for item in selector: + if isinstance(item, (str, Address)): + result.add(context.resolve_tasks([item], project), partition=str(item)) + else: + result.add([item]) + + return result + def __init__(self, tasks: Iterable[Task] = ()) -> None: self._tasks = set(tasks) self._partition_to_task_map: dict[str, set[Task]] = {} @@ -579,7 +617,7 @@ def __len__(self) -> int: return len(self._tasks) def __repr__(self) -> str: - return f"TaskSet(length={len(self._tasks)})" + return f"TaskSet(length={len(self._tasks)}, pttm={self._partition_to_task_map}, ttpm={self._task_to_partition_map})" def __contains__(self, __x: object) -> bool: return __x in self._tasks @@ -658,11 +696,16 @@ def __iter__(self) -> Iterable[str]: @overload def __getitem__(self, partition: str) -> Collection[Task]: ... + @overload + def __getitem__(self, partition: Address) -> Collection[Task]: ... + @overload def __getitem__(self, partition: Task) -> Collection[str]: ... - def __getitem__(self, partition: str | Task) -> Collection[str] | Collection[Task]: + def __getitem__(self, partition: str | Address | Task) -> Collection[str] | Collection[Task]: if isinstance(partition, str): return self._ptt.get(partition) or () + elif isinstance(partition, Address): + return self._ptt.get(str(partition)) or () else: return self._ttp.get(partition) or () diff --git a/kraken-build/src/kraken/core/system/task_test.py b/kraken-build/src/kraken/core/system/task_test.py index a68060e7..d32dc3c0 100644 --- a/kraken-build/src/kraken/core/system/task_test.py +++ b/kraken-build/src/kraken/core/system/task_test.py @@ -1,8 +1,9 @@ +from dataclasses import dataclass from pytest import raises from kraken.core.system.project import Project from kraken.core.system.property import Property -from kraken.core.system.task import Task, TaskRelationship +from kraken.core.system.task import Task, TaskRelationship, TaskSet, VoidTask def test__Task__get_relationships_lineage_through_properties(kraken_project: Project) -> None: @@ -44,3 +45,56 @@ def execute(self) -> None: with raises(TypeError) as excinfo: t1.b.set(42.0) # type: ignore[arg-type] assert str(excinfo.value) == "Property(MyTask(:t1).b): expected int, got float\nexpected str, got float" + + +@dataclass +class MyDescriptor: + name: str + + +def test__TaskSet__resolve_outputs__can_find_dataclass_in_metadata(kraken_project: Project) -> None: + kraken_project.task("carrier", VoidTask).outputs.append(MyDescriptor("foobar")) + assert list(TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).all()) == [MyDescriptor("foobar")] + + +def test__TaskSet__resolve_outputs__can_find_dataclass_in_properties(kraken_project: Project) -> None: + class MyTask(Task): + out_prop: Property[MyDescriptor] = Property.output() + + def execute(self) -> None: ... + + task = kraken_project.task("carrier", MyTask) + task.out_prop = MyDescriptor("foobar") + assert list(TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).all()) == [MyDescriptor("foobar")] + + +def test__TaskSet__resolve_outputs__can_not_find_input_property(kraken_project: Project) -> None: + class MyTask(Task): + out_prop: Property[MyDescriptor] + + def execute(self) -> None: ... + + task = kraken_project.task("carrier", MyTask) + task.out_prop = MyDescriptor("foobar") + assert list(TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).all()) == [] + + +def test__TaskSet__resolve_outputs_supplier(kraken_project: Project) -> None: + class MyTask(Task): + out_prop: Property[MyDescriptor] = Property.output() + + def execute(self) -> None: ... + + task = kraken_project.task("carrier", MyTask) + task.out_prop = MyDescriptor("foobar") + assert TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).supplier().get() == [MyDescriptor("foobar")] + + +def test__TaskSet__do__does_not_set_property_on_None_value(kraken_project: Project) -> None: + class MyTask(Task): + in_prop: Property[str] + + def execute(self) -> None: ... + + kraken_project.task("carrier", MyTask) + assert TaskSet.build(kraken_project, ":carrier").select(str).supplier().get() == [] diff --git a/kraken-build/src/kraken/std/dist.py b/kraken-build/src/kraken/std/dist.py index a6f30def..f112aa29 100644 --- a/kraken-build/src/kraken/std/dist.py +++ b/kraken-build/src/kraken/std/dist.py @@ -232,7 +232,7 @@ def dist( k: databind.json.load(v, IndividualDistOptions) if not isinstance(v, IndividualDistOptions) else v for k, v in dependencies.items() } - dependencies_set = TaskSet(project.context.resolve_tasks(dependencies_map, project)) + dependencies_set = TaskSet.build(project, dependencies_map) # This associates the IndividualDistOptions specified in *dependencies* to the Resource(s) # provided by the task(s). diff --git a/kraken-build/src/kraken/std/dist_test.py b/kraken-build/src/kraken/std/dist_test.py new file mode 100644 index 00000000..d1738b1b --- /dev/null +++ b/kraken-build/src/kraken/std/dist_test.py @@ -0,0 +1,34 @@ +import tarfile +from kraken.core import Project, Task, Property +from kraken.core.system.task import TaskStatus +from kraken.std.dist import dist +from kraken.std.descriptors.resource import Resource + + +def test_dist(kraken_project: Project) -> None: + """ + This function validates that the #dist() function works as intended with mapping individual options + to the resources provided by dependencies. + """ + + class ProducerTask(Task): + result: Property[Resource] = Property.output() + + def execute(self) -> TaskStatus | None: + output_file = self.project.build_directory / "product.txt" + output_file.write_text("Hello, World!") + self.result = Resource(name="file", path=output_file) + return None + + kraken_project.task("producer", ProducerTask) + + output_archive = kraken_project.build_directory / "archive.tgz" + dist(name="dist", dependencies={"producer": {"arcname": "result.txt"}}, output_file=output_archive) + + kraken_project.context.execute([":dist"]) + + assert output_archive.exists() + with tarfile.open(output_archive) as tarf: + fp = tarf.extractfile("result.txt") + assert fp is not None + assert fp.read().decode() == "Hello, World!" From 701b57218c9057c6fd1956f4b9c93e37a99e7924 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 14 Aug 2024 23:53:03 +0200 Subject: [PATCH 2/3] fix kraken-wrapper test --- kraken-wrapper/tests/iss-263/dependency/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kraken-wrapper/tests/iss-263/dependency/pyproject.toml b/kraken-wrapper/tests/iss-263/dependency/pyproject.toml index 1516ef1f..16d9fc7b 100644 --- a/kraken-wrapper/tests/iss-263/dependency/pyproject.toml +++ b/kraken-wrapper/tests/iss-263/dependency/pyproject.toml @@ -6,7 +6,7 @@ authors = [ {name = "Niklas Rosenstein", email = "rosensteinniklas@gmail.com"}, ] dependencies = [] -requires-python = "==3.12.*" +requires-python = ">=3.10" readme = "README.md" license = {text = "MIT"} From 53f0ffe622a170c6cf0fd1038e2e63e4c0f55775 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Wed, 14 Aug 2024 23:55:49 +0200 Subject: [PATCH 3/3] fix pytest tasks for kraken-wrapper --- .kraken.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.kraken.py b/.kraken.py index 9f0d0463..42f9b639 100644 --- a/.kraken.py +++ b/.kraken.py @@ -17,7 +17,10 @@ def configure_project() -> None: if project.directory.joinpath("tests").is_dir(): # Explicit list of test directories, Pytest skips the build directory if not specified explicitly. - python.pytest(ignore_dirs=["src/tests/integration"], include_dirs=["src/kraken/build"]) + if project.directory.name == "kraken-build": + python.pytest(ignore_dirs=["src/tests/integration"], include_dirs=["src/kraken/build"]) + elif project.directory.name == "kraken-wrapper": + python.pytest(doctest_modules=False) if project.directory.joinpath("tests/integration").is_dir(): python.pytest(