Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kraken-build/: fix: Fix dist() function not using TaskSet correctly (regression introduced in 0.37.0, 8840ec5), add TaskSet.build() to compensate #279

Merged
merged 3 commits into from
Aug 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .kraken.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ def configure_project() -> None:

if project.directory.joinpath("tests").is_dir():
# Explicit list of test directories, Pytest skips the build directory if not specified explicitly.
python.pytest(ignore_dirs=["src/tests/integration"], include_dirs=["src/kraken/build"])
if project.directory.name == "kraken-build":
python.pytest(ignore_dirs=["src/tests/integration"], include_dirs=["src/kraken/build"])
elif project.directory.name == "kraken-wrapper":
python.pytest(doctest_modules=False)

if project.directory.joinpath("tests/integration").is_dir():
python.pytest(
Expand Down
5 changes: 5 additions & 0 deletions kraken-build/.changelog/_unreleased.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[[entries]]
id = "cf6a4820-5e32-47fe-beb1-206329e6bc11"
type = "fix"
description = "Fix `dist()` function not using `TaskSet` correctly (regression introduced in 0.37.0, 8840ec5), add `TaskSet.build()` to compensate"
author = "@NiklasRosenstein"
64 changes: 1 addition & 63 deletions kraken-build/src/kraken/core/system/project_test.py
Original file line number Diff line number Diff line change
@@ -1,59 +1,7 @@
from dataclasses import dataclass

import pytest

from kraken.core.system.project import Project
from kraken.core.system.property import Property
from kraken.core.system.task import Task, TaskSet, VoidTask


@dataclass
class MyDescriptor:
name: str


def test__Project__resolve_outputs__can_find_dataclass_in_metadata(kraken_project: Project) -> None:
kraken_project.task("carrier", VoidTask).outputs.append(MyDescriptor("foobar"))
assert list(TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).all()) == [
MyDescriptor("foobar")
]


def test__Project__resolve_outputs__can_find_dataclass_in_properties(kraken_project: Project) -> None:
class MyTask(Task):
out_prop: Property[MyDescriptor] = Property.output()

def execute(self) -> None: ...

task = kraken_project.task("carrier", MyTask)
task.out_prop = MyDescriptor("foobar")
assert list(TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).all()) == [
MyDescriptor("foobar")
]


def test__Project__resolve_outputs__can_not_find_input_property(kraken_project: Project) -> None:
class MyTask(Task):
out_prop: Property[MyDescriptor]

def execute(self) -> None: ...

task = kraken_project.task("carrier", MyTask)
task.out_prop = MyDescriptor("foobar")
assert list(TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).all()) == []


def test__Project__resolve_outputs_supplier(kraken_project: Project) -> None:
class MyTask(Task):
out_prop: Property[MyDescriptor] = Property.output()

def execute(self) -> None: ...

task = kraken_project.task("carrier", MyTask)
task.out_prop = MyDescriptor("foobar")
assert TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(MyDescriptor).supplier().get() == [
MyDescriptor("foobar")
]
from kraken.core.system.task import VoidTask


def test__Project__do_normalizes_taskname_backwards_compatibility_pre_0_12_0(kraken_project: Project) -> None:
Expand All @@ -66,13 +14,3 @@ def test__Project__do_normalizes_taskname_backwards_compatibility_pre_0_12_0(kra
"Starting with kraken-core 0.12.0, Task names must follow a stricter naming convention subject to the "
"Address class' validation (must match /^[a-zA-Z0-9/_\\-\\.\\*]+$/)."
)


def test__Project__do__does_not_set_property_on_None_value(kraken_project: Project) -> None:
class MyTask(Task):
in_prop: Property[str]

def execute(self) -> None: ...

kraken_project.task("carrier", MyTask)
assert TaskSet(kraken_project.context.resolve_tasks(":carrier")).select(str).supplier().get() == []
47 changes: 45 additions & 2 deletions kraken-build/src/kraken/core/system/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from kraken.core.system.task_supplier import TaskSupplier

if TYPE_CHECKING:
from kraken.core.system.context import Context
from kraken.core.system.project import Project
else:
# Type hint evaluation in typeapi tries to fully resolve forward references to a type. In order to allow the
Expand Down Expand Up @@ -567,6 +568,43 @@ def teardown(self) -> None:
class TaskSet(Collection[Task]):
"""Represents a collection of tasks."""

@staticmethod
def build(
context: Context | Project,
selector: str | Address | Task | Iterable[str | Address | Task],
project: Project | None = None,
) -> TaskSet:
"""
For each item in *selector*, resolve tasks using [`Context.resolve_tasks()`]. If a selector is a string,
assign the resolved tasks to a partition by that selector value.

Args:
context: A Kraken context or project to resolve the *selector* in. If it is a project, string selectors
are treated relative to the project.
selector: A single selector string or task, or a sequence thereof. Note that selectors of type [`Address`]
are converted to string partitions.
"""

from kraken.core.system.project import Project

if isinstance(context, Project):
project = context
context = context.context
else:
project = None

if isinstance(selector, (str, Address, Task)):
selector = [selector]

result = TaskSet()
for item in selector:
if isinstance(item, (str, Address)):
result.add(context.resolve_tasks([item], project), partition=str(item))
else:
result.add([item])

return result

def __init__(self, tasks: Iterable[Task] = ()) -> None:
self._tasks = set(tasks)
self._partition_to_task_map: dict[str, set[Task]] = {}
Expand All @@ -579,7 +617,7 @@ def __len__(self) -> int:
return len(self._tasks)

def __repr__(self) -> str:
return f"TaskSet(length={len(self._tasks)})"
return f"TaskSet(length={len(self._tasks)}, pttm={self._partition_to_task_map}, ttpm={self._task_to_partition_map})"

def __contains__(self, __x: object) -> bool:
return __x in self._tasks
Expand Down Expand Up @@ -658,11 +696,16 @@ def __iter__(self) -> Iterable[str]:
@overload
def __getitem__(self, partition: str) -> Collection[Task]: ...

@overload
def __getitem__(self, partition: Address) -> Collection[Task]: ...

@overload
def __getitem__(self, partition: Task) -> Collection[str]: ...

def __getitem__(self, partition: str | Task) -> Collection[str] | Collection[Task]:
def __getitem__(self, partition: str | Address | Task) -> Collection[str] | Collection[Task]:
if isinstance(partition, str):
return self._ptt.get(partition) or ()
elif isinstance(partition, Address):
return self._ptt.get(str(partition)) or ()
else:
return self._ttp.get(partition) or ()
56 changes: 55 additions & 1 deletion kraken-build/src/kraken/core/system/task_test.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from dataclasses import dataclass
from pytest import raises

from kraken.core.system.project import Project
from kraken.core.system.property import Property
from kraken.core.system.task import Task, TaskRelationship
from kraken.core.system.task import Task, TaskRelationship, TaskSet, VoidTask


def test__Task__get_relationships_lineage_through_properties(kraken_project: Project) -> None:
Expand Down Expand Up @@ -44,3 +45,56 @@ def execute(self) -> None:
with raises(TypeError) as excinfo:
t1.b.set(42.0) # type: ignore[arg-type]
assert str(excinfo.value) == "Property(MyTask(:t1).b): expected int, got float\nexpected str, got float"


@dataclass
class MyDescriptor:
name: str


def test__TaskSet__resolve_outputs__can_find_dataclass_in_metadata(kraken_project: Project) -> None:
kraken_project.task("carrier", VoidTask).outputs.append(MyDescriptor("foobar"))
assert list(TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).all()) == [MyDescriptor("foobar")]


def test__TaskSet__resolve_outputs__can_find_dataclass_in_properties(kraken_project: Project) -> None:
class MyTask(Task):
out_prop: Property[MyDescriptor] = Property.output()

def execute(self) -> None: ...

task = kraken_project.task("carrier", MyTask)
task.out_prop = MyDescriptor("foobar")
assert list(TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).all()) == [MyDescriptor("foobar")]


def test__TaskSet__resolve_outputs__can_not_find_input_property(kraken_project: Project) -> None:
class MyTask(Task):
out_prop: Property[MyDescriptor]

def execute(self) -> None: ...

task = kraken_project.task("carrier", MyTask)
task.out_prop = MyDescriptor("foobar")
assert list(TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).all()) == []


def test__TaskSet__resolve_outputs_supplier(kraken_project: Project) -> None:
class MyTask(Task):
out_prop: Property[MyDescriptor] = Property.output()

def execute(self) -> None: ...

task = kraken_project.task("carrier", MyTask)
task.out_prop = MyDescriptor("foobar")
assert TaskSet.build(kraken_project, ":carrier").select(MyDescriptor).supplier().get() == [MyDescriptor("foobar")]


def test__TaskSet__do__does_not_set_property_on_None_value(kraken_project: Project) -> None:
class MyTask(Task):
in_prop: Property[str]

def execute(self) -> None: ...

kraken_project.task("carrier", MyTask)
assert TaskSet.build(kraken_project, ":carrier").select(str).supplier().get() == []
2 changes: 1 addition & 1 deletion kraken-build/src/kraken/std/dist.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ def dist(
k: databind.json.load(v, IndividualDistOptions) if not isinstance(v, IndividualDistOptions) else v
for k, v in dependencies.items()
}
dependencies_set = TaskSet(project.context.resolve_tasks(dependencies_map, project))
dependencies_set = TaskSet.build(project, dependencies_map)

# This associates the IndividualDistOptions specified in *dependencies* to the Resource(s)
# provided by the task(s).
Expand Down
34 changes: 34 additions & 0 deletions kraken-build/src/kraken/std/dist_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import tarfile
from kraken.core import Project, Task, Property
from kraken.core.system.task import TaskStatus
from kraken.std.dist import dist
from kraken.std.descriptors.resource import Resource


def test_dist(kraken_project: Project) -> None:
"""
This function validates that the #dist() function works as intended with mapping individual options
to the resources provided by dependencies.
"""

class ProducerTask(Task):
result: Property[Resource] = Property.output()

def execute(self) -> TaskStatus | None:
output_file = self.project.build_directory / "product.txt"
output_file.write_text("Hello, World!")
self.result = Resource(name="file", path=output_file)
return None

kraken_project.task("producer", ProducerTask)

output_archive = kraken_project.build_directory / "archive.tgz"
dist(name="dist", dependencies={"producer": {"arcname": "result.txt"}}, output_file=output_archive)

kraken_project.context.execute([":dist"])

assert output_archive.exists()
with tarfile.open(output_archive) as tarf:
fp = tarf.extractfile("result.txt")
assert fp is not None
assert fp.read().decode() == "Hello, World!"
2 changes: 1 addition & 1 deletion kraken-wrapper/tests/iss-263/dependency/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ authors = [
{name = "Niklas Rosenstein", email = "[email protected]"},
]
dependencies = []
requires-python = "==3.12.*"
requires-python = ">=3.10"
readme = "README.md"
license = {text = "MIT"}

Expand Down
Loading