From d2cf9b6a5fc948f678d119b1b6129e9157524e2a Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 10:42:30 -0500 Subject: [PATCH 01/11] ENH: Add Merge task in pydra.tasks.core.sequences --- pydra/tasks/core/__init__.py | 4 ++ pydra/tasks/core/sequences.py | 114 ++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 pydra/tasks/core/__init__.py create mode 100644 pydra/tasks/core/sequences.py diff --git a/pydra/tasks/core/__init__.py b/pydra/tasks/core/__init__.py new file mode 100644 index 0000000000..079cae66f6 --- /dev/null +++ b/pydra/tasks/core/__init__.py @@ -0,0 +1,4 @@ +""" +Pydra provides a small number of utility tasks that are aimed at common manipulations +of Python objects. +""" diff --git a/pydra/tasks/core/sequences.py b/pydra/tasks/core/sequences.py new file mode 100644 index 0000000000..b31c18c69b --- /dev/null +++ b/pydra/tasks/core/sequences.py @@ -0,0 +1,114 @@ +import attr +import typing as ty +from pydra.engine.specs import BaseSpec, SpecInfo +from pydra.engine.core import TaskBase +from pydra.engine.helpers import ensure_list + + +@attr.s(kw_only=True) +class MergeInputSpec(BaseSpec): + axis: ty.Literal["vstack", "hstack"] = attr.ib( + default="vstack", + metadata={ + "help_string": "Direction in which to merge, hstack requires same number of elements in each input." + }, + ) + no_flatten: bool = attr.ib( + default=False, + metadata={ + "help_string": "Append to outlist instead of extending in vstack mode." + }, + ) + ravel_inputs: bool = attr.ib( + default=False, + metadata={"help_string": "Ravel inputs when no_flatten is False."}, + ) + + +def _ravel(in_val): + if not isinstance(in_val, list): + return in_val + flat_list = [] + for val in in_val: + raveled_val = _ravel(val) + if isinstance(raveled_val, list): + flat_list.extend(raveled_val) + else: + flat_list.append(raveled_val) + return flat_list + + +class Merge(TaskBase): + """ + Task to merge inputs into a single list + + ``Merge(1)`` will merge a list of lists + + Examples + -------- + >>> from pydra.tasks.core.sequences import Merge + >>> mi = Merge(3, name="mi") + >>> mi.inputs.in1 = 1 + >>> mi.inputs.in2 = [2, 5] + >>> mi.inputs.in3 = 3 + >>> out = mi() + >>> out.output.out + [1, 2, 5, 3] + + >>> merge = Merge(1, name="merge") + >>> merge.inputs.in1 = [1, [2, 5], 3] + >>> out = merge() + >>> out.output.out + [1, [2, 5], 3] + + >>> merge = Merge(1, name="merge") + >>> merge.inputs.in1 = [1, [2, 5], 3] + >>> merge.inputs.ravel_inputs = True + >>> out = merge() + >>> out.output.out + [1, 2, 5, 3] + + >>> merge = Merge(1, name="merge") + >>> merge.inputs.in1 = [1, [2, 5], 3] + >>> merge.inputs.no_flatten = True + >>> out = merge() + >>> out.output.out + [[1, [2, 5], 3]] + """ + + _task_version = "1" + output_spec = SpecInfo(name="Outputs", fields=[("out", ty.List)], bases=(BaseSpec,)) + + def __init__(self, numinputs, *args, **kwargs): + self._numinputs = max(numinputs, 0) + self.input_spec = SpecInfo( + name="Inputs", + fields=[(f"in{i + 1}", ty.List) for i in range(self._numinputs)], + bases=(MergeInputSpec,), + ) + super().__init__(*args, **kwargs) + + def _run_task(self): + self.output_ = {"out": []} + if self._numinputs < 1: + return + + values = [ + getattr(self.inputs, f"in{i + 1}") + for i in range(self._numinputs) + if getattr(self.inputs, f"in{i + 1}") is not attr.NOTHING + ] + + if self.inputs.axis == "vstack": + for value in values: + if isinstance(value, list) and not self.inputs.no_flatten: + self.output_["out"].extend( + _ravel(value) if self.inputs.ravel_inputs else value + ) + else: + self.output_["out"].append(value) + else: + lists = [ensure_list(val) for val in values] + self.output_["out"] = [ + [val[i] for val in lists] for i in range(len(lists[0])) + ] From dd42f630b57a729dcacbdce3c04c9762cc838842 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 10:42:46 -0500 Subject: [PATCH 02/11] FIX: Only evolve inputs if inputs are provided --- pydra/engine/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 317c45ac1f..a39bdb2542 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -161,7 +161,7 @@ def __init__( raise ValueError(f"Unknown input set {inputs!r}") inputs = self._input_sets[inputs] - self.inputs = attr.evolve(self.inputs, **inputs) + self.inputs = attr.evolve(self.inputs, **inputs) # checking if metadata is set properly self.inputs.check_metadata() From a41e83ce77fa3561d1efec59bf3a1da1469dd583 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 12:42:39 -0500 Subject: [PATCH 03/11] FIX: Allow input_spec/output_spec to be class variables --- pydra/engine/core.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index a39bdb2542..7ee85c2f04 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -198,8 +198,10 @@ def __str__(self): def __getstate__(self): state = self.__dict__.copy() - state["input_spec"] = cp.dumps(state["input_spec"]) - state["output_spec"] = cp.dumps(state["output_spec"]) + if "input_spec" in state: + state["input_spec"] = cp.dumps(state["input_spec"]) + if "output_spec" in state: + state["output_spec"] = cp.dumps(state["output_spec"]) inputs = {} for k, v in attr.asdict(state["inputs"]).items(): if k.startswith("_"): @@ -209,9 +211,12 @@ def __getstate__(self): return state def __setstate__(self, state): - state["input_spec"] = cp.loads(state["input_spec"]) - state["output_spec"] = cp.loads(state["output_spec"]) - state["inputs"] = make_klass(state["input_spec"])(**state["inputs"]) + if "input_spec" in state: + state["input_spec"] = cp.loads(state["input_spec"]) + if "output_spec" in state: + state["output_spec"] = cp.loads(state["output_spec"]) + input_spec = state.get("input_spec", self.input_spec) + state["inputs"] = make_klass(input_spec)(**state["inputs"]) self.__dict__.update(state) def __getattr__(self, name): From 205d600be1b6c5980d3cb067648dab880495c7ab Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 13:44:29 -0500 Subject: [PATCH 04/11] PY37: Use typing_extensions to get Literal in Python 3.7 --- pydra/tasks/core/sequences.py | 7 ++++++- setup.cfg | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pydra/tasks/core/sequences.py b/pydra/tasks/core/sequences.py index b31c18c69b..61b9e1b6e5 100644 --- a/pydra/tasks/core/sequences.py +++ b/pydra/tasks/core/sequences.py @@ -4,10 +4,15 @@ from pydra.engine.core import TaskBase from pydra.engine.helpers import ensure_list +try: + from typing import Literal +except ImportError: # PY37 + from typing_extensions import Literal + @attr.s(kw_only=True) class MergeInputSpec(BaseSpec): - axis: ty.Literal["vstack", "hstack"] = attr.ib( + axis: Literal["vstack", "hstack"] = attr.ib( default="vstack", metadata={ "help_string": "Direction in which to merge, hstack requires same number of elements in each input." diff --git a/setup.cfg b/setup.cfg index a560fa5fe7..41acb62bae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ install_requires = cloudpickle >= 1.2.2 filelock >= 3.0.0 etelemetry >= 0.2.2 + typing_extensions ; python_version < "3.8" test_requires = pytest >= 4.4.0, < 6.0.0 From 268987a990096b4cc85f1388c86aa4fd61fdc9e4 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 17:19:50 -0500 Subject: [PATCH 05/11] ENH: Add Select task --- pydra/tasks/core/sequences.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/pydra/tasks/core/sequences.py b/pydra/tasks/core/sequences.py index 61b9e1b6e5..5e0bc69965 100644 --- a/pydra/tasks/core/sequences.py +++ b/pydra/tasks/core/sequences.py @@ -1,6 +1,7 @@ import attr import typing as ty -from pydra.engine.specs import BaseSpec, SpecInfo +import pydra +from pydra.engine.specs import BaseSpec, SpecInfo, MultiInputObj, MultiOutputObj from pydra.engine.core import TaskBase from pydra.engine.helpers import ensure_list @@ -117,3 +118,28 @@ def _run_task(self): self.output_["out"] = [ [val[i] for val in lists] for i in range(len(lists[0])) ] + + +@pydra.mark.task +def Select(inlist: MultiInputObj, index: MultiInputObj) -> MultiOutputObj: + """ + Task to select specific elements from a list + + Examples + -------- + + >>> from pydra.tasks.core.sequences import Select + >>> sl = Select(name="sl") + >>> sl.inputs.inlist = [1, 2, 3, 4, 5] + >>> sl.inputs.index = [3] + >>> out = sl() + >>> out.output.out + 4 + + >>> sl = Select(name="sl") + >>> out = sl(inlist=[1, 2, 3, 4, 5], index=[3, 4]) + >>> out.output.out + [4, 5] + + """ + return [inlist[i] for i in index] From 720ac0e012435aedc23dbeb9d7b1906769ae9463 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 17:20:23 -0500 Subject: [PATCH 06/11] FIX: Update decorator dunders to allow docstrings and doctests to pass through --- pydra/mark/functions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pydra/mark/functions.py b/pydra/mark/functions.py index 25829bd365..74e1347247 100644 --- a/pydra/mark/functions.py +++ b/pydra/mark/functions.py @@ -43,4 +43,8 @@ def task(func): def decorate(**kwargs): return FunctionTask(func=func, **kwargs) + decorate.__module__ = func.__module__ + decorate.__name__ = func.__name__ + decorate.__doc__ = func.__doc__ + return decorate From d725ecec4a546f82b2b019c18a75b46c1736aa00 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 1 Mar 2021 19:55:36 -0500 Subject: [PATCH 07/11] FIX: Not all Tasks have input_spec class attributes --- pydra/engine/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 7ee85c2f04..50155c8f65 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -215,7 +215,7 @@ def __setstate__(self, state): state["input_spec"] = cp.loads(state["input_spec"]) if "output_spec" in state: state["output_spec"] = cp.loads(state["output_spec"]) - input_spec = state.get("input_spec", self.input_spec) + input_spec = state.get("input_spec", getattr(self, "input_spec", None)) state["inputs"] = make_klass(input_spec)(**state["inputs"]) self.__dict__.update(state) From 0bf450ab5c7871c3bac7fa07ad9ce63e8a8599be Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 2 Mar 2021 08:11:54 -0500 Subject: [PATCH 08/11] ENH: Add Split task --- pydra/tasks/core/sequences.py | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pydra/tasks/core/sequences.py b/pydra/tasks/core/sequences.py index 5e0bc69965..173cd798ed 100644 --- a/pydra/tasks/core/sequences.py +++ b/pydra/tasks/core/sequences.py @@ -11,6 +11,58 @@ from typing_extensions import Literal +@attr.s(kw_only=True) +class SplitInputSpec(BaseSpec): + inlist: ty.List = attr.ib(metadata={"help_string": "List of values to split"}) + splits: ty.List[int] = attr.ib( + metadata={ + "help_string": "Number of outputs in each split - should add to number of inputs" + } + ) + squeeze: bool = attr.ib( + default=False, + metadata={"help_string": "Unfold one-element splits removing the list"}, + ) + + +class Split(TaskBase): + """ + Task to split lists into multiple outputs + + Examples + -------- + >>> from pydra.tasks.core.sequences import Split + >>> sp = Split(name="sp", splits=[5, 4, 3, 2, 1]) + >>> out = sp(inlist=list(range(15))) + >>> out.output.out1 + [0, 1, 2, 3, 4] + >>> out.output.out2 + [5, 6, 7, 8] + >>> out.output.out5 + [14] + """ + + _task_version = "1" + input_spec = SplitInputSpec + + def __init__(self, splits, *args, **kwargs): + self.output_spec = SpecInfo( + name="Outputs", + fields=[(f"out{i + 1}", list) for i in range(len(splits))], + bases=(BaseSpec,), + ) + super().__init__(*args, **kwargs) + self.inputs.splits = splits + + def _run_task(self): + self.output_ = {} + left = 0 + for i, split in enumerate(self.inputs.splits, 1): + right = left + split + self.output_[f"out{i}"] = self.inputs.inlist[left:right] + left = right + + @attr.s(kw_only=True) class MergeInputSpec(BaseSpec): axis: Literal["vstack", "hstack"] = attr.ib( From 8442d5ffa9caa2dd7e94fa29b56d59853a28013b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 2 Mar 2021 08:16:17 -0500 Subject: [PATCH 09/11] FIX: Allow input_spec to be a spec --- pydra/engine/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 50155c8f65..e5638fd5d2 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -132,7 +132,9 @@ def __init__( self.name = name if not self.input_spec: raise Exception("No input_spec in class: %s" % self.__class__.__name__) - klass = make_klass(self.input_spec) + klass = self.input_spec + if isinstance(klass, SpecInfo): + klass = make_klass(klass) self.inputs = klass( **{ From 60b062d3b2d65190d0b69f678f10834821061182 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 2 Mar 2021 08:22:54 -0500 Subject: [PATCH 10/11] CI: Build docs with Python 3.8.6 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e025c480f0..09ebd0ac44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ jobs: build_docs: docker: - - image: python:3.7.4 + - image: python:3.8.6 working_directory: /tmp/gh-pages environment: - FSLOUTPUTTYPE: NIFTI From 7894d0b55615c8d902d4ff43f1c110115e83bd33 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 2 Mar 2021 11:25:44 -0500 Subject: [PATCH 11/11] FIX: Only check for self.input_spec if not in state --- pydra/engine/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index e5638fd5d2..9582073206 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -217,7 +217,9 @@ def __setstate__(self, state): state["input_spec"] = cp.loads(state["input_spec"]) if "output_spec" in state: state["output_spec"] = cp.loads(state["output_spec"]) - input_spec = state.get("input_spec", getattr(self, "input_spec", None)) + input_spec = state.get("input_spec") + if input_spec is None: # If it is not saved, it should be a class attribute + input_spec = self.input_spec state["inputs"] = make_klass(input_spec)(**state["inputs"]) self.__dict__.update(state)