From bd76c3d6d8a74a92e3124d63ef56dd15b128a2ed Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 5 Oct 2023 14:55:42 -0400 Subject: [PATCH] adding the Singularity environment class --- pydra/engine/environments.py | 45 ++- pydra/engine/specs.py | 7 - pydra/engine/task.py | 81 ------ pydra/engine/tests/test_environments.py | 70 ++++- pydra/engine/tests/test_singularity.py | 367 +++++------------------- pydra/engine/tests/test_specs.py | 9 - pydra/engine/tests/utils.py | 3 + 7 files changed, 187 insertions(+), 395 deletions(-) diff --git a/pydra/engine/environments.py b/pydra/engine/environments.py index 9a1a0ab24..8a45a9828 100644 --- a/pydra/engine/environments.py +++ b/pydra/engine/environments.py @@ -32,9 +32,7 @@ def execute(self, task): class Docker(Environment): - def __init__( - self, image, tag="latest", binds=None, output_cpath="/output_pydra", xargs=None - ): + def __init__(self, image, tag="latest", output_cpath="/output_pydra", xargs=None): self.image = image self.tag = tag self.xargs = xargs @@ -82,3 +80,44 @@ def execute(self, task, root="/mnt/pydra"): # to be de-rooted # task.finalize_outputs("/mnt/pydra") TODO: probably don't need it return output + + +class Singularity(Docker): + def execute(self, task, root="/mnt/pydra"): + # XXX Need to mount all input locations + singularity_img = f"{self.image}:{self.tag}" + # TODO ? + # Skips over any inputs in task.cache_dir + # Needs to include `out_file`s when not relative to working dir + # Possibly a `TargetFile` type to distinguish between `File` and `str`? + mounts = task.get_bindings(root=root) + + # todo adding xargsy etc + singularity_args = [ + "singularity", + "exec", + "-B", + self.bind(task.cache_dir, "rw"), + ] + singularity_args.extend( + " ".join( + [f"-B {key}:{val[0]}:{val[1]}" for (key, val) in mounts.items()] + ).split() + ) + singularity_args.extend(["--pwd", f"{root}{task.output_dir}"]) + keys = ["return_code", "stdout", "stderr"] + + values = execute( + singularity_args + [singularity_img] + task.command_args(root="/mnt/pydra"), + strip=task.strip, + ) + output = dict(zip(keys, values)) + if output["return_code"]: + if output["stderr"]: + raise RuntimeError(output["stderr"]) + else: + raise RuntimeError(output["stdout"]) + # Any outputs that have been created with a re-rooted path need + # to be de-rooted + # task.finalize_outputs("/mnt/pydra") TODO: probably don't need it + return output diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 9de048d85..9f53f108f 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -693,13 +693,6 @@ class ContainerSpec(ShellSpec): ) -@attr.s(auto_attribs=True, kw_only=True) -class SingularitySpec(ContainerSpec): - """Particularize container specifications to Singularity.""" - - container: str = attr.ib("singularity", metadata={"help_string": "container type"}) - - @attr.s class LazyInterface: _task: "core.TaskBase" = attr.ib() diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 6cd476f12..fd98eeddc 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -58,7 +58,6 @@ ShellSpec, ShellOutSpec, ContainerSpec, - SingularitySpec, attr_fields, ) from .helpers import ( @@ -707,86 +706,6 @@ def _prepare_bindings(self): SUPPORTED_COPY_MODES = FileSet.CopyMode.any - FileSet.CopyMode.symlink -class SingularityTask(ContainerTask): - """Extend shell command task for containerized execution with Singularity.""" - - init = False - - def __init__( - self, - name=None, - audit_flags: AuditFlag = AuditFlag.NONE, - cache_dir=None, - input_spec: ty.Optional[SpecInfo] = None, - messenger_args=None, - messengers=None, - output_spec: ty.Optional[SpecInfo] = None, - rerun=False, - strip=False, - **kwargs, - ): - """ - Initialize this task. - - Parameters - ---------- - name : :obj:`str` - Name of this task. - audit_flags : :obj:`pydra.utils.messenger.AuditFlag` - Auditing configuration - cache_dir : :obj:`os.pathlike` - Cache directory - input_spec : :obj:`pydra.engine.specs.SpecInfo` - Specification of inputs. - messenger_args : - TODO - messengers : - TODO - output_spec : :obj:`pydra.engine.specs.BaseSpec` - Specification of inputs. - strip : :obj:`bool` - TODO - - """ - if not self.init: - if input_spec is None: - input_spec = SpecInfo( - name="Inputs", fields=[], bases=(SingularitySpec,) - ) - super().__init__( - name=name, - input_spec=input_spec, - output_spec=output_spec, - audit_flags=audit_flags, - messengers=messengers, - messenger_args=messenger_args, - cache_dir=cache_dir, - strip=strip, - rerun=rerun, - **kwargs, - ) - self.init = True - - @property - def container_args(self): - """Get container-specific CLI arguments.""" - if is_lazy(self.inputs): - raise Exception("can't return container_args, self.inputs has LazyFields") - self.container_check("singularity") - if self.state: - raise NotImplementedError - - cargs = ["singularity", "exec"] - - if self.inputs.container_xargs is not None: - cargs.extend(self.inputs.container_xargs) - - cargs.extend(self.binds("-B")) - cargs.extend(["--pwd", str(self.output_cpath)]) - cargs.append(self.inputs.image) - return cargs - - def split_cmd(cmd: str): """Splits a shell command line into separate arguments respecting quotes diff --git a/pydra/engine/tests/test_environments.py b/pydra/engine/tests/test_environments.py index 252cc5ea7..40e6581fa 100644 --- a/pydra/engine/tests/test_environments.py +++ b/pydra/engine/tests/test_environments.py @@ -1,6 +1,6 @@ from pathlib import Path -from ..environments import Native, Docker +from ..environments import Native, Docker, Singularity from ..task import ShellCommandTask from ..submitter import Submitter from ..specs import ( @@ -8,7 +8,7 @@ SpecInfo, File, ) -from .utils import no_win, need_docker +from .utils import no_win, need_docker, need_singularity import attr @@ -109,6 +109,72 @@ def test_docker_1_subm(tmp_path, plugin): assert env_res == shelly_call.result().output.__dict__ +@no_win +@need_singularity +def test_singularity_1(tmp_path): + """singularity env: simple command, no arguments""" + newcache = lambda x: makedir(tmp_path, x) + + cmd = ["whoami"] + sing = Singularity(image="docker://alpine") + shelly = ShellCommandTask( + name="shelly", executable=cmd, cache_dir=newcache("shelly") + ) + assert shelly.cmdline == " ".join(cmd) + env_res = sing.execute(shelly) + + shelly_env = ShellCommandTask( + name="shelly", + executable=cmd, + cache_dir=newcache("shelly_env"), + environment=sing, + ) + shelly_env() + assert env_res == shelly_env.output_ == shelly_env.result().output.__dict__ + + shelly_call = ShellCommandTask( + name="shelly", executable=cmd, cache_dir=newcache("shelly_call") + ) + shelly_call(environment=sing) + assert env_res == shelly_call.output_ == shelly_call.result().output.__dict__ + + +@no_win +@need_singularity +def test_singularity_1_subm(tmp_path, plugin): + """docker env with submitter: simple command, no arguments""" + newcache = lambda x: makedir(tmp_path, x) + + cmd = ["whoami"] + sing = Singularity(image="docker://alpine") + shelly = ShellCommandTask( + name="shelly", executable=cmd, cache_dir=newcache("shelly") + ) + assert shelly.cmdline == " ".join(cmd) + env_res = sing.execute(shelly) + + shelly_env = ShellCommandTask( + name="shelly", + executable=cmd, + cache_dir=newcache("shelly_env"), + environment=sing, + ) + with Submitter(plugin=plugin) as sub: + shelly_env(submitter=sub) + assert env_res == shelly_env.result().output.__dict__ + + shelly_call = ShellCommandTask( + name="shelly", executable=cmd, cache_dir=newcache("shelly_call") + ) + with Submitter(plugin=plugin) as sub: + shelly_call(submitter=sub, environment=sing) + for key in [ + "stdout", + "return_code", + ]: # singularity gives info about cashed image in stderr + assert env_res[key] == shelly_call.result().output.__dict__[key] + + def create_shelly_inputfile(tempdir, filename, name, executable): """creating a task with a simple input_spec""" my_input_spec = SpecInfo( diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index bb6506aa9..791575adc 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -3,10 +3,11 @@ import pytest import attr -from ..task import SingularityTask, ShellCommandTask +from ..task import ShellCommandTask from ..submitter import Submitter from ..core import Workflow -from ..specs import ShellOutSpec, SpecInfo, File, SingularitySpec +from ..specs import ShellOutSpec, SpecInfo, File, ShellSpec +from ..environments import Singularity need_docker = pytest.mark.skipif( @@ -29,18 +30,18 @@ def test_singularity_1_nosubm(tmp_path): """ cmd = "pwd" image = "docker://alpine" - singu = SingularityTask( - name="singu", executable=cmd, image=image, cache_dir=tmp_path - ) - assert singu.inputs.image == "docker://alpine" - assert singu.inputs.container == "singularity" - assert ( - singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {cmd}" + singu = ShellCommandTask( + name="singu", + executable=cmd, + environment=Singularity(image=image), + cache_dir=tmp_path, ) + assert singu.environment.image == "docker://alpine" + assert isinstance(singu.environment, Singularity) + assert singu.cmdline == cmd res = singu() - assert "output_pydra" in res.output.stdout + assert "/mnt/pydra" in res.output.stdout assert res.output.return_code == 0 @@ -51,13 +52,13 @@ def test_singularity_2_nosubm(tmp_path): """ cmd = ["echo", "hail", "pydra"] image = "docker://alpine" - singu = SingularityTask( - name="singu", executable=cmd, image=image, cache_dir=tmp_path - ) - assert ( - singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {' '.join(cmd)}" + singu = ShellCommandTask( + name="singu", + executable=cmd, + environment=Singularity(image=image), + cache_dir=tmp_path, ) + assert singu.cmdline == " ".join(cmd) res = singu() assert res.output.stdout.strip() == " ".join(cmd[1:]) @@ -71,42 +72,18 @@ def test_singularity_2(plugin, tmp_path): """ cmd = ["echo", "hail", "pydra"] image = "docker://alpine" - singu = SingularityTask( - name="singu", executable=cmd, image=image, cache_dir=tmp_path - ) - assert ( - singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {' '.join(cmd)}" - ) - - with Submitter(plugin=plugin) as sub: - singu(submitter=sub) - res = singu.result() - assert res.output.stdout.strip() == " ".join(cmd[1:]) - assert res.output.return_code == 0 - -@need_singularity -def test_singularity_2_singuflag(plugin, tmp_path): - """a command with arguments, cmd and args given as executable - using ShellComandTask with container_info=("singularity", image) - """ - cmd = ["echo", "hail", "pydra"] - image = "docker://alpine" - shingu = ShellCommandTask( - name="shingu", + singu = ShellCommandTask( + name="singu", executable=cmd, - container_info=("singularity", image), + environment=Singularity(image=image), cache_dir=tmp_path, ) - assert ( - shingu.cmdline - == f"singularity exec -B {shingu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {' '.join(cmd)}" - ) + assert singu.cmdline == " ".join(cmd) with Submitter(plugin=plugin) as sub: - shingu(submitter=sub) - res = shingu.result() + singu(submitter=sub) + res = singu.result() assert res.output.stdout.strip() == " ".join(cmd[1:]) assert res.output.return_code == 0 @@ -120,17 +97,14 @@ def test_singularity_2a(plugin, tmp_path): cmd_args = ["hail", "pydra"] # separate command into exec + args image = "docker://alpine" - singu = SingularityTask( + singu = ShellCommandTask( name="singu", executable=cmd_exec, args=cmd_args, - image=image, + environment=Singularity(image=image), cache_dir=tmp_path, ) - assert ( - singu.cmdline - == f"singularity exec -B {singu.output_dir}:/output_pydra:rw --pwd /output_pydra {image} {cmd_exec} {' '.join(cmd_args)}" - ) + assert singu.cmdline == f"{cmd_exec} {' '.join(cmd_args)}" with Submitter(plugin=plugin) as sub: singu(submitter=sub) @@ -139,84 +113,6 @@ def test_singularity_2a(plugin, tmp_path): assert res.output.return_code == 0 -@need_singularity -@pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_singularity_3(plugin, tmp_path): - """a simple command in container with bindings, - creating directory in tmp dir and checking if it is in the container - """ - # creating a new directory - (tmp_path / "new_dir").mkdir() - cmd = ["ls", "/tmp_dir"] - image = "docker://alpine" - singu = SingularityTask( - name="singu", executable=cmd, image=image, cache_dir=tmp_path - ) - # binding tmp directory to the container - singu.inputs.bindings = [(str(tmp_path), "/tmp_dir", "ro")] - - with Submitter(plugin=plugin) as sub: - singu(submitter=sub) - - res = singu.result() - assert "new_dir\n" in res.output.stdout - assert res.output.return_code == 0 - - -@need_singularity -@pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_singularity_3_singuflag(plugin, tmp_path): - """a simple command in container with bindings, - creating directory in tmp dir and checking if it is in the container - using ShellComandTask with container_info=("singularity", image) - """ - # creating a new directory - (tmp_path / "new_dir").mkdir() - cmd = ["ls", "/tmp_dir"] - image = "docker://alpine" - shingu = SingularityTask( - name="singu", - executable=cmd, - container_info=("singularity", image), - cache_dir=tmp_path, - ) - # binding tmp directory to the container - shingu.inputs.bindings = [(str(tmp_path), "/tmp_dir", "ro")] - - with Submitter(plugin=plugin) as sub: - shingu(submitter=sub) - - res = shingu.result() - assert "new_dir\n" in res.output.stdout - assert res.output.return_code == 0 - - -@need_singularity -@pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_singularity_3_singuflagbind(plugin, tmp_path): - """a simple command in container with bindings, - creating directory in tmp dir and checking if it is in the container - using ShellComandTask with container_info=("singularity", image, bindings) - """ - # creating a new directory - (tmp_path / "new_dir").mkdir() - cmd = ["ls", "/tmp_dir"] - image = "docker://alpine" - shingu = SingularityTask( - name="singu", - executable=cmd, - container_info=("singularity", image, [(str(tmp_path), "/tmp_dir", "ro")]), - cache_dir=tmp_path, - ) - - with Submitter(plugin=plugin) as sub: - shingu(submitter=sub) - - res = shingu.result() - assert "new_dir\n" in res.output.stdout - assert res.output.return_code == 0 - - # tests with State @@ -227,64 +123,33 @@ def test_singularity_st_1(plugin, tmp_path): """ cmd = ["pwd", "ls"] image = "docker://alpine" - singu = SingularityTask(name="singu", image=image, cache_dir=tmp_path).split( - "executable", executable=cmd - ) + singu = ShellCommandTask( + name="singu", environment=Singularity(image=image), cache_dir=tmp_path + ).split("executable", executable=cmd) assert singu.state.splitter == "singu.executable" res = singu(plugin=plugin) - assert "/output_pydra" in res[0].output.stdout + assert "/mnt/pydra" in res[0].output.stdout assert res[1].output.stdout == "" assert res[0].output.return_code == res[1].output.return_code == 0 -@need_singularity -def test_singularity_st_2(plugin, tmp_path): - """command with arguments in docker, checking the distribution - splitter = image - """ - cmd = ["cat", "/etc/issue"] - image = ["docker://alpine", "docker://ubuntu"] - singu = SingularityTask(name="singu", executable=cmd, cache_dir=tmp_path).split( - "image", image=image - ) - assert singu.state.splitter == "singu.image" - - res = singu(plugin=plugin) - assert "Alpine" in res[0].output.stdout - assert "Ubuntu" in res[1].output.stdout - assert res[0].output.return_code == res[1].output.return_code == 0 - - -@need_singularity -def test_singularity_st_3(plugin, tmp_path): - """outer splitter image and executable""" - cmd = ["pwd", ["cat", "/etc/issue"]] - image = ["docker://alpine", "docker://ubuntu"] - singu = SingularityTask(name="singu", cache_dir=tmp_path).split( - ["image", "executable"], executable=cmd, image=image - ) - assert singu.state.splitter == ["singu.image", "singu.executable"] - res = singu(plugin=plugin) - - assert "/output_pydra" in res[0].output.stdout - assert "Alpine" in res[1].output.stdout - assert "/output_pydra" in res[2].output.stdout - assert "Ubuntu" in res[3].output.stdout - - @need_singularity @need_slurm +@pytest.mark.skip(reason="TODO, xfail incorrect") @pytest.mark.xfail( reason="slurm can complain if the number of submitted jobs exceeds the limit" ) @pytest.mark.parametrize("n", [10, 50, 100]) -def test_singularity_st_4(tmp_path, n): +def test_singularity_st_2(tmp_path, n): """splitter over args (checking bigger splitters if slurm available)""" args_n = list(range(n)) image = "docker://alpine" - singu = SingularityTask( - name="singu", executable="echo", image=image, cache_dir=tmp_path + singu = ShellCommandTask( + name="singu", + executable="echo", + environment=Singularity(image=image), + cache_dir=tmp_path, ).split("args", args=args_n) assert singu.state.splitter == "singu.args" res = singu(plugin="slurm") @@ -293,90 +158,6 @@ def test_singularity_st_4(tmp_path, n): assert res[0].output.return_code == res[1].output.return_code == 0 -@need_singularity -@pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_wf_singularity_1(plugin, tmp_path): - """a workflow with two connected task - the first one read the file that is bounded to the container, - the second uses echo - """ - with open((tmp_path / "file_pydra.txt"), "w") as f: - f.write("hello from pydra") - - image = "docker://alpine" - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2"], cache_dir=tmp_path) - wf.inputs.cmd1 = ["cat", "/tmp_dir/file_pydra.txt"] - wf.inputs.cmd2 = ["echo", "message from the previous task:"] - wf.add( - SingularityTask( - name="singu_cat", - image=image, - executable=wf.lzin.cmd1, - bindings=[(str(tmp_path), "/tmp_dir", "ro")], - strip=True, - ) - ) - wf.add( - SingularityTask( - name="singu_echo", - image=image, - executable=wf.lzin.cmd2, - args=wf.singu_cat.lzout.stdout, - strip=True, - ) - ) - wf.set_output([("out", wf.singu_echo.lzout.stdout)]) - - with Submitter(plugin=plugin) as sub: - wf(submitter=sub) - - res = wf.result() - assert res.output.out == "message from the previous task: hello from pydra" - - -@need_docker -@need_singularity -@pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_wf_singularity_1a(plugin, tmp_path): - """a workflow with two connected task - using both containers: Docker and Singul. - the first one read the file that is bounded to the container, - the second uses echo - """ - with open((tmp_path / "file_pydra.txt"), "w") as f: - f.write("hello from pydra") - - image_sing = "docker://alpine" - image_doc = "ubuntu" - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2"], cache_dir=tmp_path) - wf.inputs.cmd1 = ["cat", "/tmp_dir/file_pydra.txt"] - wf.inputs.cmd2 = ["echo", "message from the previous task:"] - wf.add( - SingularityTask( - name="singu_cat", - image=image_sing, - executable=wf.lzin.cmd1, - bindings=[(str(tmp_path), "/tmp_dir", "ro")], - strip=True, - ) - ) - wf.add( - DockerTask( - name="singu_echo", - image=image_doc, - executable=wf.lzin.cmd2, - args=wf.singu_cat.lzout.stdout, - strip=True, - ) - ) - wf.set_output([("out", wf.singu_echo.lzout.stdout)]) - - with Submitter(plugin=plugin) as sub: - wf(submitter=sub) - - res = wf.result() - assert res.output.out == "message from the previous task: hello from pydra" - - # tests with customized output_spec @@ -394,9 +175,9 @@ def test_singularity_outputspec_1(plugin, tmp_path): fields=[("newfile", File, "newfile_tmp.txt")], bases=(ShellOutSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path, @@ -439,12 +220,12 @@ def test_singularity_inputspec_1(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, file=filename, input_spec=my_input_spec, @@ -480,12 +261,12 @@ def test_singularity_inputspec_1a(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, input_spec=my_input_spec, strip=True, @@ -537,12 +318,12 @@ def test_singularity_inputspec_2(plugin, tmp_path): ), ), ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, file1=filename_1, input_spec=my_input_spec, @@ -597,12 +378,12 @@ def test_singularity_inputspec_2a_except(plugin, tmp_path): ), ), ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, file2=filename_2, input_spec=my_input_spec, @@ -657,12 +438,12 @@ def test_singularity_inputspec_2a(plugin, tmp_path): ), ), ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, file2=filename_2, input_spec=my_input_spec, @@ -714,12 +495,12 @@ def test_singularity_cmd_inputspec_copyfile_1(plugin, tmp_path): ), ), ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, input_spec=my_input_spec, orig_file=str(file), @@ -739,7 +520,7 @@ def test_singularity_cmd_inputspec_copyfile_1(plugin, tmp_path): @need_singularity -def test_singularity_inputspec_state_1(plugin, tmp_path): +def test_singularity_inputspec_state_1(tmp_path): """a customised input spec for a singularity file with a splitter, splitter is on files """ @@ -770,12 +551,12 @@ def test_singularity_inputspec_state_1(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, input_spec=my_input_spec, strip=True, @@ -820,12 +601,12 @@ def test_singularity_inputspec_state_1b(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=cmd, input_spec=my_input_spec, strip=True, @@ -863,16 +644,16 @@ def test_singularity_wf_inputspec_1(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) wf = Workflow(name="wf", input_spec=["cmd", "file"], cache_dir=tmp_path) wf.inputs.cmd = cmd wf.inputs.file = filename - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=wf.lzin.cmd, file=wf.lzin.file, input_spec=my_input_spec, @@ -919,15 +700,15 @@ def test_singularity_wf_state_inputspec_1(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) wf = Workflow(name="wf", input_spec=["cmd", "file"], cache_dir=tmp_path) wf.inputs.cmd = cmd - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=wf.lzin.cmd, file=wf.lzin.file, input_spec=my_input_spec, @@ -976,16 +757,16 @@ def test_singularity_wf_ndst_inputspec_1(plugin, tmp_path): ), ) ], - bases=(SingularitySpec,), + bases=(ShellSpec,), ) wf = Workflow(name="wf", input_spec=["cmd", "file"], cache_dir=tmp_path) wf.inputs.cmd = cmd wf.inputs.file = filename - singu = SingularityTask( + singu = ShellCommandTask( name="singu", - image=image, + environment=Singularity(image=image), executable=wf.lzin.cmd, input_spec=my_input_spec, strip=True, diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 51af95a75..d2ed6b235 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -12,7 +12,6 @@ Result, ShellSpec, ContainerSpec, - SingularitySpec, LazyIn, LazyOut, LazyField, @@ -65,14 +64,6 @@ def test_container(): assert hasattr(spec, "executable") -def test_singularity(): - with pytest.raises(TypeError): - spec = SingularitySpec() - spec = SingularitySpec(executable="ls", image="busybox") - assert all(hasattr(spec, attr) for attr in container_attrs) - assert getattr(spec, "container") == "singularity" - - class NodeTesting: @attrs.define() class Input: diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index a219a397b..5b0858866 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -18,6 +18,9 @@ shutil.which("docker") is None or sp.call(["docker", "info"]), reason="no docker within the container", ) +need_singularity = pytest.mark.skipif( + shutil.which("singularity") is None, reason="no singularity available" +) no_win = pytest.mark.skipif( sys.platform.startswith("win"), reason="docker command not adjusted for windows docker",