diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 3230d8da8..50a05fbad 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -157,7 +157,7 @@ jobs: run: | echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests - + # Upload artifacts on failure, ignoring binary files - name: Upload Artifact if: failure() diff --git a/pyproject.toml b/pyproject.toml index 5df64aa97..e11c252ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,7 +147,6 @@ module = [ # FIXME: DO NOT MERGE THIS INTO DEVELOP BRANCH UNLESS THESE ARE PASSING OR # REMOVED!! "smartsim._core._cli.*", - "smartsim._core.commands.*", "smartsim._core.control.controller", "smartsim._core.control.manifest", "smartsim._core.entrypoints.dragon_client", diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index d89aa41ad..3f41f32fe 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -26,25 +26,18 @@ import typing as t from collections.abc import MutableSequence +from copy import deepcopy -from ...settings.launchCommand import LauncherType +from typing_extensions import Self class Command(MutableSequence[str]): """Basic container for command information""" - def __init__(self, launcher: LauncherType, command: t.List[str]) -> None: + def __init__(self, command: t.List[str]) -> None: """Command constructor""" - self._launcher = launcher self._command = command - @property - def launcher(self) -> LauncherType: - """Get the launcher type. - Return a reference to the LauncherType. - """ - return self._launcher - @property def command(self) -> t.List[str]: """Get the command list. @@ -52,15 +45,41 @@ def command(self) -> t.List[str]: """ return self._command - def __getitem__(self, idx: int) -> str: + @t.overload + def __getitem__(self, idx: int) -> str: ... + @t.overload + def __getitem__(self, idx: slice) -> Self: ... + def __getitem__(self, idx: t.Union[int, slice]) -> t.Union[str, Self]: """Get the command at the specified index.""" - return self._command[idx] + cmd = self._command[idx] + if isinstance(cmd, str): + return cmd + return type(self)(cmd) - def __setitem__(self, idx: int, value: str) -> None: + @t.overload + def __setitem__(self, idx: int, value: str) -> None: ... + @t.overload + def __setitem__(self, idx: slice, value: t.Iterable[str]) -> None: ... + def __setitem__( + self, idx: t.Union[int, slice], value: t.Union[str, t.Iterable[str]] + ) -> None: """Set the command at the specified index.""" - self._command[idx] = value + if isinstance(idx, int): + if not isinstance(value, str): + raise ValueError( + "Value must be of type `str` when assigning to an index" + ) + self._command[idx] = deepcopy(value) + return + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "Value must be a list of strings when assigning to a slice" + ) + self._command[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int) -> None: + def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the command at the specified index.""" del self._command[idx] @@ -73,6 +92,5 @@ def insert(self, idx: int, value: str) -> None: self._command.insert(idx, value) def __str__(self) -> str: # pragma: no cover - string = f"\nLauncher: {self.launcher.value}\n" - string += f"Command: {' '.join(str(cmd) for cmd in self.command)}" + string = f"\nCommand: {' '.join(str(cmd) for cmd in self.command)}" return string diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 08b95bbfd..34743063e 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -26,6 +26,7 @@ import typing as t from collections.abc import MutableSequence +from copy import deepcopy from .command import Command @@ -46,15 +47,45 @@ def commands(self) -> t.List[Command]: """ return self._commands - def __getitem__(self, idx: int) -> Command: + @t.overload + def __getitem__(self, idx: int) -> Command: ... + @t.overload + def __getitem__(self, idx: slice) -> t.List[Command]: ... + def __getitem__( + self, idx: t.Union[slice, int] + ) -> t.Union[Command, t.List[Command]]: """Get the Command at the specified index.""" return self._commands[idx] - def __setitem__(self, idx: int, value: Command) -> None: - """Set the Command at the specified index.""" - self._commands[idx] = value + @t.overload + def __setitem__(self, idx: int, value: Command) -> None: ... + @t.overload + def __setitem__(self, idx: slice, value: t.Iterable[Command]) -> None: ... + def __setitem__( + self, idx: t.Union[int, slice], value: t.Union[Command, t.Iterable[Command]] + ) -> None: + """Set the Commands at the specified index.""" + if isinstance(idx, int): + if not isinstance(value, Command): + raise ValueError( + "Value must be of type `Command` when assigning to an index" + ) + self._commands[idx] = deepcopy(value) + return + if not isinstance(value, list): + raise ValueError( + "Value must be a list of Commands when assigning to a slice" + ) + for sublist in value: + if not isinstance(sublist.command, list) or not all( + isinstance(item, str) for item in sublist.command + ): + raise ValueError( + "Value sublists must be a list of Commands when assigning to a slice" + ) + self._commands[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int) -> None: + def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the Command at the specified index.""" del self._commands[idx] diff --git a/smartsim/_core/dispatch.py b/smartsim/_core/dispatch.py index b774baade..551c27d18 100644 --- a/smartsim/_core/dispatch.py +++ b/smartsim/_core/dispatch.py @@ -28,6 +28,7 @@ import dataclasses import os +import pathlib import typing as t from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack @@ -42,10 +43,11 @@ from smartsim.experiment import Experiment from smartsim.settings.arguments import LaunchArguments + _Ts = TypeVarTuple("_Ts") -_WorkingDirectory: TypeAlias = t.Union[str, os.PathLike[str]] +WorkingDirectory: TypeAlias = pathlib.Path """A working directory represented as a string or PathLike object""" _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") @@ -57,20 +59,30 @@ to the to the `LauncherProtocol.start` method """ -_EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] +EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] """A mapping of user provided mapping of environment variables in which to run a job """ -_FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType], +FormatterType: TypeAlias = t.Callable[ + [ + _DispatchableT, + "ExecutableProtocol", + WorkingDirectory, + EnvironMappingType, + pathlib.Path, + pathlib.Path, + ], _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ -_LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType]" -) +_LaunchConfigType: TypeAlias = """_LauncherAdapter[ + ExecutableProtocol, + WorkingDirectory, + EnvironMappingType, + pathlib.Path, + pathlib.Path]""" """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -133,7 +145,7 @@ def dispatch( # Signature when used as a decorator self, args: None = ..., *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]]: ... @@ -142,7 +154,7 @@ def dispatch( # Signature when used as a method self, args: type[_DispatchableT], *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> None: ... @@ -150,7 +162,7 @@ def dispatch( # Actual implementation self, args: type[_DispatchableT] | None = None, *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = False, ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]] | None: @@ -216,7 +228,7 @@ class _DispatchRegistration(t.Generic[_DispatchableT, _LaunchableT]): to be launched by the afore mentioned launcher. """ - formatter: _FormatterType[_DispatchableT, _LaunchableT] + formatter: FormatterType[_DispatchableT, _LaunchableT] launcher_type: type[LauncherProtocol[_LaunchableT]] def _is_compatible_launcher(self, launcher: LauncherProtocol[t.Any]) -> bool: @@ -260,10 +272,12 @@ def create_adapter_from_launcher( def format_( exe: ExecutableProtocol, - path: str | os.PathLike[str], - env: _EnvironMappingType, + path: pathlib.Path, + env: EnvironMappingType, + out: pathlib.Path, + err: pathlib.Path, ) -> _LaunchableT: - return self.formatter(arguments, exe, path, env) + return self.formatter(arguments, exe, path, env, out, err) return _LauncherAdapter(launcher, format_) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 9c58cceaa..e4018ccc3 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -108,7 +108,17 @@ def _log_file(log_path: pathlib.Path) -> pathlib.Path: """ return pathlib.Path(log_path) / "smartsim_params.txt" - def generate_job(self, job: Job, job_index: int) -> pathlib.Path: + @staticmethod + def _output_files( + log_path: pathlib.Path, job_name: str + ) -> t.Tuple[pathlib.Path, pathlib.Path]: + out_file_path = log_path / f"{job_name}.out" + err_file_path = log_path / f"{job_name}.err" + return out_file_path, err_file_path + + def generate_job( + self, job: Job, job_index: int + ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Write and configure input files for a Job. To have files or directories present in the created Job @@ -136,10 +146,13 @@ def generate_job(self, job: Job, job_index: int) -> pathlib.Path: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") + # Create output files + out_file, err_file = self._output_files(log_path, job.entity.name) + # Perform file system operations on attached files self._build_operations(job, job_path) - return job_path + return job_path, out_file, err_file @classmethod def _build_operations(cls, job: Job, job_path: pathlib.Path) -> None: diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index af26a8b82..654e461ce 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -27,6 +27,7 @@ from __future__ import annotations import os +import pathlib import typing as t from smartsim._core.schemas.dragonRequests import DragonRunPolicy @@ -376,6 +377,8 @@ def _as_run_request_args_and_policy( exe: ExecutableProtocol, path: str | os.PathLike[str], env: t.Mapping[str, str | None], + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, ) -> tuple[DragonRunRequestView, DragonRunPolicy]: # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # FIXME: This type is 100% unacceptable, but I don't want to spend too much @@ -397,8 +400,8 @@ def _as_run_request_args_and_policy( env=env, # TODO: Not sure how this info is injected name=None, - output_file=None, - error_file=None, + output_file=stdout_path, + error_file=stderr_path, **run_args, ), policy, diff --git a/smartsim/_core/shell/shellLauncher.py b/smartsim/_core/shell/shellLauncher.py index 50e7573c5..8210cecbb 100644 --- a/smartsim/_core/shell/shellLauncher.py +++ b/smartsim/_core/shell/shellLauncher.py @@ -27,14 +27,15 @@ from __future__ import annotations -import os +import io +import pathlib import subprocess as sp import typing as t import psutil from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import _EnvironMappingType, _FormatterType, dispatch +from smartsim._core.dispatch import EnvironMappingType, FormatterType, WorkingDirectory from smartsim._core.utils import helpers from smartsim._core.utils.launcher import ExecutableProtocol, create_job_id from smartsim.error import errors @@ -51,20 +52,95 @@ logger = get_logger(__name__) +class ShellLauncherCommand(t.NamedTuple): + env: EnvironMappingType + path: pathlib.Path + stdout: io.TextIOWrapper | int + stderr: io.TextIOWrapper | int + command_tuple: tuple[str, tuple[str, ...]] | t.Sequence[str] + + +def make_shell_format_fn( + run_command: str | None, +) -> FormatterType[ShellLaunchArguments, ShellLauncherCommand]: + """A function that builds a function that formats a `LaunchArguments` as a + shell executable sequence of strings for a given launching utility. + + Example usage: + + .. highlight:: python + .. code-block:: python + + echo_hello_world: ExecutableProtocol = ... + env = {} + slurm_args: SlurmLaunchArguments = ... + slurm_args.set_nodes(3) + + as_srun_command = make_shell_format_fn("srun") + fmt_cmd = as_srun_command(slurm_args, echo_hello_world, env) + print(list(fmt_cmd)) + # prints: "['srun', '--nodes=3', '--', 'echo', 'Hello World!']" + + .. note:: + This function was/is a kind of slap-dash implementation, and is likely + to change or be removed entierely as more functionality is added to the + shell launcher. Use with caution and at your own risk! + + :param run_command: Name or path of the launching utility to invoke with + the arguments. + :returns: A function to format an arguments, an executable, and an + environment as a shell launchable sequence for strings. + """ + + def impl( + args: ShellLaunchArguments, + exe: ExecutableProtocol, + path: WorkingDirectory, + env: EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, + ) -> ShellLauncherCommand: + command_tuple = ( + ( + run_command, + *(args.format_launch_args() or ()), + "--", + *exe.as_program_arguments(), + ) + if run_command is not None + else exe.as_program_arguments() + ) + # pylint: disable-next=consider-using-with + return ShellLauncherCommand( + env, pathlib.Path(path), open(stdout_path), open(stderr_path), command_tuple + ) + + return impl + + class ShellLauncher: """Mock launcher for launching/tracking simple shell commands""" def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def start( - self, command: tuple[str | os.PathLike[str], t.Sequence[str]] - ) -> LaunchedJobID: + def check_popen_inputs(self, shell_command: ShellLauncherCommand) -> None: + if not shell_command.path.exists(): + raise ValueError("Please provide a valid path to ShellLauncherCommand.") + + def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: + self.check_popen_inputs(shell_command) id_ = create_job_id() - path, args = command - exe, *rest = args + exe, *rest = shell_command.command_tuple + expanded_exe = helpers.expand_exe_path(exe) # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) + self._launched[id_] = sp.Popen( + (expanded_exe, *rest), + cwd=shell_command.path, + env={k: v for k, v in shell_command.env.items() if v is not None}, + stdout=shell_command.stdout, + stderr=shell_command.stderr, + ) return id_ def _get_proc_from_job_id(self, id_: LaunchedJobID, /) -> sp.Popen[bytes]: @@ -130,57 +206,3 @@ def _stop(self, id_: LaunchedJobID, /) -> JobStatus: @classmethod def create(cls, _: Experiment) -> Self: return cls() - - -def make_shell_format_fn( - run_command: str | None, -) -> _FormatterType[ - ShellLaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]] -]: - """A function that builds a function that formats a `LaunchArguments` as a - shell executable sequence of strings for a given launching utility. - - Example usage: - - .. highlight:: python - .. code-block:: python - - echo_hello_world: ExecutableProtocol = ... - env = {} - slurm_args: SlurmLaunchArguments = ... - slurm_args.set_nodes(3) - - as_srun_command = make_shell_format_fn("srun") - fmt_cmd = as_srun_command(slurm_args, echo_hello_world, env) - print(list(fmt_cmd)) - # prints: "['srun', '--nodes=3', '--', 'echo', 'Hello World!']" - - .. note:: - This function was/is a kind of slap-dash implementation, and is likely - to change or be removed entierely as more functionality is added to the - shell launcher. Use with caution and at your own risk! - - :param run_command: Name or path of the launching utility to invoke with - the arguments. - :returns: A function to format an arguments, an executable, and an - environment as a shell launchable sequence for strings. - """ - - def impl( - args: ShellLaunchArguments, - exe: ExecutableProtocol, - path: str | os.PathLike[str], - _env: _EnvironMappingType, - ) -> t.Tuple[str | os.PathLike[str], t.Sequence[str]]: - return path, ( - ( - run_command, - *(args.format_launch_args() or ()), - "--", - *exe.as_program_arguments(), - ) - if run_command is not None - else exe.as_program_arguments() - ) - - return impl diff --git a/smartsim/experiment.py b/smartsim/experiment.py index c67e0e134..d7857bd23 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -232,8 +232,9 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: launch_config = dispatch.create_new_launcher_configuration( for_experiment=self, with_arguments=args ) - job_execution_path = self._generate(generator, job, idx) - id_ = launch_config.start(exe, job_execution_path, env) + # Generate the job directory and return the generated job path + job_execution_path, out, err = self._generate(generator, job, idx) + id_ = launch_config.start(exe, job_execution_path, env, out, err) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each # individual job, and the experiment can monitor job statuses. @@ -277,7 +278,9 @@ def get_status( return tuple(stats) @_contextualize - def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.Path: + def _generate( + self, generator: Generator, job: Job, job_index: int + ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Generate the directory structure and files for a ``Job`` If files or directories are attached to an ``Application`` object @@ -293,8 +296,8 @@ def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.P :raises: A SmartSimError if an error occurs during the generation process. """ try: - job_run_path = generator.generate_job(job, job_index) - return job_run_path + job_path, out, err = generator.generate_job(job, job_index) + return (job_path, out, err) except SmartSimError as e: logger.error(e) raise diff --git a/smartsim/launchable/baseJobGroup.py b/smartsim/launchable/baseJobGroup.py index d662550f5..b7becba56 100644 --- a/smartsim/launchable/baseJobGroup.py +++ b/smartsim/launchable/baseJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/colocatedJobGroup.py b/smartsim/launchable/colocatedJobGroup.py index 97e7aa4a3..1c3b96fba 100644 --- a/smartsim/launchable/colocatedJobGroup.py +++ b/smartsim/launchable/colocatedJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index 65914cde4..3de767711 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 54427d5a7..6177cb6b6 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -26,18 +26,41 @@ from __future__ import annotations +import pathlib +import subprocess import typing as t from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import dispatch -from smartsim._core.shell.shellLauncher import ShellLauncher, make_shell_format_fn +from smartsim._core.dispatch import EnvironMappingType, dispatch +from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand +from smartsim._core.utils.launcher import ExecutableProtocol from smartsim.log import get_logger from ...common import set_check_input from ...launchCommand import LauncherType logger = get_logger(__name__) -_as_jsrun_command = make_shell_format_fn(run_command="jsrun") + + +def _as_jsrun_command( + args: ShellLaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, +) -> ShellLauncherCommand: + command_tuple = ( + "jsrun", + *(args.format_launch_args() or ()), + f"--stdio_stdout={stdout_path}", + f"--stdio_stderr={stderr_path}", + "--", + *exe.as_program_arguments(), + ) + return ShellLauncherCommand( + env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple + ) @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index a1b12728b..adbbfab93 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -27,19 +27,42 @@ from __future__ import annotations import os +import pathlib import re +import subprocess import typing as t from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import dispatch -from smartsim._core.shell.shellLauncher import ShellLauncher, make_shell_format_fn +from smartsim._core.dispatch import EnvironMappingType, dispatch +from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand +from smartsim._core.utils.launcher import ExecutableProtocol from smartsim.log import get_logger from ...common import set_check_input from ...launchCommand import LauncherType logger = get_logger(__name__) -_as_srun_command = make_shell_format_fn(run_command="srun") + + +def _as_srun_command( + args: ShellLaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, +) -> ShellLauncherCommand: + command_tuple = ( + "srun", + *(args.format_launch_args() or ()), + f"--output={stdout_path}", + f"--error={stderr_path}", + "--", + *exe.as_program_arguments(), + ) + return ShellLauncherCommand( + env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple + ) @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 71b1b87ff..2d1ddfbe8 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -27,33 +27,50 @@ import pytest from smartsim._core.commands.command import Command -from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a def test_command_init(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] - assert cmd.launcher == LauncherType.Slurm -def test_command_getitem(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) +def test_command_getitem_int(): + cmd = Command(command=["salloc", "-N", "1"]) get_value = cmd[0] assert get_value == "salloc" -def test_command_setitem(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) +def test_command_getitem_slice(): + cmd = Command(command=["salloc", "-N", "1"]) + get_value = cmd[0:2] + assert get_value.command == ["salloc", "-N"] + + +def test_command_setitem_int(): + cmd = Command(command=["salloc", "-N", "1"]) cmd[0] = "srun" cmd[1] = "-n" assert cmd.command == ["srun", "-n", "1"] +def test_command_setitem_slice(): + cmd = Command(command=["salloc", "-N", "1"]) + cmd[0:2] = ["srun", "-n"] + assert cmd.command == ["srun", "-n", "1"] + + +def test_command_setitem_fail(): + cmd = Command(command=["salloc", "-N", "1"]) + with pytest.raises(ValueError): + cmd[0] = 1 + with pytest.raises(ValueError): + cmd[0:2] = [1, "-n"] + + def test_command_delitem(): cmd = Command( - launcher=LauncherType.Slurm, command=["salloc", "-N", "1", "--constraint", "P100"], ) del cmd.command[3] @@ -62,11 +79,11 @@ def test_command_delitem(): def test_command_len(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) assert len(cmd) is 3 def test_command_insert(): - cmd = Command(launcher=LauncherType.Slurm, command=["-N", "1"]) + cmd = Command(command=["-N", "1"]) cmd.insert(0, "salloc") assert cmd.command == ["salloc", "-N", "1"] diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 1a8c25179..79d6f7e78 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -32,9 +32,9 @@ pytestmark = pytest.mark.group_a -salloc_cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) -srun_cmd = Command(launcher=LauncherType.Slurm, command=["srun", "-n", "1"]) -sacct_cmd = Command(launcher=LauncherType.Slurm, command=["sacct", "--user"]) +salloc_cmd = Command(command=["salloc", "-N", "1"]) +srun_cmd = Command(command=["srun", "-n", "1"]) +sacct_cmd = Command(command=["sacct", "--user"]) def test_command_init(): @@ -42,16 +42,47 @@ def test_command_init(): assert cmd_list.commands == [salloc_cmd, srun_cmd] -def test_command_getitem(): +def test_command_getitem_int(): cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) get_value = cmd_list[0] assert get_value == salloc_cmd -def test_command_setitem(): +def test_command_getitem_slice(): + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) + get_value = cmd_list[0:2] + assert get_value == [salloc_cmd, srun_cmd] + + +def test_command_setitem_idx(): cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) cmd_list[0] = sacct_cmd - assert cmd_list.commands == [sacct_cmd, srun_cmd] + for cmd in cmd_list.commands: + assert cmd.command in [sacct_cmd.command, srun_cmd.command] + + +def test_command_setitem_slice(): + cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) + cmd_list[0:2] = [sacct_cmd, sacct_cmd] + for cmd in cmd_list.commands: + assert cmd.command == sacct_cmd.command + + +def test_command_setitem_fail(): + cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) + with pytest.raises(ValueError): + cmd_list[0] = "fail" + with pytest.raises(ValueError): + cmd_list[0:1] = "fail" + with pytest.raises(ValueError): + cmd_list[0:1] = "fail" + cmd_1 = Command(command=["salloc", "-N", 1]) + cmd_2 = Command(command=["salloc", "-N", "1"]) + cmd_3 = Command(command=1) + with pytest.raises(ValueError): + cmd_list[0:1] = [cmd_1, cmd_2] + with pytest.raises(ValueError): + cmd_list[0:1] = [cmd_3, cmd_2] def test_command_delitem(): diff --git a/tests/temp_tests/test_core/test_commands/test_launchCommands.py b/tests/temp_tests/test_core/test_commands/test_launchCommands.py index 913de208b..0c5e719cc 100644 --- a/tests/temp_tests/test_core/test_commands/test_launchCommands.py +++ b/tests/temp_tests/test_core/test_commands/test_launchCommands.py @@ -33,9 +33,9 @@ pytestmark = pytest.mark.group_a -pre_cmd = Command(launcher=LauncherType.Slurm, command=["pre", "cmd"]) -launch_cmd = Command(launcher=LauncherType.Slurm, command=["launch", "cmd"]) -post_cmd = Command(launcher=LauncherType.Slurm, command=["post", "cmd"]) +pre_cmd = Command(command=["pre", "cmd"]) +launch_cmd = Command(command=["launch", "cmd"]) +post_cmd = Command(command=["post", "cmd"]) pre_commands_list = CommandList(commands=[pre_cmd]) launch_command_list = CommandList(commands=[launch_cmd]) post_command_list = CommandList(commands=[post_cmd]) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 360e487df..3628bc351 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -23,8 +23,13 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import os +import pathlib + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.alps import ( AprunLaunchArguments, @@ -211,8 +216,17 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_aprun_command( - AprunLaunchArguments(args), mock_echo_executable, test_dir, {} + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, out, err ) - assert tuple(cmd) == expected - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 38ee11486..6e3722dde 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -78,7 +78,7 @@ def test_formatting_launch_args_into_request( if gpu_affinity is not NOT_SET: launch_args.set_gpu_affinity(gpu_affinity) req, policy = _as_run_request_args_and_policy( - launch_args, mock_echo_executable, test_dir, {} + launch_args, mock_echo_executable, test_dir, {}, "output.txt", "error.txt" ) expected_args = { @@ -90,7 +90,13 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET } expected_run_req = DragonRunRequestView( - exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, **expected_args + exe="echo", + exe_args=["hello", "world"], + path=test_dir, + env={}, + output_file="output.txt", + error_file="error.txt", + **expected_args, ) assert req.exe == expected_run_req.exe assert req.exe_args == expected_run_req.exe_args @@ -99,6 +105,8 @@ def test_formatting_launch_args_into_request( assert req.hostlist == expected_run_req.hostlist assert req.pmi_enabled == expected_run_req.pmi_enabled assert req.path == expected_run_req.path + assert req.output_file == expected_run_req.output_file + assert req.error_file == expected_run_req.error_file expected_run_policy_args = { k: v diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index e33684d4a..251659c6f 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -23,8 +23,13 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import os +import pathlib + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.local import ( LocalLaunchArguments, @@ -148,8 +153,17 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): - path, cmd = _as_local_command( - LocalLaunchArguments({}), mock_echo_executable, test_dir, {} + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, out, err ) - assert tuple(cmd) == ("echo", "hello", "world") - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == ("echo", "hello", "world") + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 54046d06e..2e56e4a6c 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -23,6 +23,8 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import subprocess + import pytest from smartsim.settings import LaunchSettings @@ -91,37 +93,102 @@ def test_launch_args(): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("jsrun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {}, + ( + "jsrun", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), + id="Empty Args", + ), pytest.param( {"n": "1"}, - ("jsrun", "-n", "1", "--", "echo", "hello", "world"), + ( + "jsrun", + "-n", + "1", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg", ), pytest.param( {"nrs": "1"}, - ("jsrun", "--nrs=1", "--", "echo", "hello", "world"), + ( + "jsrun", + "--nrs=1", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg", ), pytest.param( {"v": None}, - ("jsrun", "-v", "--", "echo", "hello", "world"), + ( + "jsrun", + "-v", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("jsrun", "--verbose", "--", "echo", "hello", "world"), + ( + "jsrun", + "--verbose", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg (No Value)", ), pytest.param( {"tasks_per_rs": "1", "n": "123"}, - ("jsrun", "--tasks_per_rs=1", "-n", "123", "--", "echo", "hello", "world"), + ( + "jsrun", + "--tasks_per_rs=1", + "-n", + "123", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_jsrun_command( - JsrunLaunchArguments(args), mock_echo_executable, test_dir, {} + outfile = "output.txt" + errfile = "error.txt" + env, path, stdin, stdout, args = _as_jsrun_command( + JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(cmd) == expected + assert tuple(args) == expected assert path == test_dir + assert env == {} + assert stdin == subprocess.DEVNULL + assert stdout == subprocess.DEVNULL diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index edd2f22e3..f2513a2f7 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -24,10 +24,14 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io import itertools +import os +import pathlib import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.mpi import ( MpiexecLaunchArguments, @@ -286,6 +290,15 @@ def test_invalid_hostlist_format(launcher): def test_formatting_launch_args( mock_echo_executable, cls, fmt, cmd, args, expected, test_dir ): - path, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}) - assert tuple(fmt_cmd) == (cmd,) + expected - assert path == test_dir + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, out, err) + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == (cmd,) + expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 8ea2f64f8..857b3799a 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -24,8 +24,13 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import os +import pathlib + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.pals import ( PalsMpiexecLaunchArguments, @@ -132,8 +137,22 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_pals_command( - PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {} + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_pals_command( + PalsMpiexecLaunchArguments(args), + mock_echo_executable, + test_dir, + {}, + out, + err, ) - assert tuple(cmd) == expected - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 5f86bf7db..9ec4f2022 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -23,8 +23,11 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import subprocess + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, @@ -290,37 +293,106 @@ def test_set_het_groups(monkeypatch): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("srun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {}, + ( + "srun", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), + id="Empty Args", + ), pytest.param( {"N": "1"}, - ("srun", "-N", "1", "--", "echo", "hello", "world"), + ( + "srun", + "-N", + "1", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg", ), pytest.param( {"nodes": "1"}, - ("srun", "--nodes=1", "--", "echo", "hello", "world"), + ( + "srun", + "--nodes=1", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg", ), pytest.param( {"v": None}, - ("srun", "-v", "--", "echo", "hello", "world"), + ( + "srun", + "-v", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("srun", "--verbose", "--", "echo", "hello", "world"), + ( + "srun", + "--verbose", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg (No Value)", ), pytest.param( {"nodes": "1", "n": "123"}, - ("srun", "--nodes=1", "-n", "123", "--", "echo", "hello", "world"), + ( + "srun", + "--nodes=1", + "-n", + "123", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_srun_command( - SlurmLaunchArguments(args), mock_echo_executable, test_dir, {} + shell_launch_cmd = _as_srun_command( + args=SlurmLaunchArguments(args), + exe=mock_echo_executable, + path=test_dir, + env={}, + stdout_path="output.txt", + stderr_path="error.txt", ) - assert tuple(cmd) == expected - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == subprocess.DEVNULL + assert shell_launch_cmd.stderr == subprocess.DEVNULL diff --git a/tests/test_experiment.py b/tests/test_experiment.py index c39afee1b..b7fb9732a 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -29,7 +29,6 @@ import dataclasses import itertools import random -import tempfile import typing as t import uuid @@ -55,7 +54,11 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job") + monkeypatch.setattr( + exp, + "_generate", + lambda gen, job, idx: ("/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt"), + ) yield exp @@ -65,8 +68,10 @@ def dispatcher(): dispatches any jobs with `MockLaunchArgs` to a `NoOpRecordLauncher` """ d = dispatch.Dispatcher() - to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( - lambda settings, exe, path, env: LaunchRecord(settings, exe, env, path) + to_record: dispatch.FormatterType[MockLaunchArgs, LaunchRecord] = ( + lambda settings, exe, path, env, out, err: LaunchRecord( + settings, exe, env, path, out, err + ) ) d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) yield d @@ -146,6 +151,8 @@ class LaunchRecord: entity: entity.SmartSimEntity env: t.Mapping[str, str | None] path: str + out: str + err: str @classmethod def from_job(cls, job: job.Job): @@ -161,7 +168,9 @@ def from_job(cls, job: job.Job): entity = job._entity env = job._launch_settings.env_vars path = "/tmp/job" - return cls(args, entity, env, path) + out = "/tmp/job/out.txt" + err = "/tmp/job/err.txt" + return cls(args, entity, env, path, out, err) class MockLaunchArgs(launchArguments.LaunchArguments): diff --git a/tests/test_generator.py b/tests/test_generator.py index 3bf69ef76..f1f18ceec 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -10,9 +10,8 @@ import pytest from smartsim import Experiment -from smartsim._core import dispatch from smartsim._core.generation.generator import Generator -from smartsim.entity import Application, Ensemble, SmartSimEntity, _mock +from smartsim.entity import Application, Ensemble from smartsim.entity.files import EntityFiles from smartsim.launchable import Job from smartsim.settings import LaunchSettings @@ -21,11 +20,11 @@ pytestmark = pytest.mark.group_a -ID_GENERATOR = (str(i) for i in itertools.count()) +_ID_GENERATOR = (str(i) for i in itertools.count()) def random_id(): - return next(ID_GENERATOR) + return next(_ID_GENERATOR) @pytest.fixture @@ -68,7 +67,7 @@ def test_generate_job_directory(test_dir, wlmutils, generator_instance): # Mock id run_id = "temp_id" # Call Generator.generate_job - job_run_path = generator_instance.generate_job(job, 0) + job_run_path, _, _ = generator_instance.generate_job(job, 0) assert isinstance(job_run_path, pathlib.Path) expected_run_path = ( pathlib.Path(test_dir) @@ -105,7 +104,7 @@ def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): job = Job(app, launch_settings) # Generate Job directory job_index = 1 - job_execution_path = exp._generate(generator_instance, job, job_index) + job_execution_path, _, _ = exp._generate(generator_instance, job, job_index) # Assert Job run directory exists assert osp.isdir(job_execution_path) # Assert Job log directory exists @@ -124,7 +123,7 @@ def test_generate_copy_file(generator_instance, fileutils, wlmutils): job = Job(app, launch_settings) # Create the experiment - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_file = pathlib.Path(path) / "sleep.py" assert osp.isfile(expected_file) @@ -137,7 +136,7 @@ def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance) job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_folder = path / "to_copy_dir" assert osp.isdir(expected_folder) @@ -152,7 +151,7 @@ def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlin job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_folder = path / "to_symlink_dir" assert osp.isdir(expected_folder) assert expected_folder.is_symlink() @@ -179,7 +178,7 @@ def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_file = path / "mock2.txt" assert osp.isfile(expected_file) assert expected_file.is_symlink() @@ -217,7 +216,7 @@ def test_generate_configure(fileutils, wlmutils, generator_instance): job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 0) + path, _, _ = generator_instance.generate_job(job, 0) # Retrieve the list of configured files in the test directory configured_files = sorted(glob(str(path) + "/*")) # Use filecmp.cmp to check that the corresponding files are equal @@ -232,7 +231,7 @@ def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_inst job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) for i, job in enumerate(job_list): - job_run_path = exp._generate(generator_instance, job, i) + job_run_path, _, _ = exp._generate(generator_instance, job, i) head, _ = os.path.split(job_run_path) expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(job_run_path) @@ -245,7 +244,7 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): job_list = ensemble.as_jobs(launch_settings) for i, job in enumerate(job_list): # Call Generator.generate_job - path = generator_instance.generate_job(job, i) + path, _, _ = generator_instance.generate_job(job, i) # Assert run directory created assert osp.isdir(path) # Assert smartsim params file created @@ -262,7 +261,7 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) @@ -282,7 +281,7 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) @@ -304,7 +303,7 @@ def test_generate_ensemble_symlink( ): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( "ensemble-name", @@ -331,7 +330,7 @@ def test_generate_ensemble_configure( ): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} # Retrieve a list of files for configuration @@ -346,7 +345,7 @@ def test_generate_ensemble_configure( launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) + id = exp.start(*job_list) run_dir = listdir(test_dir) jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py new file mode 100644 index 000000000..6b03f8501 --- /dev/null +++ b/tests/test_shell_launcher.py @@ -0,0 +1,312 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import pathlib +import subprocess +import unittest.mock + +import psutil +import pytest + +from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand, sp +from smartsim._core.utils import helpers +from smartsim._core.utils.shell import * +from smartsim.entity import _mock, entity +from smartsim.error.errors import LauncherJobNotFound +from smartsim.status import JobStatus + +pytestmark = pytest.mark.group_a + + +class EchoHelloWorldEntity(entity.SmartSimEntity): + """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" + + def __init__(self): + super().__init__("test-entity", _mock.Mock()) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return self.as_program_arguments() == other.as_program_arguments() + + def as_program_arguments(self): + return (helpers.expand_exe_path("echo"), "Hello", "World!") + + +def create_directory(directory_path: str) -> pathlib.Path: + """Creates the execution directory for testing.""" + tmp_dir = pathlib.Path(directory_path) + tmp_dir.mkdir(exist_ok=True, parents=True) + return tmp_dir + + +def generate_output_files(tmp_dir: pathlib.Path): + """Generates output and error files within the run directory for testing.""" + out_file = tmp_dir / "tmp.out" + err_file = tmp_dir / "tmp.err" + return out_file, err_file + + +def generate_directory(test_dir: str): + """Generates a execution directory, output file, and error file for testing.""" + execution_dir = create_directory(os.path.join(test_dir, "tmp")) + out_file, err_file = generate_output_files(execution_dir) + return execution_dir, out_file, err_file + + +@pytest.fixture +def shell_launcher(): + launcher = ShellLauncher() + yield launcher + if any(proc.poll() is None for proc in launcher._launched.values()): + raise ("Test leaked processes") + + +@pytest.fixture +def shell_cmd(test_dir: str) -> ShellLauncherCommand: + """Fixture to create an instance of Generator.""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + yield ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) + + +# UNIT TESTS + + +def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: str): + """Test that ShellLauncherCommand initializes correctly""" + assert shell_cmd.env == {} + assert shell_cmd.path == pathlib.Path(test_dir) / "tmp" + assert shell_cmd.stdout.name == os.path.join(test_dir, "tmp", "tmp.out") + assert shell_cmd.stderr.name == os.path.join(test_dir, "tmp", "tmp.err") + assert shell_cmd.command_tuple == EchoHelloWorldEntity().as_program_arguments() + + +def test_shell_launcher_init(shell_launcher: ShellLauncher): + """Test that ShellLauncher initializes correctly""" + assert shell_launcher._launched == {} + + +def test_check_popen_inputs(shell_launcher: ShellLauncher, test_dir: str): + """Test that ShellLauncher.check_popen_inputs throws correctly""" + cmd = ShellLauncherCommand( + {}, + pathlib.Path(test_dir) / "directory_dne", + subprocess.DEVNULL, + subprocess.DEVNULL, + EchoHelloWorldEntity().as_program_arguments(), + ) + with pytest.raises(ValueError): + _ = shell_launcher.start(cmd) + + +def test_shell_launcher_start_calls_popen( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand +): + """Test that the process leading up to the shell launcher popen call was correct""" + with unittest.mock.patch( + "smartsim._core.shell.shellLauncher.sp.Popen" + ) as mock_open: + _ = shell_launcher.start(shell_cmd) + mock_open.assert_called_once() + + +def test_shell_launcher_start_calls_popen_with_value( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand +): + """Test that popen was called with correct values""" + with unittest.mock.patch( + "smartsim._core.shell.shellLauncher.sp.Popen" + ) as mock_open: + _ = shell_launcher.start(shell_cmd) + mock_open.assert_called_once_with( + shell_cmd.command_tuple, + cwd=shell_cmd.path, + env=shell_cmd.env, + stdout=shell_cmd.stdout, + stderr=shell_cmd.stderr, + ) + + +def test_popen_returns_popen_object( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Test that the popen call returns a popen object""" + id = shell_launcher.start(shell_cmd) + with shell_launcher._launched[id] as proc: + assert isinstance(proc, sp.Popen) + + +def test_popen_writes_to_output_file( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Test that popen writes to .out file upon successful process call""" + _, out_file, err_file = generate_directory(test_dir) + id = shell_launcher.start(shell_cmd) + proc = shell_launcher._launched[id] + assert proc.wait() == 0 + assert proc.returncode == 0 + with open(out_file, "r", encoding="utf-8") as out: + assert out.read() == "Hello World!\n" + with open(err_file, "r", encoding="utf-8") as err: + assert err.read() == "" + + +def test_popen_fails_with_invalid_cmd(shell_launcher: ShellLauncher, test_dir: str): + """Test that popen returns a non zero returncode after failure""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + args = (helpers.expand_exe_path("ls"), "--flag_dne") + cmd = ShellLauncherCommand({}, run_dir, out, err, args) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + proc.wait() + assert proc.returncode != 0 + with open(out_file, "r", encoding="utf-8") as out: + assert out.read() == "" + with open(err_file, "r", encoding="utf-8") as err: + content = err.read() + assert "unrecognized option" in content + + +def test_popen_issues_unique_ids( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Validate that all ids are unique within ShellLauncher._launched""" + seen = set() + for _ in range(5): + id = shell_launcher.start(shell_cmd) + assert id not in seen, "Duplicate ID issued" + seen.add(id) + assert len(shell_launcher._launched) == 5 + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) + + +def test_retrieve_status_dne(shell_launcher: ShellLauncher): + """Test tht ShellLauncher returns the status of completed Jobs""" + with pytest.raises(LauncherJobNotFound): + _ = shell_launcher.get_status("dne") + + +def test_shell_launcher_returns_complete_status( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Test tht ShellLauncher returns the status of completed Jobs""" + for _ in range(5): + id = shell_launcher.start(shell_cmd) + proc = shell_launcher._launched[id] + proc.wait() + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.COMPLETED + + +def test_shell_launcher_returns_failed_status( + shell_launcher: ShellLauncher, test_dir: str +): + """Test tht ShellLauncher returns the status of completed Jobs""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + args = (helpers.expand_exe_path("ls"), "--flag_dne") + cmd = ShellLauncherCommand({}, run_dir, out, err, args) + for _ in range(5): + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + proc.wait() + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.FAILED + + +def test_shell_launcher_returns_running_status( + shell_launcher: ShellLauncher, test_dir: str +): + """Test tht ShellLauncher returns the status of completed Jobs""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5") + ) + for _ in range(5): + id = shell_launcher.start(cmd) + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.RUNNING + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) + + +@pytest.mark.parametrize( + "psutil_status,job_status", + [ + pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="running"), + pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="sleeping"), + pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="waking"), + pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="disk_sleep"), + pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="dead"), + pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="tracing_stop"), + pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="waiting"), + pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="stopped"), + pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="locked"), + pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="parked"), + pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="idle"), + pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="zombie"), + pytest.param( + "some-brand-new-unknown-status-str", JobStatus.UNKNOWN, id="unknown" + ), + ], +) +def test_get_status_maps_correctly( + psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir: str +): + """Test tht ShellLauncher.get_status returns correct mapping""" + shell_launcher = ShellLauncher() + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + monkeypatch.setattr(proc, "poll", lambda: None) + monkeypatch.setattr(psutil.Process, "status", lambda self: psutil_status) + value = shell_launcher.get_status(id) + assert value.get(id) == job_status + assert proc.wait() == 0