From f9a86d9c7e568880e4f348c09b17671f7722fa4f Mon Sep 17 00:00:00 2001 From: amandarichardsonn <30413257+amandarichardsonn@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:15:44 -0700 Subject: [PATCH] Unit tests for ShellLauncher & additional shell launch context (#671) This PR merges in additional context to the ShellLauncher.start function to open a subprocess. The working directory is now set in popen, the environment variables are now set in popen, and the file paths to which standard output and standard error streams should be redirected to have been set. Additionally this PR merges in Unit tests for the ShellLauncher class. [ reviewed by @MattToast ] [ committed by @amandarichardsonn ] --- .github/workflows/run_tests.yml | 2 +- pyproject.toml | 1 - smartsim/_core/commands/command.py | 52 ++- smartsim/_core/commands/commandList.py | 41 ++- smartsim/_core/dispatch.py | 42 ++- smartsim/_core/generation/generator.py | 17 +- .../_core/launcher/dragon/dragonLauncher.py | 7 +- smartsim/_core/shell/shellLauncher.py | 138 ++++---- smartsim/experiment.py | 13 +- smartsim/launchable/baseJobGroup.py | 26 ++ smartsim/launchable/colocatedJobGroup.py | 26 ++ smartsim/launchable/jobGroup.py | 26 ++ smartsim/settings/arguments/launch/lsf.py | 29 +- smartsim/settings/arguments/launch/slurm.py | 29 +- .../test_core/test_commands/test_command.py | 37 ++- .../test_commands/test_commandList.py | 43 ++- .../test_commands/test_launchCommands.py | 6 +- .../test_settings/test_alpsLauncher.py | 22 +- .../test_settings/test_dragonLauncher.py | 12 +- .../test_settings/test_localLauncher.py | 22 +- .../test_settings/test_lsfLauncher.py | 85 ++++- .../test_settings/test_mpiLauncher.py | 19 +- .../test_settings/test_palsLauncher.py | 27 +- .../test_settings/test_slurmLauncher.py | 92 +++++- tests/test_experiment.py | 19 +- tests/test_generator.py | 44 ++- tests/test_shell_launcher.py | 312 ++++++++++++++++++ 27 files changed, 1001 insertions(+), 188 deletions(-) create mode 100644 tests/test_shell_launcher.py diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 3230d8da8..50a05fbad 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -157,7 +157,7 @@ jobs: run: | echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests - + # Upload artifacts on failure, ignoring binary files - name: Upload Artifact if: failure() diff --git a/pyproject.toml b/pyproject.toml index 5df64aa97..e11c252ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,7 +147,6 @@ module = [ # FIXME: DO NOT MERGE THIS INTO DEVELOP BRANCH UNLESS THESE ARE PASSING OR # REMOVED!! "smartsim._core._cli.*", - "smartsim._core.commands.*", "smartsim._core.control.controller", "smartsim._core.control.manifest", "smartsim._core.entrypoints.dragon_client", diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index d89aa41ad..3f41f32fe 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -26,25 +26,18 @@ import typing as t from collections.abc import MutableSequence +from copy import deepcopy -from ...settings.launchCommand import LauncherType +from typing_extensions import Self class Command(MutableSequence[str]): """Basic container for command information""" - def __init__(self, launcher: LauncherType, command: t.List[str]) -> None: + def __init__(self, command: t.List[str]) -> None: """Command constructor""" - self._launcher = launcher self._command = command - @property - def launcher(self) -> LauncherType: - """Get the launcher type. - Return a reference to the LauncherType. - """ - return self._launcher - @property def command(self) -> t.List[str]: """Get the command list. @@ -52,15 +45,41 @@ def command(self) -> t.List[str]: """ return self._command - def __getitem__(self, idx: int) -> str: + @t.overload + def __getitem__(self, idx: int) -> str: ... + @t.overload + def __getitem__(self, idx: slice) -> Self: ... + def __getitem__(self, idx: t.Union[int, slice]) -> t.Union[str, Self]: """Get the command at the specified index.""" - return self._command[idx] + cmd = self._command[idx] + if isinstance(cmd, str): + return cmd + return type(self)(cmd) - def __setitem__(self, idx: int, value: str) -> None: + @t.overload + def __setitem__(self, idx: int, value: str) -> None: ... + @t.overload + def __setitem__(self, idx: slice, value: t.Iterable[str]) -> None: ... + def __setitem__( + self, idx: t.Union[int, slice], value: t.Union[str, t.Iterable[str]] + ) -> None: """Set the command at the specified index.""" - self._command[idx] = value + if isinstance(idx, int): + if not isinstance(value, str): + raise ValueError( + "Value must be of type `str` when assigning to an index" + ) + self._command[idx] = deepcopy(value) + return + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "Value must be a list of strings when assigning to a slice" + ) + self._command[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int) -> None: + def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the command at the specified index.""" del self._command[idx] @@ -73,6 +92,5 @@ def insert(self, idx: int, value: str) -> None: self._command.insert(idx, value) def __str__(self) -> str: # pragma: no cover - string = f"\nLauncher: {self.launcher.value}\n" - string += f"Command: {' '.join(str(cmd) for cmd in self.command)}" + string = f"\nCommand: {' '.join(str(cmd) for cmd in self.command)}" return string diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 08b95bbfd..34743063e 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -26,6 +26,7 @@ import typing as t from collections.abc import MutableSequence +from copy import deepcopy from .command import Command @@ -46,15 +47,45 @@ def commands(self) -> t.List[Command]: """ return self._commands - def __getitem__(self, idx: int) -> Command: + @t.overload + def __getitem__(self, idx: int) -> Command: ... + @t.overload + def __getitem__(self, idx: slice) -> t.List[Command]: ... + def __getitem__( + self, idx: t.Union[slice, int] + ) -> t.Union[Command, t.List[Command]]: """Get the Command at the specified index.""" return self._commands[idx] - def __setitem__(self, idx: int, value: Command) -> None: - """Set the Command at the specified index.""" - self._commands[idx] = value + @t.overload + def __setitem__(self, idx: int, value: Command) -> None: ... + @t.overload + def __setitem__(self, idx: slice, value: t.Iterable[Command]) -> None: ... + def __setitem__( + self, idx: t.Union[int, slice], value: t.Union[Command, t.Iterable[Command]] + ) -> None: + """Set the Commands at the specified index.""" + if isinstance(idx, int): + if not isinstance(value, Command): + raise ValueError( + "Value must be of type `Command` when assigning to an index" + ) + self._commands[idx] = deepcopy(value) + return + if not isinstance(value, list): + raise ValueError( + "Value must be a list of Commands when assigning to a slice" + ) + for sublist in value: + if not isinstance(sublist.command, list) or not all( + isinstance(item, str) for item in sublist.command + ): + raise ValueError( + "Value sublists must be a list of Commands when assigning to a slice" + ) + self._commands[idx] = (deepcopy(val) for val in value) - def __delitem__(self, idx: int) -> None: + def __delitem__(self, idx: t.Union[int, slice]) -> None: """Delete the Command at the specified index.""" del self._commands[idx] diff --git a/smartsim/_core/dispatch.py b/smartsim/_core/dispatch.py index b774baade..551c27d18 100644 --- a/smartsim/_core/dispatch.py +++ b/smartsim/_core/dispatch.py @@ -28,6 +28,7 @@ import dataclasses import os +import pathlib import typing as t from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack @@ -42,10 +43,11 @@ from smartsim.experiment import Experiment from smartsim.settings.arguments import LaunchArguments + _Ts = TypeVarTuple("_Ts") -_WorkingDirectory: TypeAlias = t.Union[str, os.PathLike[str]] +WorkingDirectory: TypeAlias = pathlib.Path """A working directory represented as a string or PathLike object""" _DispatchableT = t.TypeVar("_DispatchableT", bound="LaunchArguments") @@ -57,20 +59,30 @@ to the to the `LauncherProtocol.start` method """ -_EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] +EnvironMappingType: TypeAlias = t.Mapping[str, "str | None"] """A mapping of user provided mapping of environment variables in which to run a job """ -_FormatterType: TypeAlias = t.Callable[ - [_DispatchableT, "ExecutableProtocol", _WorkingDirectory, _EnvironMappingType], +FormatterType: TypeAlias = t.Callable[ + [ + _DispatchableT, + "ExecutableProtocol", + WorkingDirectory, + EnvironMappingType, + pathlib.Path, + pathlib.Path, + ], _LaunchableT, ] """A callable that is capable of formatting the components of a job into a type capable of being launched by a launcher. """ -_LaunchConfigType: TypeAlias = ( - "_LauncherAdapter[ExecutableProtocol, _WorkingDirectory, _EnvironMappingType]" -) +_LaunchConfigType: TypeAlias = """_LauncherAdapter[ + ExecutableProtocol, + WorkingDirectory, + EnvironMappingType, + pathlib.Path, + pathlib.Path]""" """A launcher adapater that has configured a launcher to launch the components of a job with some pre-determined launch settings @@ -133,7 +145,7 @@ def dispatch( # Signature when used as a decorator self, args: None = ..., *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]]: ... @@ -142,7 +154,7 @@ def dispatch( # Signature when used as a method self, args: type[_DispatchableT], *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = ..., ) -> None: ... @@ -150,7 +162,7 @@ def dispatch( # Actual implementation self, args: type[_DispatchableT] | None = None, *, - with_format: _FormatterType[_DispatchableT, _LaunchableT], + with_format: FormatterType[_DispatchableT, _LaunchableT], to_launcher: type[LauncherProtocol[_LaunchableT]], allow_overwrite: bool = False, ) -> t.Callable[[type[_DispatchableT]], type[_DispatchableT]] | None: @@ -216,7 +228,7 @@ class _DispatchRegistration(t.Generic[_DispatchableT, _LaunchableT]): to be launched by the afore mentioned launcher. """ - formatter: _FormatterType[_DispatchableT, _LaunchableT] + formatter: FormatterType[_DispatchableT, _LaunchableT] launcher_type: type[LauncherProtocol[_LaunchableT]] def _is_compatible_launcher(self, launcher: LauncherProtocol[t.Any]) -> bool: @@ -260,10 +272,12 @@ def create_adapter_from_launcher( def format_( exe: ExecutableProtocol, - path: str | os.PathLike[str], - env: _EnvironMappingType, + path: pathlib.Path, + env: EnvironMappingType, + out: pathlib.Path, + err: pathlib.Path, ) -> _LaunchableT: - return self.formatter(arguments, exe, path, env) + return self.formatter(arguments, exe, path, env, out, err) return _LauncherAdapter(launcher, format_) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 9c58cceaa..e4018ccc3 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -108,7 +108,17 @@ def _log_file(log_path: pathlib.Path) -> pathlib.Path: """ return pathlib.Path(log_path) / "smartsim_params.txt" - def generate_job(self, job: Job, job_index: int) -> pathlib.Path: + @staticmethod + def _output_files( + log_path: pathlib.Path, job_name: str + ) -> t.Tuple[pathlib.Path, pathlib.Path]: + out_file_path = log_path / f"{job_name}.out" + err_file_path = log_path / f"{job_name}.err" + return out_file_path, err_file_path + + def generate_job( + self, job: Job, job_index: int + ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Write and configure input files for a Job. To have files or directories present in the created Job @@ -136,10 +146,13 @@ def generate_job(self, job: Job, job_index: int) -> pathlib.Path: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") + # Create output files + out_file, err_file = self._output_files(log_path, job.entity.name) + # Perform file system operations on attached files self._build_operations(job, job_path) - return job_path + return job_path, out_file, err_file @classmethod def _build_operations(cls, job: Job, job_path: pathlib.Path) -> None: diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 39e195881..e6ade8dba 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -27,6 +27,7 @@ from __future__ import annotations import os +import pathlib import typing as t from smartsim._core.schemas.dragonRequests import DragonRunPolicy @@ -371,6 +372,8 @@ def _as_run_request_args_and_policy( exe: ExecutableProtocol, path: str | os.PathLike[str], env: t.Mapping[str, str | None], + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, ) -> tuple[DragonRunRequestView, DragonRunPolicy]: # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # FIXME: This type is 100% unacceptable, but I don't want to spend too much @@ -392,8 +395,8 @@ def _as_run_request_args_and_policy( env=env, # TODO: Not sure how this info is injected name=None, - output_file=None, - error_file=None, + output_file=stdout_path, + error_file=stderr_path, **run_args, ), policy, diff --git a/smartsim/_core/shell/shellLauncher.py b/smartsim/_core/shell/shellLauncher.py index 95ded35dd..1197e2569 100644 --- a/smartsim/_core/shell/shellLauncher.py +++ b/smartsim/_core/shell/shellLauncher.py @@ -27,14 +27,15 @@ from __future__ import annotations -import os +import io +import pathlib import subprocess as sp import typing as t import psutil from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import _EnvironMappingType, _FormatterType, dispatch +from smartsim._core.dispatch import EnvironMappingType, FormatterType, WorkingDirectory from smartsim._core.utils import helpers from smartsim._core.utils.launcher import ExecutableProtocol, create_job_id from smartsim.error import errors @@ -51,62 +52,17 @@ logger = get_logger(__name__) -class ShellLauncher: - """Mock launcher for launching/tracking simple shell commands""" - - def __init__(self) -> None: - self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - - def start( - self, command: tuple[str | os.PathLike[str], t.Sequence[str]] - ) -> LaunchedJobID: - id_ = create_job_id() - path, args = command - exe, *rest = args - # pylint: disable-next=consider-using-with - self._launched[id_] = sp.Popen((helpers.expand_exe_path(exe), *rest), cwd=path) - return id_ - - def get_status( - self, *launched_ids: LaunchedJobID - ) -> t.Mapping[LaunchedJobID, JobStatus]: - return {id_: self._get_status(id_) for id_ in launched_ids} - - def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: - if (proc := self._launched.get(id_)) is None: - msg = f"Launcher `{self}` has not launched a job with id `{id_}`" - raise errors.LauncherJobNotFound(msg) - ret_code = proc.poll() - if ret_code is None: - status = psutil.Process(proc.pid).status() - return { - psutil.STATUS_RUNNING: JobStatus.RUNNING, - psutil.STATUS_SLEEPING: JobStatus.RUNNING, - psutil.STATUS_WAKING: JobStatus.RUNNING, - psutil.STATUS_DISK_SLEEP: JobStatus.RUNNING, - psutil.STATUS_DEAD: JobStatus.FAILED, - psutil.STATUS_TRACING_STOP: JobStatus.PAUSED, - psutil.STATUS_WAITING: JobStatus.PAUSED, - psutil.STATUS_STOPPED: JobStatus.PAUSED, - psutil.STATUS_LOCKED: JobStatus.PAUSED, - psutil.STATUS_PARKED: JobStatus.PAUSED, - psutil.STATUS_IDLE: JobStatus.PAUSED, - psutil.STATUS_ZOMBIE: JobStatus.COMPLETED, - }.get(status, JobStatus.UNKNOWN) - if ret_code == 0: - return JobStatus.COMPLETED - return JobStatus.FAILED - - @classmethod - def create(cls, _: Experiment) -> Self: - return cls() +class ShellLauncherCommand(t.NamedTuple): + env: EnvironMappingType + path: pathlib.Path + stdout: io.TextIOWrapper | int + stderr: io.TextIOWrapper | int + command_tuple: tuple[str, tuple[str, ...]] | t.Sequence[str] def make_shell_format_fn( run_command: str | None, -) -> _FormatterType[ - ShellLaunchArguments, tuple[str | os.PathLike[str], t.Sequence[str]] -]: +) -> FormatterType[ShellLaunchArguments, ShellLauncherCommand]: """A function that builds a function that formats a `LaunchArguments` as a shell executable sequence of strings for a given launching utility. @@ -139,10 +95,12 @@ def make_shell_format_fn( def impl( args: ShellLaunchArguments, exe: ExecutableProtocol, - path: str | os.PathLike[str], - _env: _EnvironMappingType, - ) -> t.Tuple[str | os.PathLike[str], t.Sequence[str]]: - return path, ( + path: WorkingDirectory, + env: EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, + ) -> ShellLauncherCommand: + command_tuple = ( ( run_command, *(args.format_launch_args() or ()), @@ -152,5 +110,69 @@ def impl( if run_command is not None else exe.as_program_arguments() ) + # pylint: disable-next=consider-using-with + return ShellLauncherCommand( + env, pathlib.Path(path), open(stdout_path), open(stderr_path), command_tuple + ) return impl + + +class ShellLauncher: + """Mock launcher for launching/tracking simple shell commands""" + + def __init__(self) -> None: + self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} + + def check_popen_inputs(self, shell_command: ShellLauncherCommand) -> None: + if not shell_command.path.exists(): + raise ValueError("Please provide a valid path to ShellLauncherCommand.") + + def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: + self.check_popen_inputs(shell_command) + id_ = create_job_id() + exe, *rest = shell_command.command_tuple + expanded_exe = helpers.expand_exe_path(exe) + # pylint: disable-next=consider-using-with + self._launched[id_] = sp.Popen( + (expanded_exe, *rest), + cwd=shell_command.path, + env={k: v for k, v in shell_command.env.items() if v is not None}, + stdout=shell_command.stdout, + stderr=shell_command.stderr, + ) + return id_ + + def get_status( + self, *launched_ids: LaunchedJobID + ) -> t.Mapping[LaunchedJobID, JobStatus]: + return {id_: self._get_status(id_) for id_ in launched_ids} + + def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: + if (proc := self._launched.get(id_)) is None: + msg = f"Launcher `{self}` has not launched a job with id `{id_}`" + raise errors.LauncherJobNotFound(msg) + ret_code = proc.poll() + if ret_code is None: + status = psutil.Process(proc.pid).status() + return { + psutil.STATUS_RUNNING: JobStatus.RUNNING, + psutil.STATUS_SLEEPING: JobStatus.RUNNING, + psutil.STATUS_WAKING: JobStatus.RUNNING, + psutil.STATUS_DISK_SLEEP: JobStatus.RUNNING, + psutil.STATUS_DEAD: JobStatus.FAILED, + psutil.STATUS_TRACING_STOP: JobStatus.PAUSED, + psutil.STATUS_WAITING: JobStatus.PAUSED, + psutil.STATUS_STOPPED: JobStatus.PAUSED, + psutil.STATUS_LOCKED: JobStatus.PAUSED, + psutil.STATUS_PARKED: JobStatus.PAUSED, + psutil.STATUS_IDLE: JobStatus.PAUSED, + psutil.STATUS_ZOMBIE: JobStatus.COMPLETED, + }.get(status, JobStatus.UNKNOWN) + if ret_code == 0: + return JobStatus.COMPLETED + return JobStatus.FAILED + + @classmethod + def create(cls, _: Experiment) -> Self: + return cls() diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 8cb4dad24..94e172c36 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -232,8 +232,9 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: launch_config = dispatch.create_new_launcher_configuration( for_experiment=self, with_arguments=args ) - job_execution_path = self._generate(generator, job, idx) - id_ = launch_config.start(exe, job_execution_path, env) + # Generate the job directory and return the generated job path + job_execution_path, out, err = self._generate(generator, job, idx) + id_ = launch_config.start(exe, job_execution_path, env, out, err) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each # individual job, and the experiment can monitor job statuses. @@ -277,7 +278,9 @@ def get_status( return tuple(stats) @_contextualize - def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.Path: + def _generate( + self, generator: Generator, job: Job, job_index: int + ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: """Generate the directory structure and files for a ``Job`` If files or directories are attached to an ``Application`` object @@ -293,8 +296,8 @@ def _generate(self, generator: Generator, job: Job, job_index: int) -> pathlib.P :raises: A SmartSimError if an error occurs during the generation process. """ try: - job_run_path = generator.generate_job(job, job_index) - return job_run_path + job_path, out, err = generator.generate_job(job, job_index) + return (job_path, out, err) except SmartSimError as e: logger.error(e) raise diff --git a/smartsim/launchable/baseJobGroup.py b/smartsim/launchable/baseJobGroup.py index d662550f5..b7becba56 100644 --- a/smartsim/launchable/baseJobGroup.py +++ b/smartsim/launchable/baseJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/colocatedJobGroup.py b/smartsim/launchable/colocatedJobGroup.py index 97e7aa4a3..1c3b96fba 100644 --- a/smartsim/launchable/colocatedJobGroup.py +++ b/smartsim/launchable/colocatedJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index 65914cde4..3de767711 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index 54427d5a7..6177cb6b6 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -26,18 +26,41 @@ from __future__ import annotations +import pathlib +import subprocess import typing as t from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import dispatch -from smartsim._core.shell.shellLauncher import ShellLauncher, make_shell_format_fn +from smartsim._core.dispatch import EnvironMappingType, dispatch +from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand +from smartsim._core.utils.launcher import ExecutableProtocol from smartsim.log import get_logger from ...common import set_check_input from ...launchCommand import LauncherType logger = get_logger(__name__) -_as_jsrun_command = make_shell_format_fn(run_command="jsrun") + + +def _as_jsrun_command( + args: ShellLaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, +) -> ShellLauncherCommand: + command_tuple = ( + "jsrun", + *(args.format_launch_args() or ()), + f"--stdio_stdout={stdout_path}", + f"--stdio_stderr={stderr_path}", + "--", + *exe.as_program_arguments(), + ) + return ShellLauncherCommand( + env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple + ) @dispatch(with_format=_as_jsrun_command, to_launcher=ShellLauncher) diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index a1b12728b..adbbfab93 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -27,19 +27,42 @@ from __future__ import annotations import os +import pathlib import re +import subprocess import typing as t from smartsim._core.arguments.shell import ShellLaunchArguments -from smartsim._core.dispatch import dispatch -from smartsim._core.shell.shellLauncher import ShellLauncher, make_shell_format_fn +from smartsim._core.dispatch import EnvironMappingType, dispatch +from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand +from smartsim._core.utils.launcher import ExecutableProtocol from smartsim.log import get_logger from ...common import set_check_input from ...launchCommand import LauncherType logger = get_logger(__name__) -_as_srun_command = make_shell_format_fn(run_command="srun") + + +def _as_srun_command( + args: ShellLaunchArguments, + exe: ExecutableProtocol, + path: pathlib.Path, + env: EnvironMappingType, + stdout_path: pathlib.Path, + stderr_path: pathlib.Path, +) -> ShellLauncherCommand: + command_tuple = ( + "srun", + *(args.format_launch_args() or ()), + f"--output={stdout_path}", + f"--error={stderr_path}", + "--", + *exe.as_program_arguments(), + ) + return ShellLauncherCommand( + env, path, subprocess.DEVNULL, subprocess.DEVNULL, command_tuple + ) @dispatch(with_format=_as_srun_command, to_launcher=ShellLauncher) diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 71b1b87ff..2d1ddfbe8 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -27,33 +27,50 @@ import pytest from smartsim._core.commands.command import Command -from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a def test_command_init(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] - assert cmd.launcher == LauncherType.Slurm -def test_command_getitem(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) +def test_command_getitem_int(): + cmd = Command(command=["salloc", "-N", "1"]) get_value = cmd[0] assert get_value == "salloc" -def test_command_setitem(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) +def test_command_getitem_slice(): + cmd = Command(command=["salloc", "-N", "1"]) + get_value = cmd[0:2] + assert get_value.command == ["salloc", "-N"] + + +def test_command_setitem_int(): + cmd = Command(command=["salloc", "-N", "1"]) cmd[0] = "srun" cmd[1] = "-n" assert cmd.command == ["srun", "-n", "1"] +def test_command_setitem_slice(): + cmd = Command(command=["salloc", "-N", "1"]) + cmd[0:2] = ["srun", "-n"] + assert cmd.command == ["srun", "-n", "1"] + + +def test_command_setitem_fail(): + cmd = Command(command=["salloc", "-N", "1"]) + with pytest.raises(ValueError): + cmd[0] = 1 + with pytest.raises(ValueError): + cmd[0:2] = [1, "-n"] + + def test_command_delitem(): cmd = Command( - launcher=LauncherType.Slurm, command=["salloc", "-N", "1", "--constraint", "P100"], ) del cmd.command[3] @@ -62,11 +79,11 @@ def test_command_delitem(): def test_command_len(): - cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) + cmd = Command(command=["salloc", "-N", "1"]) assert len(cmd) is 3 def test_command_insert(): - cmd = Command(launcher=LauncherType.Slurm, command=["-N", "1"]) + cmd = Command(command=["-N", "1"]) cmd.insert(0, "salloc") assert cmd.command == ["salloc", "-N", "1"] diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 1a8c25179..79d6f7e78 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -32,9 +32,9 @@ pytestmark = pytest.mark.group_a -salloc_cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) -srun_cmd = Command(launcher=LauncherType.Slurm, command=["srun", "-n", "1"]) -sacct_cmd = Command(launcher=LauncherType.Slurm, command=["sacct", "--user"]) +salloc_cmd = Command(command=["salloc", "-N", "1"]) +srun_cmd = Command(command=["srun", "-n", "1"]) +sacct_cmd = Command(command=["sacct", "--user"]) def test_command_init(): @@ -42,16 +42,47 @@ def test_command_init(): assert cmd_list.commands == [salloc_cmd, srun_cmd] -def test_command_getitem(): +def test_command_getitem_int(): cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) get_value = cmd_list[0] assert get_value == salloc_cmd -def test_command_setitem(): +def test_command_getitem_slice(): + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) + get_value = cmd_list[0:2] + assert get_value == [salloc_cmd, srun_cmd] + + +def test_command_setitem_idx(): cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) cmd_list[0] = sacct_cmd - assert cmd_list.commands == [sacct_cmd, srun_cmd] + for cmd in cmd_list.commands: + assert cmd.command in [sacct_cmd.command, srun_cmd.command] + + +def test_command_setitem_slice(): + cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) + cmd_list[0:2] = [sacct_cmd, sacct_cmd] + for cmd in cmd_list.commands: + assert cmd.command == sacct_cmd.command + + +def test_command_setitem_fail(): + cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) + with pytest.raises(ValueError): + cmd_list[0] = "fail" + with pytest.raises(ValueError): + cmd_list[0:1] = "fail" + with pytest.raises(ValueError): + cmd_list[0:1] = "fail" + cmd_1 = Command(command=["salloc", "-N", 1]) + cmd_2 = Command(command=["salloc", "-N", "1"]) + cmd_3 = Command(command=1) + with pytest.raises(ValueError): + cmd_list[0:1] = [cmd_1, cmd_2] + with pytest.raises(ValueError): + cmd_list[0:1] = [cmd_3, cmd_2] def test_command_delitem(): diff --git a/tests/temp_tests/test_core/test_commands/test_launchCommands.py b/tests/temp_tests/test_core/test_commands/test_launchCommands.py index 913de208b..0c5e719cc 100644 --- a/tests/temp_tests/test_core/test_commands/test_launchCommands.py +++ b/tests/temp_tests/test_core/test_commands/test_launchCommands.py @@ -33,9 +33,9 @@ pytestmark = pytest.mark.group_a -pre_cmd = Command(launcher=LauncherType.Slurm, command=["pre", "cmd"]) -launch_cmd = Command(launcher=LauncherType.Slurm, command=["launch", "cmd"]) -post_cmd = Command(launcher=LauncherType.Slurm, command=["post", "cmd"]) +pre_cmd = Command(command=["pre", "cmd"]) +launch_cmd = Command(command=["launch", "cmd"]) +post_cmd = Command(command=["post", "cmd"]) pre_commands_list = CommandList(commands=[pre_cmd]) launch_command_list = CommandList(commands=[launch_cmd]) post_command_list = CommandList(commands=[post_cmd]) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 360e487df..3628bc351 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -23,8 +23,13 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import os +import pathlib + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.alps import ( AprunLaunchArguments, @@ -211,8 +216,17 @@ def test_invalid_exclude_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_aprun_command( - AprunLaunchArguments(args), mock_echo_executable, test_dir, {} + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_aprun_command( + AprunLaunchArguments(args), mock_echo_executable, test_dir, {}, out, err ) - assert tuple(cmd) == expected - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 38ee11486..6e3722dde 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -78,7 +78,7 @@ def test_formatting_launch_args_into_request( if gpu_affinity is not NOT_SET: launch_args.set_gpu_affinity(gpu_affinity) req, policy = _as_run_request_args_and_policy( - launch_args, mock_echo_executable, test_dir, {} + launch_args, mock_echo_executable, test_dir, {}, "output.txt", "error.txt" ) expected_args = { @@ -90,7 +90,13 @@ def test_formatting_launch_args_into_request( if v is not NOT_SET } expected_run_req = DragonRunRequestView( - exe="echo", exe_args=["hello", "world"], path=test_dir, env={}, **expected_args + exe="echo", + exe_args=["hello", "world"], + path=test_dir, + env={}, + output_file="output.txt", + error_file="error.txt", + **expected_args, ) assert req.exe == expected_run_req.exe assert req.exe_args == expected_run_req.exe_args @@ -99,6 +105,8 @@ def test_formatting_launch_args_into_request( assert req.hostlist == expected_run_req.hostlist assert req.pmi_enabled == expected_run_req.pmi_enabled assert req.path == expected_run_req.path + assert req.output_file == expected_run_req.output_file + assert req.error_file == expected_run_req.error_file expected_run_policy_args = { k: v diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index e33684d4a..251659c6f 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -23,8 +23,13 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import os +import pathlib + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.local import ( LocalLaunchArguments, @@ -148,8 +153,17 @@ def test_format_env_vars(): def test_formatting_returns_original_exe(mock_echo_executable, test_dir): - path, cmd = _as_local_command( - LocalLaunchArguments({}), mock_echo_executable, test_dir, {} + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_local_command( + LocalLaunchArguments({}), mock_echo_executable, test_dir, {}, out, err ) - assert tuple(cmd) == ("echo", "hello", "world") - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == ("echo", "hello", "world") + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 54046d06e..2e56e4a6c 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -23,6 +23,8 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import subprocess + import pytest from smartsim.settings import LaunchSettings @@ -91,37 +93,102 @@ def test_launch_args(): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("jsrun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {}, + ( + "jsrun", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), + id="Empty Args", + ), pytest.param( {"n": "1"}, - ("jsrun", "-n", "1", "--", "echo", "hello", "world"), + ( + "jsrun", + "-n", + "1", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg", ), pytest.param( {"nrs": "1"}, - ("jsrun", "--nrs=1", "--", "echo", "hello", "world"), + ( + "jsrun", + "--nrs=1", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg", ), pytest.param( {"v": None}, - ("jsrun", "-v", "--", "echo", "hello", "world"), + ( + "jsrun", + "-v", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("jsrun", "--verbose", "--", "echo", "hello", "world"), + ( + "jsrun", + "--verbose", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg (No Value)", ), pytest.param( {"tasks_per_rs": "1", "n": "123"}, - ("jsrun", "--tasks_per_rs=1", "-n", "123", "--", "echo", "hello", "world"), + ( + "jsrun", + "--tasks_per_rs=1", + "-n", + "123", + "--stdio_stdout=output.txt", + "--stdio_stderr=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_jsrun_command( - JsrunLaunchArguments(args), mock_echo_executable, test_dir, {} + outfile = "output.txt" + errfile = "error.txt" + env, path, stdin, stdout, args = _as_jsrun_command( + JsrunLaunchArguments(args), mock_echo_executable, test_dir, {}, outfile, errfile ) - assert tuple(cmd) == expected + assert tuple(args) == expected assert path == test_dir + assert env == {} + assert stdin == subprocess.DEVNULL + assert stdout == subprocess.DEVNULL diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index edd2f22e3..f2513a2f7 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -24,10 +24,14 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io import itertools +import os +import pathlib import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.mpi import ( MpiexecLaunchArguments, @@ -286,6 +290,15 @@ def test_invalid_hostlist_format(launcher): def test_formatting_launch_args( mock_echo_executable, cls, fmt, cmd, args, expected, test_dir ): - path, fmt_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}) - assert tuple(fmt_cmd) == (cmd,) + expected - assert path == test_dir + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = fmt(cls(args), mock_echo_executable, test_dir, {}, out, err) + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == (cmd,) + expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 8ea2f64f8..857b3799a 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -24,8 +24,13 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import os +import pathlib + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.pals import ( PalsMpiexecLaunchArguments, @@ -132,8 +137,22 @@ def test_invalid_hostlist_format(): ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_pals_command( - PalsMpiexecLaunchArguments(args), mock_echo_executable, test_dir, {} + out = os.path.join(test_dir, "out.txt") + err = os.path.join(test_dir, "err.txt") + open(out, "w"), open(err, "w") + shell_launch_cmd = _as_pals_command( + PalsMpiexecLaunchArguments(args), + mock_echo_executable, + test_dir, + {}, + out, + err, ) - assert tuple(cmd) == expected - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == pathlib.Path(test_dir) + assert shell_launch_cmd.env == {} + assert isinstance(shell_launch_cmd.stdout, io.TextIOWrapper) + assert shell_launch_cmd.stdout.name == out + assert isinstance(shell_launch_cmd.stderr, io.TextIOWrapper) + assert shell_launch_cmd.stderr.name == err diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 5f86bf7db..9ec4f2022 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -23,8 +23,11 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import subprocess + import pytest +from smartsim._core.shell.shellLauncher import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, @@ -290,37 +293,106 @@ def test_set_het_groups(monkeypatch): @pytest.mark.parametrize( "args, expected", ( - pytest.param({}, ("srun", "--", "echo", "hello", "world"), id="Empty Args"), + pytest.param( + {}, + ( + "srun", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), + id="Empty Args", + ), pytest.param( {"N": "1"}, - ("srun", "-N", "1", "--", "echo", "hello", "world"), + ( + "srun", + "-N", + "1", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg", ), pytest.param( {"nodes": "1"}, - ("srun", "--nodes=1", "--", "echo", "hello", "world"), + ( + "srun", + "--nodes=1", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg", ), pytest.param( {"v": None}, - ("srun", "-v", "--", "echo", "hello", "world"), + ( + "srun", + "-v", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short Arg (No Value)", ), pytest.param( {"verbose": None}, - ("srun", "--verbose", "--", "echo", "hello", "world"), + ( + "srun", + "--verbose", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Long Arg (No Value)", ), pytest.param( {"nodes": "1", "n": "123"}, - ("srun", "--nodes=1", "-n", "123", "--", "echo", "hello", "world"), + ( + "srun", + "--nodes=1", + "-n", + "123", + "--output=output.txt", + "--error=error.txt", + "--", + "echo", + "hello", + "world", + ), id="Short and Long Args", ), ), ) def test_formatting_launch_args(mock_echo_executable, args, expected, test_dir): - path, cmd = _as_srun_command( - SlurmLaunchArguments(args), mock_echo_executable, test_dir, {} + shell_launch_cmd = _as_srun_command( + args=SlurmLaunchArguments(args), + exe=mock_echo_executable, + path=test_dir, + env={}, + stdout_path="output.txt", + stderr_path="error.txt", ) - assert tuple(cmd) == expected - assert path == test_dir + assert isinstance(shell_launch_cmd, ShellLauncherCommand) + assert shell_launch_cmd.command_tuple == expected + assert shell_launch_cmd.path == test_dir + assert shell_launch_cmd.env == {} + assert shell_launch_cmd.stdout == subprocess.DEVNULL + assert shell_launch_cmd.stderr == subprocess.DEVNULL diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 2af864ab8..855068619 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -29,7 +29,6 @@ import dataclasses import itertools import random -import tempfile import typing as t import uuid @@ -55,7 +54,11 @@ def experiment(monkeypatch, test_dir, dispatcher): """ exp = Experiment(f"test-exp-{uuid.uuid4()}", test_dir) monkeypatch.setattr(dispatch, "DEFAULT_DISPATCHER", dispatcher) - monkeypatch.setattr(exp, "_generate", lambda gen, job, idx: "/tmp/job") + monkeypatch.setattr( + exp, + "_generate", + lambda gen, job, idx: ("/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt"), + ) yield exp @@ -65,8 +68,10 @@ def dispatcher(): dispatches any jobs with `MockLaunchArgs` to a `NoOpRecordLauncher` """ d = dispatch.Dispatcher() - to_record: dispatch._FormatterType[MockLaunchArgs, LaunchRecord] = ( - lambda settings, exe, path, env: LaunchRecord(settings, exe, env, path) + to_record: dispatch.FormatterType[MockLaunchArgs, LaunchRecord] = ( + lambda settings, exe, path, env, out, err: LaunchRecord( + settings, exe, env, path, out, err + ) ) d.dispatch(MockLaunchArgs, with_format=to_record, to_launcher=NoOpRecordLauncher) yield d @@ -143,6 +148,8 @@ class LaunchRecord: entity: entity.SmartSimEntity env: t.Mapping[str, str | None] path: str + out: str + err: str @classmethod def from_job(cls, job: job.Job): @@ -158,7 +165,9 @@ def from_job(cls, job: job.Job): entity = job._entity env = job._launch_settings.env_vars path = "/tmp/job" - return cls(args, entity, env, path) + out = "/tmp/job/out.txt" + err = "/tmp/job/err.txt" + return cls(args, entity, env, path, out, err) class MockLaunchArgs(launchArguments.LaunchArguments): diff --git a/tests/test_generator.py b/tests/test_generator.py index 4ecda339b..e44022779 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -10,9 +10,8 @@ import pytest from smartsim import Experiment -from smartsim._core import dispatch from smartsim._core.generation.generator import Generator -from smartsim.entity import Application, Ensemble, SmartSimEntity, _mock +from smartsim.entity import Application, Ensemble from smartsim.entity.files import EntityFiles from smartsim.launchable import Job from smartsim.settings import LaunchSettings @@ -21,9 +20,15 @@ pytestmark = pytest.mark.group_a +ids = set() + def random_id(): - return str(random.randint(1, 100)) + while True: + num = str(random.randint(1, 100)) + if num not in ids: + ids.add(num) + return num @pytest.fixture @@ -66,7 +71,7 @@ def test_generate_job_directory(test_dir, wlmutils, generator_instance): # Mock id run_id = "temp_id" # Call Generator.generate_job - job_run_path = generator_instance.generate_job(job, 0) + job_run_path, _, _ = generator_instance.generate_job(job, 0) assert isinstance(job_run_path, pathlib.Path) expected_run_path = ( pathlib.Path(test_dir) @@ -103,7 +108,7 @@ def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): job = Job(app, launch_settings) # Generate Job directory job_index = 1 - job_execution_path = exp._generate(generator_instance, job, job_index) + job_execution_path, _, _ = exp._generate(generator_instance, job, job_index) # Assert Job run directory exists assert osp.isdir(job_execution_path) # Assert Job log directory exists @@ -122,7 +127,7 @@ def test_generate_copy_file(generator_instance, fileutils, wlmutils): job = Job(app, launch_settings) # Create the experiment - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_file = pathlib.Path(path) / "sleep.py" assert osp.isfile(expected_file) @@ -135,7 +140,7 @@ def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance) job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_folder = path / "to_copy_dir" assert osp.isdir(expected_folder) @@ -150,7 +155,7 @@ def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlin job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_folder = path / "to_symlink_dir" assert osp.isdir(expected_folder) assert expected_folder.is_symlink() @@ -177,7 +182,7 @@ def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 1) + path, _, _ = generator_instance.generate_job(job, 1) expected_file = path / "mock2.txt" assert osp.isfile(expected_file) assert expected_file.is_symlink() @@ -215,7 +220,7 @@ def test_generate_configure(fileutils, wlmutils, generator_instance): job = Job(app, launch_settings) # Call Generator.generate_job - path = generator_instance.generate_job(job, 0) + path, _, _ = generator_instance.generate_job(job, 0) # Retrieve the list of configured files in the test directory configured_files = sorted(glob(str(path) + "/*")) # Use filecmp.cmp to check that the corresponding files are equal @@ -230,7 +235,7 @@ def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_inst job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) for i, job in enumerate(job_list): - job_run_path = exp._generate(generator_instance, job, i) + job_run_path, _, _ = exp._generate(generator_instance, job, i) head, _ = os.path.split(job_run_path) expected_log_path = pathlib.Path(head) / "log" assert osp.isdir(job_run_path) @@ -243,7 +248,7 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): job_list = ensemble.as_jobs(launch_settings) for i, job in enumerate(job_list): # Call Generator.generate_job - path = generator_instance.generate_job(job, i) + path, _, _ = generator_instance.generate_job(job, i) # Assert run directory created assert osp.isdir(path) # Assert smartsim params file created @@ -260,7 +265,7 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble("ensemble-name", "echo", replicas=2) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) @@ -275,12 +280,13 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): log_path = os.path.join(jobs_dir, ensemble_dir, "log") assert osp.isdir(run_path) assert osp.isdir(log_path) + ids.clear() def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) @@ -295,6 +301,7 @@ def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_di for ensemble_dir in job_dir: copy_folder_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_copy_dir") assert osp.isdir(copy_folder_path) + ids.clear() def test_generate_ensemble_symlink( @@ -302,7 +309,7 @@ def test_generate_ensemble_symlink( ): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( "ensemble-name", @@ -322,6 +329,7 @@ def test_generate_ensemble_symlink( assert osp.isdir(sym_file_path) assert sym_file_path.is_symlink() assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) + ids.clear() def test_generate_ensemble_configure( @@ -329,7 +337,7 @@ def test_generate_ensemble_configure( ): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env: random_id(), + lambda launch, exe, job_execution_path, env, out, err: random_id(), ) params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} # Retrieve a list of files for configuration @@ -344,7 +352,8 @@ def test_generate_ensemble_configure( launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.as_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) + id = exp.start(*job_list) + print(id) run_dir = listdir(test_dir) jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") @@ -365,3 +374,4 @@ def _check_generated(param_0, param_1, dir): _check_generated(1, 2, os.path.join(jobs_dir, "ensemble-name-2-2", "run")) _check_generated(1, 3, os.path.join(jobs_dir, "ensemble-name-3-3", "run")) _check_generated(0, 2, os.path.join(jobs_dir, "ensemble-name-0-0", "run")) + ids.clear() diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py new file mode 100644 index 000000000..6b03f8501 --- /dev/null +++ b/tests/test_shell_launcher.py @@ -0,0 +1,312 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import pathlib +import subprocess +import unittest.mock + +import psutil +import pytest + +from smartsim._core.shell.shellLauncher import ShellLauncher, ShellLauncherCommand, sp +from smartsim._core.utils import helpers +from smartsim._core.utils.shell import * +from smartsim.entity import _mock, entity +from smartsim.error.errors import LauncherJobNotFound +from smartsim.status import JobStatus + +pytestmark = pytest.mark.group_a + + +class EchoHelloWorldEntity(entity.SmartSimEntity): + """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" + + def __init__(self): + super().__init__("test-entity", _mock.Mock()) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + return self.as_program_arguments() == other.as_program_arguments() + + def as_program_arguments(self): + return (helpers.expand_exe_path("echo"), "Hello", "World!") + + +def create_directory(directory_path: str) -> pathlib.Path: + """Creates the execution directory for testing.""" + tmp_dir = pathlib.Path(directory_path) + tmp_dir.mkdir(exist_ok=True, parents=True) + return tmp_dir + + +def generate_output_files(tmp_dir: pathlib.Path): + """Generates output and error files within the run directory for testing.""" + out_file = tmp_dir / "tmp.out" + err_file = tmp_dir / "tmp.err" + return out_file, err_file + + +def generate_directory(test_dir: str): + """Generates a execution directory, output file, and error file for testing.""" + execution_dir = create_directory(os.path.join(test_dir, "tmp")) + out_file, err_file = generate_output_files(execution_dir) + return execution_dir, out_file, err_file + + +@pytest.fixture +def shell_launcher(): + launcher = ShellLauncher() + yield launcher + if any(proc.poll() is None for proc in launcher._launched.values()): + raise ("Test leaked processes") + + +@pytest.fixture +def shell_cmd(test_dir: str) -> ShellLauncherCommand: + """Fixture to create an instance of Generator.""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + yield ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) + + +# UNIT TESTS + + +def test_shell_launcher_command_init(shell_cmd: ShellLauncherCommand, test_dir: str): + """Test that ShellLauncherCommand initializes correctly""" + assert shell_cmd.env == {} + assert shell_cmd.path == pathlib.Path(test_dir) / "tmp" + assert shell_cmd.stdout.name == os.path.join(test_dir, "tmp", "tmp.out") + assert shell_cmd.stderr.name == os.path.join(test_dir, "tmp", "tmp.err") + assert shell_cmd.command_tuple == EchoHelloWorldEntity().as_program_arguments() + + +def test_shell_launcher_init(shell_launcher: ShellLauncher): + """Test that ShellLauncher initializes correctly""" + assert shell_launcher._launched == {} + + +def test_check_popen_inputs(shell_launcher: ShellLauncher, test_dir: str): + """Test that ShellLauncher.check_popen_inputs throws correctly""" + cmd = ShellLauncherCommand( + {}, + pathlib.Path(test_dir) / "directory_dne", + subprocess.DEVNULL, + subprocess.DEVNULL, + EchoHelloWorldEntity().as_program_arguments(), + ) + with pytest.raises(ValueError): + _ = shell_launcher.start(cmd) + + +def test_shell_launcher_start_calls_popen( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand +): + """Test that the process leading up to the shell launcher popen call was correct""" + with unittest.mock.patch( + "smartsim._core.shell.shellLauncher.sp.Popen" + ) as mock_open: + _ = shell_launcher.start(shell_cmd) + mock_open.assert_called_once() + + +def test_shell_launcher_start_calls_popen_with_value( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand +): + """Test that popen was called with correct values""" + with unittest.mock.patch( + "smartsim._core.shell.shellLauncher.sp.Popen" + ) as mock_open: + _ = shell_launcher.start(shell_cmd) + mock_open.assert_called_once_with( + shell_cmd.command_tuple, + cwd=shell_cmd.path, + env=shell_cmd.env, + stdout=shell_cmd.stdout, + stderr=shell_cmd.stderr, + ) + + +def test_popen_returns_popen_object( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Test that the popen call returns a popen object""" + id = shell_launcher.start(shell_cmd) + with shell_launcher._launched[id] as proc: + assert isinstance(proc, sp.Popen) + + +def test_popen_writes_to_output_file( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Test that popen writes to .out file upon successful process call""" + _, out_file, err_file = generate_directory(test_dir) + id = shell_launcher.start(shell_cmd) + proc = shell_launcher._launched[id] + assert proc.wait() == 0 + assert proc.returncode == 0 + with open(out_file, "r", encoding="utf-8") as out: + assert out.read() == "Hello World!\n" + with open(err_file, "r", encoding="utf-8") as err: + assert err.read() == "" + + +def test_popen_fails_with_invalid_cmd(shell_launcher: ShellLauncher, test_dir: str): + """Test that popen returns a non zero returncode after failure""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + args = (helpers.expand_exe_path("ls"), "--flag_dne") + cmd = ShellLauncherCommand({}, run_dir, out, err, args) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + proc.wait() + assert proc.returncode != 0 + with open(out_file, "r", encoding="utf-8") as out: + assert out.read() == "" + with open(err_file, "r", encoding="utf-8") as err: + content = err.read() + assert "unrecognized option" in content + + +def test_popen_issues_unique_ids( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Validate that all ids are unique within ShellLauncher._launched""" + seen = set() + for _ in range(5): + id = shell_launcher.start(shell_cmd) + assert id not in seen, "Duplicate ID issued" + seen.add(id) + assert len(shell_launcher._launched) == 5 + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) + + +def test_retrieve_status_dne(shell_launcher: ShellLauncher): + """Test tht ShellLauncher returns the status of completed Jobs""" + with pytest.raises(LauncherJobNotFound): + _ = shell_launcher.get_status("dne") + + +def test_shell_launcher_returns_complete_status( + shell_launcher: ShellLauncher, shell_cmd: ShellLauncherCommand, test_dir: str +): + """Test tht ShellLauncher returns the status of completed Jobs""" + for _ in range(5): + id = shell_launcher.start(shell_cmd) + proc = shell_launcher._launched[id] + proc.wait() + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.COMPLETED + + +def test_shell_launcher_returns_failed_status( + shell_launcher: ShellLauncher, test_dir: str +): + """Test tht ShellLauncher returns the status of completed Jobs""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + args = (helpers.expand_exe_path("ls"), "--flag_dne") + cmd = ShellLauncherCommand({}, run_dir, out, err, args) + for _ in range(5): + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + proc.wait() + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.FAILED + + +def test_shell_launcher_returns_running_status( + shell_launcher: ShellLauncher, test_dir: str +): + """Test tht ShellLauncher returns the status of completed Jobs""" + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5") + ) + for _ in range(5): + id = shell_launcher.start(cmd) + code = shell_launcher.get_status(id)[id] + assert code == JobStatus.RUNNING + assert all(proc.wait() == 0 for proc in shell_launcher._launched.values()) + + +@pytest.mark.parametrize( + "psutil_status,job_status", + [ + pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="running"), + pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="sleeping"), + pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="waking"), + pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="disk_sleep"), + pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="dead"), + pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="tracing_stop"), + pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="waiting"), + pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="stopped"), + pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="locked"), + pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="parked"), + pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="idle"), + pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="zombie"), + pytest.param( + "some-brand-new-unknown-status-str", JobStatus.UNKNOWN, id="unknown" + ), + ], +) +def test_get_status_maps_correctly( + psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir: str +): + """Test tht ShellLauncher.get_status returns correct mapping""" + shell_launcher = ShellLauncher() + run_dir, out_file, err_file = generate_directory(test_dir) + with ( + open(out_file, "w", encoding="utf-8") as out, + open(err_file, "w", encoding="utf-8") as err, + ): + cmd = ShellLauncherCommand( + {}, run_dir, out, err, EchoHelloWorldEntity().as_program_arguments() + ) + id = shell_launcher.start(cmd) + proc = shell_launcher._launched[id] + monkeypatch.setattr(proc, "poll", lambda: None) + monkeypatch.setattr(psutil.Process, "status", lambda self: psutil_status) + value = shell_launcher.get_status(id) + assert value.get(id) == job_status + assert proc.wait() == 0