From 0c552234163d4417196716bbd853c8725df61083 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 20 Aug 2024 11:58:25 -0500 Subject: [PATCH] usused import deletions --- conftest.py | 34 ++++++------ smartsim/experiment.py | 2 - smartsim/launchable/baseJobGroup.py | 26 +++++++++ smartsim/launchable/colocatedJobGroup.py | 26 +++++++++ smartsim/launchable/job.py | 1 - smartsim/launchable/jobGroup.py | 26 +++++++++ smartsim/settings/arguments/launch/local.py | 1 - smartsim/settings/arguments/launch/slurm.py | 3 - smartsim/settings/dispatch.py | 18 +----- .../test_settings/test_slurmLauncher.py | 2 - tests/test_experiment.py | 1 - tests/test_generator.py | 2 +- tests/test_shell_launcher.py | 55 +++++-------------- 13 files changed, 112 insertions(+), 85 deletions(-) diff --git a/conftest.py b/conftest.py index 63a8c8081..3facd09a9 100644 --- a/conftest.py +++ b/conftest.py @@ -154,23 +154,23 @@ def pytest_sessionfinish( Called after whole test run finished, right before returning the exit status to the system. """ - # if exitstatus == 0: - # cleanup_attempts = 5 - # while cleanup_attempts > 0: - # try: - # shutil.rmtree(test_output_root) - # except OSError as e: - # cleanup_attempts -= 1 - # time.sleep(1) - # if not cleanup_attempts: - # raise - # else: - # break - # else: - # # kill all spawned processes - # if CONFIG.test_launcher == "dragon": - # time.sleep(5) - # kill_all_test_spawned_processes() + if exitstatus == 0: + cleanup_attempts = 5 + while cleanup_attempts > 0: + try: + shutil.rmtree(test_output_root) + except OSError as e: + cleanup_attempts -= 1 + time.sleep(1) + if not cleanup_attempts: + raise + else: + break + else: + # kill all spawned processes + if CONFIG.test_launcher == "dragon": + time.sleep(5) + kill_all_test_spawned_processes() def build_mpi_app() -> t.Optional[pathlib.Path]: diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 7137bed36..50b06cabd 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -234,8 +234,6 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: ) # Generate the job directory and return the generated job path ret = self._generate(generator, job, idx) - print(f"the type: {type(ret)}") - print(f"the val: {ret}") job_execution_path, out, err = ret id_ = launch_config.start(exe, job_execution_path, env, out, err) # Save the underlying launcher instance and launched job id. That diff --git a/smartsim/launchable/baseJobGroup.py b/smartsim/launchable/baseJobGroup.py index d662550f5..b7becba56 100644 --- a/smartsim/launchable/baseJobGroup.py +++ b/smartsim/launchable/baseJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/colocatedJobGroup.py b/smartsim/launchable/colocatedJobGroup.py index 97e7aa4a3..1c3b96fba 100644 --- a/smartsim/launchable/colocatedJobGroup.py +++ b/smartsim/launchable/colocatedJobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index e680e5f14..4c29c3c9f 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -26,7 +26,6 @@ from __future__ import annotations -import os import typing as t from copy import deepcopy diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index fd288deb4..d2e64c454 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index a61068cd4..0bbba2584 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -27,7 +27,6 @@ from __future__ import annotations import typing as t -from subprocess import PIPE from smartsim.log import get_logger from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index 141e9adcd..f8e514ab2 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -32,16 +32,13 @@ import subprocess import typing as t -from smartsim._core.commands import Command from smartsim.log import get_logger from smartsim.settings.dispatch import ( ExecutableProtocol, ShellLauncher, ShellLauncherCommand, _EnvironMappingType, - _FormatterType, dispatch, - make_shell_format_fn, ) from ...common import set_check_input diff --git a/smartsim/settings/dispatch.py b/smartsim/settings/dispatch.py index 77ac6c74a..3c66afda6 100644 --- a/smartsim/settings/dispatch.py +++ b/smartsim/settings/dispatch.py @@ -30,25 +30,19 @@ import collections.abc import dataclasses import io -import os import pathlib -import subprocess import subprocess as sp import typing as t import uuid -from subprocess import STDOUT import psutil from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack -from smartsim._core.commands import Command, CommandList from smartsim._core.utils import helpers from smartsim.error import errors from smartsim.status import JobStatus from smartsim.types import LaunchedJobID -from ..settings.launchCommand import LauncherType - if t.TYPE_CHECKING: from smartsim.experiment import Experiment from smartsim.settings.arguments import LaunchArguments @@ -522,10 +516,7 @@ class ShellLauncher: def __init__(self) -> None: self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - # covariant, contravariant, + boliscoff substitution princ - def start( - self, shell_command: ShellLauncherCommand # this should be a named tuple - ) -> LaunchedJobID: + def start(self, shell_command: ShellLauncherCommand) -> LaunchedJobID: id_ = create_job_id() # raise ValueError -> invalid stuff throw exe, *rest = shell_command.command_tuple @@ -537,7 +528,6 @@ def start( stdout=shell_command.stdout, stderr=shell_command.stderr, ) - # Popen starts a new process and gives you back a handle to process, getting back the pid - process id return id_ def get_status( @@ -549,14 +539,10 @@ def _get_status(self, id_: LaunchedJobID, /) -> JobStatus: if (proc := self._launched.get(id_)) is None: msg = f"Launcher `{self}` has not launched a job with id `{id_}`" raise errors.LauncherJobNotFound(msg) - ret_code = ( - proc.poll() - ) # add a test that mocks out poll and raise some exception - terminal -> import subprocess -> start something echo blah - then poll and see what a valid fake output is - print(ret_code) + ret_code = proc.poll() # try/catch around here and then reaise a smartsim.error if ret_code is None: status = psutil.Process(proc.pid).status() - print(status) return { psutil.STATUS_RUNNING: JobStatus.RUNNING, psutil.STATUS_SLEEPING: JobStatus.RUNNING, diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 892978dfb..4160e60f1 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -23,12 +23,10 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pathlib import subprocess import pytest -from smartsim._core.commands import Command from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 6e4fd8a7f..3f1195e9c 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -29,7 +29,6 @@ import dataclasses import itertools import random -import tempfile import typing as t import uuid diff --git a/tests/test_generator.py b/tests/test_generator.py index 602f300ea..3b2c95cf2 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -14,7 +14,7 @@ from smartsim.entity import Application, Ensemble from smartsim.entity.files import EntityFiles from smartsim.launchable import Job -from smartsim.settings import LaunchSettings, dispatch +from smartsim.settings import LaunchSettings # TODO Add JobGroup tests when JobGroup becomes a Launchable diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index 3eb6a7766..d9143c52a 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -24,34 +24,20 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import difflib import os import pathlib import subprocess -import tempfile import unittest.mock -import uuid -import weakref import psutil import pytest -from smartsim import Experiment -from smartsim._core.commands import Command from smartsim._core.utils import helpers from smartsim._core.utils.shell import * -from smartsim.entity import Application, _mock, entity +from smartsim.entity import _mock, entity from smartsim.error.errors import LauncherJobNotFound -from smartsim.launchable import Job -from smartsim.settings import LaunchSettings -from smartsim.settings.arguments.launch.slurm import ( - SlurmLaunchArguments, - _as_srun_command, -) from smartsim.settings.dispatch import ShellLauncher, ShellLauncherCommand, sp -from smartsim.settings.launchCommand import LauncherType from smartsim.status import JobStatus -from smartsim.types import LaunchedJobID # TODO tests bad vars in Popen call at beginning # tests -> helper.exe : pass in None, empty str, path with a space at beginning, a non valid command @@ -258,73 +244,60 @@ def test_shell_launcher_returns_complete_status(test_dir): def test_shell_launcher_returns_failed_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - # Init ShellLauncher shell_launcher = ShellLauncher() - # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err, ): - # Construct a invalid command to execute args = (helpers.expand_exe_path("srun"), "--flag_dne") cmd = ShellLauncherCommand({}, run_dir, out, err, args) - # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) - # Retrieve popen object proc = shell_launcher._launched[id] - # Wait for subprocess to complete proc.wait() - # Retrieve status of subprocess code = shell_launcher.get_status(id) val = list(code.keys())[0] - # Assert that subprocess has completed assert code[val] == JobStatus.FAILED def test_shell_launcher_returns_running_status(test_dir): """Test tht ShellLauncher returns the status of completed Jobs""" - # Init ShellLauncher shell_launcher = ShellLauncher() - # Generate testing directory run_dir, out_file, err_file = generate_directory(test_dir) with ( open(out_file, "w", encoding="utf-8") as out, open(err_file, "w", encoding="utf-8") as err, ): - # Construct a command to execute cmd = ShellLauncherCommand( {}, run_dir, out, err, (helpers.expand_exe_path("sleep"), "5") ) - # Start the execution of the command using a ShellLauncher for _ in range(5): id = shell_launcher.start(cmd) - # Retrieve status of subprocess code = shell_launcher.get_status(id) val = list(code.keys())[0] - # Assert that subprocess has completed assert code[val] == JobStatus.RUNNING @pytest.mark.parametrize( "psutil_status,job_status", [ - pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="merp"), - pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="merp"), - pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="merp"), - pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="merp"), + pytest.param(psutil.STATUS_RUNNING, JobStatus.RUNNING, id="running"), + pytest.param(psutil.STATUS_SLEEPING, JobStatus.RUNNING, id="sleeping"), + pytest.param(psutil.STATUS_WAKING, JobStatus.RUNNING, id="waking"), + pytest.param(psutil.STATUS_DISK_SLEEP, JobStatus.RUNNING, id="disk_sleep"), + pytest.param(psutil.STATUS_DEAD, JobStatus.FAILED, id="dead"), + pytest.param(psutil.STATUS_TRACING_STOP, JobStatus.PAUSED, id="tracing_stop"), + pytest.param(psutil.STATUS_WAITING, JobStatus.PAUSED, id="waiting"), + pytest.param(psutil.STATUS_STOPPED, JobStatus.PAUSED, id="stopped"), + pytest.param(psutil.STATUS_LOCKED, JobStatus.PAUSED, id="locked"), + pytest.param(psutil.STATUS_PARKED, JobStatus.PAUSED, id="parked"), + pytest.param(psutil.STATUS_IDLE, JobStatus.PAUSED, id="idle"), + pytest.param(psutil.STATUS_ZOMBIE, JobStatus.COMPLETED, id="zombie"), ], ) def test_this(psutil_status, job_status, monkeypatch: pytest.MonkeyPatch, test_dir): + """Test tht ShellLauncher.get_status returns correct mapping""" shell_launcher = ShellLauncher() run_dir, out_file, err_file = generate_directory(test_dir) with (