diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 65aadc03c8..9a7375557c 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -32,6 +32,8 @@ on: push: branches: - develop + branches-ignore: + - smartsim-refactor jobs: build_docs: diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index cd4ab58fa8..3b62a750f1 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -30,6 +30,8 @@ name: enforce_changelog on: pull_request: + branches-ignore: + - smartsim-refactor push: branches: - develop diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 5076870d7d..ca6ec4cc40 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -140,8 +140,8 @@ jobs: make check-mypy # TODO: Re-enable static analysis once API is firmed up - # - name: Run Pylint - # run: make check-lint + - name: Run Pylint + run: make check-lint # Run isort/black style check - name: Run isort @@ -172,7 +172,7 @@ jobs: run: | echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests - + # Upload artifacts on failure, ignoring binary files - name: Upload Artifact if: failure() diff --git a/doc/changelog.md b/doc/changelog.md index 752957bfdc..e666de03a2 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -49,6 +49,7 @@ Description - Implement workaround for Tensorflow that allows RedisAI to build with GCC-14 - Add instructions for installing SmartSim on PML's Scylla +- Fix typos in documentation Detailed Notes @@ -64,6 +65,7 @@ Detailed Notes have yet to be installed at a system-wide level. Scylla has its own entry in the documentation. ([SmartSim-PR733](https://github.com/CrayLabs/SmartSim/pull/733)) +- Fix typos in the `train_surrogate` tutorial documentation ### 0.8.0 diff --git a/doc/tutorials/ml_training/surrogate/README.md b/doc/tutorials/ml_training/surrogate/README.md index 823d27500b..ea2e8a68ff 100644 --- a/doc/tutorials/ml_training/surrogate/README.md +++ b/doc/tutorials/ml_training/surrogate/README.md @@ -3,6 +3,6 @@ In this example, a neural network is trained to act like a surrogate model and to solve a well-known physical problem, i.e. computing the steady state of heat diffusion. The training -dataset is constructed by running simualations *while* the model is being trained. +dataset is constructed by running simulations *while* the model is being trained. -The notebook also displays how the surrogate model prediction improves during training. \ No newline at end of file +The notebook also displays how the surrogate model prediction improves during training. diff --git a/doc/tutorials/ml_training/surrogate/train_surrogate.ipynb b/doc/tutorials/ml_training/surrogate/train_surrogate.ipynb index 5625b86b99..76a7b8c05f 100644 --- a/doc/tutorials/ml_training/surrogate/train_surrogate.ipynb +++ b/doc/tutorials/ml_training/surrogate/train_surrogate.ipynb @@ -15,7 +15,7 @@ "\n", "In this notebook, a neural network is trained to act like a surrogate model and to solve a\n", "well-known physical problem, i.e. computing the steady state of heat diffusion. The training\n", - "dataset is constructed by running simualations *while* the model is being trained.\n", + "dataset is constructed by running simulations *while* the model is being trained.\n", "\n", "## 2D Heat Diffusion and Steady State\n", "Throughout this notebook, the heat equation will be solved on the two-dimensional domain $[0,1]\\times[0,1]$, setting the initial temperature to 0 K, except\n", diff --git a/smartsim/_core/_cli/build.py b/smartsim/_core/_cli/build.py index 58ef31ab8a..a0dc489f6a 100644 --- a/smartsim/_core/_cli/build.py +++ b/smartsim/_core/_cli/build.py @@ -28,12 +28,10 @@ import importlib.metadata import operator import os -import platform +from pathlib import Path import re -import shutil import textwrap import typing as t -from pathlib import Path from tabulate import tabulate @@ -44,10 +42,8 @@ display_post_install_logs, install_dragon, ) -from smartsim._core._cli.utils import SMART_LOGGER_FORMAT, pip -from smartsim._core._install import builder -from smartsim._core._install.buildenv import BuildEnv, SetupError, Version_, Versioner -from smartsim._core._install.builder import BuildError +from smartsim._core._cli.utils import SMART_LOGGER_FORMAT +from smartsim._core._install.buildenv import BuildEnv, Version_, Versioner from smartsim._core._install.mlpackages import ( DEFAULT_MLPACKAGE_PATH, DEFAULT_MLPACKAGES, @@ -60,13 +56,17 @@ OperatingSystem, Platform, ) -from smartsim._core._install.redisaiBuilder import RedisAIBuilder + from smartsim._core.config import CONFIG -from smartsim.error import SSConfigError from smartsim.log import get_logger logger = get_logger("Smart", fmt=SMART_LOGGER_FORMAT) +# *************************************** +# TODO: Remove pylint disable after merge +# *************************************** +# pylint: disable=too-many-statements,unused-variable,no-value-for-parameter,no-member,invalid-name,condition-evals-to-constant,unused-variable + # NOTE: all smartsim modules need full paths as the smart cli # may be installed into a different directory. @@ -143,7 +143,8 @@ def _format_incompatible_python_env_message( conflict_str = fmt_list("Conflicting", conflicting) sep = "\n" if missing_str and conflict_str else "" - return textwrap.dedent(f"""\ + return textwrap.dedent( + f"""\ Python Package Warning: Requested packages are missing or have a version mismatch with @@ -153,7 +154,8 @@ def _format_incompatible_python_env_message( Consider uninstalling any conflicting packages and rerunning `smart build` if you encounter issues. - """) + """ + ) # pylint: disable-next=too-many-statements diff --git a/smartsim/_core/_cli/cli.py b/smartsim/_core/_cli/cli.py index 71d0c3a398..397eb991b4 100644 --- a/smartsim/_core/_cli/cli.py +++ b/smartsim/_core/_cli/cli.py @@ -35,7 +35,6 @@ from smartsim._core._cli.clean import configure_parser as clean_parser from smartsim._core._cli.clean import execute as clean_execute from smartsim._core._cli.clean import execute_all as clobber_execute -from smartsim._core._cli.dbcli import execute as dbcli_execute from smartsim._core._cli.info import execute as info_execute from smartsim._core._cli.plugin import plugins from smartsim._core._cli.site import execute as site_execute @@ -118,11 +117,6 @@ def default_cli() -> SmartCli: clean_execute, clean_parser, ), - MenuItemConfig( - "dbcli", - "Print the path to the redis-cli binary", - dbcli_execute, - ), MenuItemConfig( "site", "Print the installation site of SmartSim", diff --git a/smartsim/_core/_cli/validate.py b/smartsim/_core/_cli/validate.py index 0e21e01ac6..a87642e49f 100644 --- a/smartsim/_core/_cli/validate.py +++ b/smartsim/_core/_cli/validate.py @@ -26,7 +26,6 @@ import argparse import contextlib -import io import os import os.path import tempfile diff --git a/smartsim/_core/_install/buildenv.py b/smartsim/_core/_install/buildenv.py index b8c6775120..bd96ffd02c 100644 --- a/smartsim/_core/_install/buildenv.py +++ b/smartsim/_core/_install/buildenv.py @@ -24,7 +24,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# pylint: disable=invalid-name import importlib.metadata import os @@ -153,7 +152,8 @@ class Versioner: ``smart build`` command to determine which dependency versions to look for and download. - Default versions for SmartSim, Redis, and RedisAI are specified here. + Default versions for SmartSim and its machine learning library dependencies + all defined here. """ # compatible Python version diff --git a/smartsim/_core/_install/builder.py b/smartsim/_core/_install/builder.py index 7b850a9158..a1a4cb93b5 100644 --- a/smartsim/_core/_install/builder.py +++ b/smartsim/_core/_install/builder.py @@ -35,12 +35,6 @@ from pathlib import Path from subprocess import SubprocessError -from smartsim._core._install.platform import Architecture, OperatingSystem, Platform -from smartsim._core._install.utils import retrieve -from smartsim._core.utils import expand_exe_path - -if t.TYPE_CHECKING: - from typing_extensions import Never # TODO: check cmake version and use system if possible to avoid conflicts diff --git a/smartsim/_core/_install/platform.py b/smartsim/_core/_install/platform.py index bef13c6a0a..f49195498b 100644 --- a/smartsim/_core/_install/platform.py +++ b/smartsim/_core/_install/platform.py @@ -82,6 +82,8 @@ class Device(enum.Enum): CUDA12 = "cuda-12" ROCM5 = "rocm-5" ROCM6 = "rocm-6" + # TODO: check to see if this got overwritten/ gpu was removed + GPU = "gpu" @classmethod def from_str(cls, str_: str) -> "Device": diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 0968759afd..f53af326e8 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -89,9 +89,9 @@ def __len__(self) -> int: """Get the length of the command list.""" return len(self._command) - def insert(self, idx: int, value: str) -> None: + def insert(self, index: int, value: str) -> None: """Insert a command at the specified index.""" - self._command.insert(idx, value) + self._command.insert(index, value) def __str__(self) -> str: # pragma: no cover string = f"\nCommand: {' '.join(str(cmd) for cmd in self.command)}" diff --git a/smartsim/_core/commands/command_list.py b/smartsim/_core/commands/command_list.py index fcffe42a2a..d3d6eace4d 100644 --- a/smartsim/_core/commands/command_list.py +++ b/smartsim/_core/commands/command_list.py @@ -83,7 +83,8 @@ def __setitem__( isinstance(item, str) for item in sublist.command ): raise TypeError( - "Value sublists must be a list of Commands when assigning to a slice" + "Value sublists must be a list of Commands when assigning \ +to a slice" ) self._commands[idx] = (deepcopy(val) for val in value) @@ -95,9 +96,9 @@ def __len__(self) -> int: """Get the length of the Command list.""" return len(self._commands) - def insert(self, idx: int, value: Command) -> None: + def insert(self, index: int, value: Command) -> None: """Insert a Command at the specified index.""" - self._commands.insert(idx, value) + self._commands.insert(index, value) def __str__(self) -> str: # pragma: no cover string = "\n\nCommand List:\n\n" diff --git a/smartsim/_core/commands/launch_commands.py b/smartsim/_core/commands/launch_commands.py index 74303ac942..d31551671a 100644 --- a/smartsim/_core/commands/launch_commands.py +++ b/smartsim/_core/commands/launch_commands.py @@ -40,11 +40,9 @@ def postlaunch_command(self) -> CommandList: def __str__(self) -> str: # pragma: no cover string = "\n\nPrelaunch Command List:\n" - for pre_cmd in self.prelaunch_command: - string += f"{pre_cmd}\n" + string += f"\n{' '.join(str(pre_cmd) for pre_cmd in self.prelaunch_command)}" string += "\n\nLaunch Command List:\n" - for launch_cmd in self.launch_command: - string += f"{launch_cmd}\n" + string += f"\n{' '.join(str(launch_cmd) for launch_cmd in self.launch_command)}" string += "\n\nPostlaunch Command List:\n" for post_cmd in self.postlaunch_command: string += f"{post_cmd}\n" diff --git a/smartsim/_core/config/config.py b/smartsim/_core/config/config.py index b4cbae8d22..478ab02da3 100644 --- a/smartsim/_core/config/config.py +++ b/smartsim/_core/config/config.py @@ -32,8 +32,6 @@ import psutil -from ...error import SSConfigError -from ..utils import expand_exe_path # Configuration Values # diff --git a/smartsim/_core/control/job.py b/smartsim/_core/control/job.py index 91609349ad..bb8ab31ea5 100644 --- a/smartsim/_core/control/job.py +++ b/smartsim/_core/control/job.py @@ -76,7 +76,8 @@ def __init__(self) -> None: @property def is_fs(self) -> bool: - """Returns `True` if the entity represents a feature store or feature store shard""" + """Returns `True` if the entity represents a feature store or feature + store shard""" return self.type in ["featurestore", "fsnode"] @property diff --git a/smartsim/_core/dispatch.py b/smartsim/_core/dispatch.py index be096366df..0405f929f8 100644 --- a/smartsim/_core/dispatch.py +++ b/smartsim/_core/dispatch.py @@ -27,7 +27,6 @@ from __future__ import annotations import dataclasses -import os import pathlib import typing as t diff --git a/smartsim/_core/entrypoints/dragon_client.py b/smartsim/_core/entrypoints/dragon_client.py index 0131124121..1dd62abebd 100644 --- a/smartsim/_core/entrypoints/dragon_client.py +++ b/smartsim/_core/entrypoints/dragon_client.py @@ -148,7 +148,8 @@ def execute_entrypoint(args: DragonClientEntrypointArgs) -> int: requests.append(DragonShutdownRequest(immediate=False, frontend_shutdown=True)) - connector = DragonConnector() + # TODO: needs path + connector = DragonConnector(".") for request in requests: response = connector.send_request(request) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 1cc1670655..ced3a0d1e2 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -52,18 +52,19 @@ class _GenerableProtocol(t.Protocol): and parameters.""" files: FileSysOperationSet - # TODO change when file_parameters taken off Application during Ensemble refactor ticket + # TODO change when file_parameters taken off Application + # during Ensemble refactor ticket file_parameters: t.Mapping[str, str] -Job_Path = namedtuple("Job_Path", ["run_path", "out_path", "err_path"]) +JobPath = namedtuple("JobPath", ["run_path", "out_path", "err_path"]) """Namedtuple that stores a Job's run directory, output file path, and error file path.""" class Generator: - """The Generator class creates the directory structure for a SmartSim Job by building - and executing file operation commands. + """The Generator class creates the directory structure for a + SmartSim Job by building and executing file operation commands. """ run_directory = "run" @@ -74,8 +75,8 @@ class Generator: def __init__(self, root: pathlib.Path) -> None: """Initialize a Generator object - The Generator class is responsible for constructing a Job's directory, performing - the following tasks: + The Generator class is responsible for constructing a + Job's directory, performing the following tasks: - Creating the run and log directories - Generating the output and error files @@ -85,7 +86,8 @@ def __init__(self, root: pathlib.Path) -> None: :param root: The base path for job-related files and directories """ self.root = root - """The root directory under which all generated files and directories will be placed.""" + """The root directory under which all generated files and + directories will be placed.""" def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's base directory. The path is created by combining the @@ -102,9 +104,9 @@ def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: return pathlib.Path(job_path) def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: - """Build and return a Job's run directory. The path is formed by combining - the base directory with the `run_directory` class-level constant, which specifies - the name of the Job's run folder. + """Build and return a Job's run directory. The path is formed by + combining the base directory with the `run_directory` class-level + constant, which specifies the name of the Job's run folder. :param job: Job object :param job_index: Job index @@ -115,8 +117,8 @@ def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's log directory. The path is formed by combining - the base directory with the `log_directory` class-level constant, which specifies - the name of the Job's log folder. + the base directory with the `log_directory` class-level constant, + which specifies the name of the Job's log folder. :param job: Job object :param job_index: Job index @@ -137,8 +139,9 @@ def _build_log_file_path(log_path: pathlib.Path) -> pathlib.Path: @staticmethod def _build_out_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path: - """Build and return the path to the output file. The path is created by combining - the Job's log directory with the job name and appending the `.out` extension. + """Build and return the path to the output file. + The path is created by combining the Job's log directory with the + job name and appending the `.out` extension. :param log_path: Path to log directory :param job_name: Name of the Job @@ -159,7 +162,7 @@ def _build_err_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path: err_file_path = log_path / f"{job_name}.err" return err_file_path - def generate_job(self, job: Job, job_index: int) -> Job_Path: + def generate_job(self, job: Job, job_index: int) -> JobPath: """Build and return the Job's run directory, output file, and error file. This method creates the Job's run and log directories, generates the @@ -189,21 +192,21 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - return Job_Path(job_path, out_file, err_file) + return JobPath(job_path, out_file, err_file) @classmethod def _build_commands( cls, - entity: entity.SmartSimEntity, + smartsim_entity: entity.SmartSimEntity, job_path: pathlib.Path, log_path: pathlib.Path, ) -> CommandList: """Build file operation commands for a Job's entity. - This method constructs commands for copying, symlinking, and writing tagged files - associated with the Job's entity. This method builds the constructs the commands to - generate the Job's run and log directory. It aggregates these commands into a CommandList - to return. + This method constructs commands for copying, symlinking, and writing + tagged files associated with the Job's entity. This method builds + the constructs the commands to generate the Job's run and log directory. + It aggregates these commands into a CommandList to return. :param job: Job object :param job_path: The file path for the Job run folder @@ -215,8 +218,8 @@ def _build_commands( cls._append_mkdir_commands(cmd_list, job_path, log_path) - if isinstance(entity, _GenerableProtocol): - cls._append_file_operations(cmd_list, entity, context) + if isinstance(smartsim_entity, _GenerableProtocol): + cls._append_file_operations(cmd_list, smartsim_entity, context) return cmd_list @@ -237,23 +240,27 @@ def _append_mkdir_commands( def _append_file_operations( cls, cmd_list: CommandList, - entity: _GenerableProtocol, + smartsim_entity: _GenerableProtocol, context: GenerationContext, ) -> None: """Append file operation Commands (copy, symlink, configure) for all files attached to the entity. :param cmd_list: A CommandList object containing the commands to be executed - :param entity: The Job's attached entity + :param smartsim_entity: The Job's attached entity :param context: A GenerationContext object that holds the Job's run directory """ - copy_ret = cls._copy_files(entity.files.copy_operations, context) + copy_ret = cls._copy_files(smartsim_entity.files.copy_operations, context) cmd_list.extend(copy_ret) - symlink_ret = cls._symlink_files(entity.files.symlink_operations, context) + symlink_ret = cls._symlink_files( + smartsim_entity.files.symlink_operations, context + ) cmd_list.extend(symlink_ret) - configure_ret = cls._configure_files(entity.files.configure_operations, context) + configure_ret = cls._configure_files( + smartsim_entity.files.configure_operations, context + ) cmd_list.extend(configure_ret) @classmethod @@ -266,7 +273,7 @@ def _execute_commands(cls, cmd_list: CommandList) -> None: :param cmd_list: A CommandList object containing the commands to be executed """ for cmd in cmd_list: - subprocess.run(cmd.command) + subprocess.run(cmd.command, check=False) @staticmethod def _mkdir_file(file_path: pathlib.Path) -> Command: diff --git a/smartsim/_core/launcher/dragon/dragon_launcher.py b/smartsim/_core/launcher/dragon/dragon_launcher.py index 752b6c2495..4af93b68ed 100644 --- a/smartsim/_core/launcher/dragon/dragon_launcher.py +++ b/smartsim/_core/launcher/dragon/dragon_launcher.py @@ -69,6 +69,11 @@ logger = get_logger(__name__) +# *************************************** +# TODO: Remove pylint disable after merge +# *************************************** +# pylint: disable=protected-access,wrong-import-position + class DragonLauncher(WLMLauncher): """This class encapsulates the functionality needed @@ -378,7 +383,9 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: # TODO: Remove this registry and move back to builder file after fixing # circular import caused by `DragonLauncher.supported_rs` # ----------------------------------------------------------------------------- -from smartsim.settings.arguments.launch.dragon import DragonLaunchArguments +from smartsim.settings.arguments.launch.dragon import ( + DragonLaunchArguments, +) # pylint: disable=wrong-import-position def _as_run_request_args_and_policy( diff --git a/smartsim/_core/launcher/step/dragon_step.py b/smartsim/_core/launcher/step/dragon_step.py index 63bc1e6c4b..f1e8662e2a 100644 --- a/smartsim/_core/launcher/step/dragon_step.py +++ b/smartsim/_core/launcher/step/dragon_step.py @@ -48,6 +48,7 @@ logger = get_logger(__name__) +# pylint: disable=too-many-function-args class DragonStep(Step): def __init__(self, name: str, cwd: str, run_settings: DragonRunSettings) -> None: """Initialize a srun job step diff --git a/smartsim/_core/launcher/step/sge_step.py b/smartsim/_core/launcher/step/sge_step.py index 2406b19da5..505f5c1b9b 100644 --- a/smartsim/_core/launcher/step/sge_step.py +++ b/smartsim/_core/launcher/step/sge_step.py @@ -33,6 +33,9 @@ logger = get_logger(__name__) +# pylint: disable=too-many-function-args + + class SgeQsubBatchStep(Step): def __init__( self, name: str, cwd: str, batch_settings: SgeQsubBatchSettings diff --git a/smartsim/_core/launcher/step/slurm_step.py b/smartsim/_core/launcher/step/slurm_step.py index 90d457f1b3..410d14d269 100644 --- a/smartsim/_core/launcher/step/slurm_step.py +++ b/smartsim/_core/launcher/step/slurm_step.py @@ -29,7 +29,6 @@ import typing as t from shlex import split as sh_split -from ....builders import Ensemble from ....entity import Application, FSNode from ....error import AllocationError from ....log import get_logger @@ -211,9 +210,10 @@ def _build_exe(self) -> t.List[str]: args = self._get_exe_args_list(self.entity) return exe + args - # There is an issue here, exe and exe_args are no longer attached to the runsettings - # This functions is looping through the list of run_settings.mpmd and build the variable - # cmd + # There is an issue here, exe and exe_args are no longer attached to the + # runsettings + # This functions is looping through the list of run_settings.mpmd and + # build the variable cmd def _make_mpmd(self) -> t.List[str]: """Build Slurm multi-prog (MPMD) executable""" exe = self.entity.exe @@ -221,7 +221,7 @@ def _make_mpmd(self) -> t.List[str]: cmd = exe + args compound_env_vars = [] - for mpmd_rs in self._get_mpmd(): + for mpmd_rs in self._get_mpmd(): # returns a list of runsettings cmd += [" : "] cmd += mpmd_rs.format_run_args() cmd += ["--job-name", self.name] diff --git a/smartsim/_core/launcher/step/step.py b/smartsim/_core/launcher/step/step.py index b5e79a3638..484159158e 100644 --- a/smartsim/_core/launcher/step/step.py +++ b/smartsim/_core/launcher/step/step.py @@ -38,7 +38,6 @@ from smartsim._core.config import CONFIG from smartsim.error.errors import SmartSimError, UnproxyableStepError -from ....builders import Ensemble from ....entity import Application, FSNode from ....log import get_logger from ....settings import RunSettings, SettingsBase @@ -46,6 +45,8 @@ logger = get_logger(__name__) +# pylint: disable=too-many-function-args + def write_colocated_launch_script(): pass diff --git a/smartsim/_core/mli/client/protoclient.py b/smartsim/_core/mli/client/protoclient.py index 46598a8171..1a38d42412 100644 --- a/smartsim/_core/mli/client/protoclient.py +++ b/smartsim/_core/mli/client/protoclient.py @@ -30,6 +30,7 @@ import dragon.channels from dragon.globalservices.api_setup import connect_to_infrastructure +# TODO: fix import for mpi4py.MPI, add in setup.py --> mpi4py try: from mpi4py import MPI # type: ignore[import-not-found] except Exception: diff --git a/smartsim/_core/mli/infrastructure/control/dragon_util.py b/smartsim/_core/mli/infrastructure/control/dragon_util.py index 93bae64e69..166d9ce3da 100644 --- a/smartsim/_core/mli/infrastructure/control/dragon_util.py +++ b/smartsim/_core/mli/infrastructure/control/dragon_util.py @@ -32,8 +32,6 @@ import pytest -dragon = pytest.importorskip("dragon") - # isort: off import dragon.infrastructure.policy as dragon_policy @@ -44,6 +42,8 @@ from smartsim.log import get_logger +dragon = pytest.importorskip("dragon") + logger = get_logger(__name__) diff --git a/smartsim/_core/shell/shell_launcher.py b/smartsim/_core/shell/shell_launcher.py index 9f88d0545c..385b31fd0c 100644 --- a/smartsim/_core/shell/shell_launcher.py +++ b/smartsim/_core/shell/shell_launcher.py @@ -40,7 +40,6 @@ from smartsim._core.utils.launcher import create_job_id from smartsim.error import errors from smartsim.log import get_logger -from smartsim.settings.arguments.launch_arguments import LaunchArguments from smartsim.status import JobStatus from smartsim.types import LaunchedJobID @@ -51,6 +50,8 @@ logger = get_logger(__name__) +# pylint: disable=unspecified-encoding + class ShellLauncherCommand(t.NamedTuple): env: EnvironMappingType @@ -125,7 +126,8 @@ def __init__(self) -> None: """Initialize a new shell launcher.""" self._launched: dict[LaunchedJobID, sp.Popen[bytes]] = {} - def check_popen_inputs(self, shell_command: ShellLauncherCommand) -> None: + @staticmethod + def check_popen_inputs(shell_command: ShellLauncherCommand) -> None: """Validate that the contents of a shell command are valid. :param shell_command: The command to validate diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 265205bef4..e498c26209 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -70,7 +70,7 @@ def unpack(value: _NestedJobSequenceType) -> t.Generator[Job, None, None]: :param value: Sequence containing elements of type Job or other sequences that are also of type _NestedJobSequenceType :return: flattened list of Jobs""" - from smartsim.launchable.job import Job + from smartsim.launchable.job import Job # pylint: disable=import-outside-toplevel for item in value: @@ -606,45 +606,46 @@ def push_unique(self, fn: _TSignalHandlerFn) -> bool: self.push(fn) return did_push - def _create_pinning_string( - pin_ids: t.Optional[t.Iterable[t.Union[int, t.Iterable[int]]]], cpus: int - ) -> t.Optional[str]: - """Create a comma-separated string of CPU ids. By default, ``None`` - returns 0,1,...,cpus-1; an empty iterable will disable pinning - altogether, and an iterable constructs a comma separated string of - integers (e.g. ``[0, 2, 5]`` -> ``"0,2,5"``) - - :params pin_ids: CPU ids - :params cpu: number of CPUs - :raises TypeError: if pin id is not an iterable of ints - :returns: a comma separated string of CPU ids - """ - try: - pin_ids = tuple(pin_ids) if pin_ids is not None else None - except TypeError: - raise TypeError( - "Expected a cpu pinning specification of type iterable of ints or " - f"iterables of ints. Instead got type `{type(pin_ids)}`" - ) from None - - # Deal with MacOSX limitations first. The "None" (default) disables pinning - # and is equivalent to []. The only invalid option is a non-empty pinning - if sys.platform == "darwin": - if pin_ids: - warnings.warn( - "CPU pinning is not supported on MacOSX. Ignoring pinning " - "specification.", - RuntimeWarning, - ) - return None - - # Flatten the iterable into a list and check to make sure that the resulting - # elements are all ints - if pin_ids is None: - return ",".join(_stringify_id(i) for i in range(cpus)) - if not pin_ids: - return None - pin_ids = ((x,) if isinstance(x, int) else x for x in pin_ids) - to_fmt = itertools.chain.from_iterable(pin_ids) - return ",".join(sorted({_stringify_id(x) for x in to_fmt})) +def _create_pinning_string( + pin_ids: t.Optional[t.Iterable[t.Union[int, t.Iterable[int]]]], cpus: int +) -> t.Optional[str]: + """Create a comma-separated string of CPU ids. By default, ``None`` + returns 0,1,...,cpus-1; an empty iterable will disable pinning + altogether, and an iterable constructs a comma separated string of + integers (e.g. ``[0, 2, 5]`` -> ``"0,2,5"``) + + :params pin_ids: CPU ids + :params cpu: number of CPUs + :raises TypeError: if pin id is not an iterable of ints + :returns: a comma separated string of CPU ids + """ + + try: + pin_ids = tuple(pin_ids) if pin_ids is not None else None + except TypeError: + raise TypeError( + "Expected a cpu pinning specification of type iterable of ints or " + f"iterables of ints. Instead got type `{type(pin_ids)}`" + ) from None + + # Deal with MacOSX limitations first. The "None" (default) disables pinning + # and is equivalent to []. The only invalid option is a non-empty pinning + if sys.platform == "darwin": + if pin_ids: + warnings.warn( + "CPU pinning is not supported on MacOSX. Ignoring pinning " + "specification.", + RuntimeWarning, + ) + return None + + # Flatten the iterable into a list and check to make sure that the resulting + # elements are all ints + if pin_ids is None: + return ",".join(_stringify_id(i) for i in range(cpus)) + if not pin_ids: + return None + pin_ids = ((x,) if isinstance(x, int) else x for x in pin_ids) + to_fmt = itertools.chain.from_iterable(pin_ids) + return ",".join(sorted({_stringify_id(x) for x in to_fmt})) diff --git a/smartsim/_core/utils/serialize.py b/smartsim/_core/utils/serialize.py index 46c0a2c1da..c2f70a25ab 100644 --- a/smartsim/_core/utils/serialize.py +++ b/smartsim/_core/utils/serialize.py @@ -33,14 +33,15 @@ import smartsim._core._cli.utils as _utils import smartsim.log +from smartsim.settings.batch_settings import BatchSettings +from smartsim.settings.launch_settings import LaunchSettings if t.TYPE_CHECKING: from smartsim._core.control.manifest import LaunchedManifest as _Manifest from smartsim.builders import Ensemble - from smartsim.database.orchestrator import FeatureStore + from smartsim.database.feature_store import FeatureStore from smartsim.entity import Application, FSNode from smartsim.entity.dbobject import FSModel, FSScript - from smartsim.settings.base import BatchSettings, RunSettings TStepLaunchMetaData = t.Tuple[ @@ -51,6 +52,11 @@ _LOGGER = smartsim.log.get_logger(__name__) +# *************************************** +# TODO: Remove pylint disable after merge +# *************************************** +# pylint: disable=assignment-from-none + def save_launch_manifest(manifest: _Manifest[TStepLaunchMetaData]) -> None: manifest.metadata.run_telemetry_subdirectory.mkdir(parents=True, exist_ok=True) @@ -193,7 +199,7 @@ def _dictify_ensemble( } -def _dictify_run_settings(run_settings: RunSettings) -> t.Dict[str, t.Any]: +def _dictify_run_settings(run_settings: LaunchSettings) -> t.Dict[str, t.Any]: # TODO: remove this downcast if hasattr(run_settings, "mpmd") and run_settings.mpmd: _LOGGER.warning( @@ -219,7 +225,7 @@ def _dictify_batch_settings(batch_settings: BatchSettings) -> t.Dict[str, t.Any] def _dictify_fs( - fs: FeatureStore, + feature_store: FeatureStore, nodes: t.Sequence[t.Tuple[FSNode, TStepLaunchMetaData]], ) -> t.Dict[str, t.Any]: fs_path = _utils.get_fs_path() @@ -229,9 +235,9 @@ def _dictify_fs( fs_type = "Unknown" return { - "name": fs.name, + "name": feature_store.fs_identifier, "type": fs_type, - "interface": fs._interfaces, # pylint: disable=protected-access + "interface": feature_store._interfaces, # pylint: disable=protected-access "shards": [ { **shard.to_dict(), @@ -239,14 +245,18 @@ def _dictify_fs( "out_file": out_file, "err_file": err_file, "memory_file": ( - str(status_dir / "memory.csv") if fs.telemetry.is_enabled else "" + str(status_dir / "memory.csv") + if feature_store.telemetry.is_enabled + else "" ), "client_file": ( - str(status_dir / "client.csv") if fs.telemetry.is_enabled else "" + str(status_dir / "client.csv") + if feature_store.telemetry.is_enabled + else "" ), "client_count_file": ( str(status_dir / "client_count.csv") - if fs.telemetry.is_enabled + if feature_store.telemetry.is_enabled else "" ), "telemetry_metadata": { diff --git a/smartsim/_core/utils/telemetry/telemetry.py b/smartsim/_core/utils/telemetry/telemetry.py index c8ff3bf25e..d19db1ef45 100644 --- a/smartsim/_core/utils/telemetry/telemetry.py +++ b/smartsim/_core/utils/telemetry/telemetry.py @@ -28,7 +28,6 @@ import logging import os import pathlib -import threading import typing as t from watchdog.events import ( diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index d87ada15aa..ee83b343ea 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -41,9 +41,6 @@ from smartsim.launchable.job import Job from smartsim.settings.launch_settings import LaunchSettings -if t.TYPE_CHECKING: - from smartsim.settings.launch_settings import LaunchSettings - class Ensemble(entity.CompoundEntity): """An Ensemble is a builder class that parameterizes the creation of multiple @@ -69,63 +66,76 @@ def __init__( **Parameter Expansion** - Parameter expansion allows users to assign different parameter values to - multiple Applications. This is done by specifying input to `Ensemble.file_parameters`, - `Ensemble.exe_arg_parameters` and `Ensemble.permutation_strategy`. The `permutation_strategy` - argument accepts three options: + Parameter expansion allows users to assign different parameter + values to multiple Applications. This is done by specifying input + to `Ensemble.file_parameters`, `Ensemble.exe_arg_parameters` and + `Ensemble.permutation_strategy`. The `permutation_strategy` argument + accepts three options: - 1. "all_perm": Generates all possible parameter permutations for exhaustive exploration. - 2. "step": Collects identically indexed values across parameter lists to create parameter sets. + 1. "all_perm": Generates all possible parameter permutations for + exhaustive exploration. + 2. "step": Collects identically indexed values across parameter + lists to create parameter sets. 3. "random": Enables random selection from predefined parameter spaces. - The example below demonstrates creating an Ensemble via parameter expansion, resulting in - the creation of two Applications: + The example below demonstrates creating an Ensemble via parameter + expansion, resulting in the creation of two Applications: .. highlight:: python .. code-block:: python file_params={"SPAM": ["a", "b"], "EGGS": ["c", "d"]} - exe_arg_parameters = {"EXE": [["a"], ["b", "c"]], "ARGS": [["d"], ["e", "f"]]} - ensemble = Ensemble(name="name",exe="python",exe_arg_parameters=exe_arg_parameters, + exe_arg_parameters = {"EXE": [["a"], ["b", "c"]], + "ARGS": [["d"], ["e", "f"]]} + ensemble = Ensemble(name="name",exe="python", + exe_arg_parameters=exe_arg_parameters, file_parameters=file_params,permutation_strategy="step") This configuration will yield the following permutations: .. highlight:: python .. code-block:: python - [ParamSet(params={'SPAM': 'a', 'EGGS': 'c'}, exe_args={'EXE': ['a'], 'ARGS': ['d']}), - ParamSet(params={'SPAM': 'b', 'EGGS': 'd'}, exe_args={'EXE': ['b', 'c'], 'ARGS': ['e', 'f']})] + [ParamSet(params={'SPAM': 'a', 'EGGS': 'c'}, + exe_args={'EXE': ['a'], 'ARGS': ['d']}), + ParamSet(params={'SPAM': 'b', 'EGGS': 'd'}, + exe_args={'EXE': ['b', 'c'], 'ARGS': ['e', 'f']})] - Each ParamSet contains the parameters assigned from file_params and the corresponding executable - arguments from exe_arg_parameters. + Each ParamSet contains the parameters assigned from file_params and the + corresponding executable arguments from exe_arg_parameters. **Replication** - The replication strategy involves creating identical Applications within an Ensemble. - This is achieved by specifying the `replicas` argument in the Ensemble. + The replication strategy involves creating identical Applications within + an Ensemble. This is achieved by specifying the `replicas` argument in + the Ensemble. - For example, by applying the `replicas` argument to the previous parameter expansion - example, we can double our Application output: + For example, by applying the `replicas` argument to the previous + parameter expansion example, we can double our Application output: .. highlight:: python .. code-block:: python file_params={"SPAM": ["a", "b"], "EGGS": ["c", "d"]} - exe_arg_parameters = {"EXE": [["a"], ["b", "c"]], "ARGS": [["d"], ["e", "f"]]} - ensemble = Ensemble(name="name",exe="python",exe_arg_parameters=exe_arg_parameters, - file_parameters=file_params,permutation_strategy="step", replicas=2) + exe_arg_parameters = {"EXE": [["a"], ["b", "c"]], + "ARGS": [["d"], ["e", "f"]]} + ensemble = Ensemble(name="name",exe="python", + exe_arg_parameters=exe_arg_parameters, + file_parameters=file_params, + permutation_strategy="step", replicas=2) - This configuration will result in each ParamSet being replicated, effectively doubling - the number of Applications created. + This configuration will result in each ParamSet being replicated, + effectively doubling the number of Applications created. :param name: name of the ensemble :param exe: executable to run :param exe_args: executable arguments - :param exe_arg_parameters: parameters and values to be used when configuring entities + :param exe_arg_parameters: parameters and values to be used + when configuring entities :param files: files to be copied, symlinked, and/or configured prior to execution :param file_parameters: parameters and values to be used when configuring files - :param permutation_strategy: strategy to control how the param values are applied to the Ensemble + :param permutation_strategy: strategy to control how the param values are + applied to the Ensemble :param max_permutations: max parameter permutations to set for the ensemble :param replicas: number of identical entities to create within an Ensemble """ @@ -365,11 +375,13 @@ def replicas(self, value: int) -> None: self._replicas = value def _create_applications(self) -> tuple[Application, ...]: - """Generate a collection of Application instances based on the Ensembles attributes. + """Generate a collection of Application instances based + on the Ensembles attributes. - This method uses a permutation strategy to create various combinations of file - parameters and executable arguments. Each combination is then replicated according - to the specified number of replicas, resulting in a set of Application instances. + This method uses a permutation strategy to create various + combinations of file parameters and executable arguments. + Each combination is then replicated according to the specified + number of replicas, resulting in a set of Application instances. :return: A tuple of Application instances """ diff --git a/smartsim/database/__init__.py b/smartsim/database/__init__.py index 0801c682bd..e35143e47e 100644 --- a/smartsim/database/__init__.py +++ b/smartsim/database/__init__.py @@ -24,4 +24,4 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .orchestrator import FeatureStore +from .feature_store import FeatureStore diff --git a/smartsim/database/orchestrator.py b/smartsim/database/feature_store.py similarity index 95% rename from smartsim/database/orchestrator.py rename to smartsim/database/feature_store.py index c29c781a17..4082062ba1 100644 --- a/smartsim/database/orchestrator.py +++ b/smartsim/database/feature_store.py @@ -26,6 +26,11 @@ # pylint: disable=too-many-lines +# *************************************** +# TODO: Remove pylint disable after merge +# *************************************** +# pylint: disable=no-member,too-many-function-args,assignment-from-no-return,abstract-class-instantiated,no-value-for-parameter,unused-argument + import itertools import os.path as osp import shutil @@ -361,8 +366,8 @@ def reset_hosts(self) -> None: def remove_stale_files(self) -> None: """Can be used to remove feature store files of a previous launch""" - for fs in self.entities: - fs.remove_stale_fsnode_files() + for feature_store in self.entities: + feature_store.remove_stale_fsnode_files() def get_address(self) -> t.List[str]: """Return feature store addresses @@ -420,10 +425,10 @@ def set_cpus(self, num_cpus: int) -> None: if hasattr(self.batch_settings, "set_cpus_per_task"): self.batch_settings.set_cpus_per_task(num_cpus) - for fs in self.entities: - fs.run_settings.set_cpus_per_task(num_cpus) - if fs.is_mpmd and hasattr(fs.run_settings, "mpmd"): - for mpmd in fs.run_settings.mpmd: + for feature_store in self.entities: + feature_store.run_settings.set_cpus_per_task(num_cpus) + if feature_store.is_mpmd and hasattr(feature_store.run_settings, "mpmd"): + for mpmd in feature_store.run_settings.mpmd: mpmd.set_cpus_per_task(num_cpus) def set_walltime(self, walltime: str) -> None: @@ -458,8 +463,8 @@ def set_hosts(self, host_list: t.Union[t.List[str], str]) -> None: self.batch_settings.set_hostlist(host_list) if self.launcher == "lsf": - for fs in self.entities: - fs.set_hosts(host_list) + for feature_store in self.entities: + feature_store.set_hosts(host_list) elif ( self.launcher == "pals" and isinstance(self.entities[0].run_settings, PalsMpiexecSettings) @@ -468,15 +473,19 @@ def set_hosts(self, host_list: t.Union[t.List[str], str]) -> None: # In this case, --hosts is a global option, set it to first run command self.entities[0].run_settings.set_hostlist(host_list) else: - for host, fs in zip(host_list, self.entities): - if isinstance(fs.run_settings, AprunSettings): + for host, feature_store in zip(host_list, self.entities): + if isinstance(feature_store.run_settings, AprunSettings): if not self.batch: - fs.run_settings.set_hostlist([host]) + feature_store.run_settings.set_hostlist([host]) else: - fs.run_settings.set_hostlist([host]) - - if fs.is_mpmd and hasattr(fs.run_settings, "mpmd"): - for i, mpmd_runsettings in enumerate(fs.run_settings.mpmd, 1): + feature_store.run_settings.set_hostlist([host]) + + if feature_store.is_mpmd and hasattr( + feature_store.run_settings, "mpmd" + ): + for i, mpmd_runsettings in enumerate( + feature_store.run_settings.mpmd, 1 + ): mpmd_runsettings.set_hostlist(host_list[i]) def set_batch_arg(self, arg: str, value: t.Optional[str] = None) -> None: @@ -517,10 +526,12 @@ def set_run_arg(self, arg: str, value: t.Optional[str] = None) -> None: "it is a reserved keyword in FeatureStore" ) else: - for fs in self.entities: - fs.run_settings.run_args[arg] = value - if fs.is_mpmd and hasattr(fs.run_settings, "mpmd"): - for mpmd in fs.run_settings.mpmd: + for feature_store in self.entities: + feature_store.run_settings.run_args[arg] = value + if feature_store.is_mpmd and hasattr( + feature_store.run_settings, "mpmd" + ): + for mpmd in feature_store.run_settings.mpmd: mpmd.run_args[arg] = value def enable_checkpoints(self, frequency: int) -> None: @@ -854,11 +865,11 @@ def _get_start_script_args( def _get_fs_hosts(self) -> t.List[str]: hosts = [] - for fs in self.entities: - if not fs.is_mpmd: - hosts.append(fs.host) + for feature_store in self.entities: + if not feature_store.is_mpmd: + hosts.append(feature_store.host) else: - hosts.extend(fs.hosts) + hosts.extend(feature_store.hosts) return hosts def _check_network_interface(self) -> None: diff --git a/smartsim/entity/_mock.py b/smartsim/entity/_mock.py index 8f1043ed3c..7b9c43c5c8 100644 --- a/smartsim/entity/_mock.py +++ b/smartsim/entity/_mock.py @@ -34,6 +34,18 @@ import typing as t +import pytest + +from smartsim._core.mli.infrastructure.control.worker_manager import build_failure_reply + +dragon = pytest.importorskip("dragon") + +if t.TYPE_CHECKING: + from smartsim._core.mli.mli_schemas.response.response_capnp import Status + +# The tests in this file belong to the dragon group +pytestmark = pytest.mark.dragon + class Mock: """Base mock class""" @@ -44,3 +56,28 @@ def __getattr__(self, _: str) -> Mock: def __deepcopy__(self, _: dict[t.Any, t.Any]) -> Mock: return type(self)() + + +@pytest.mark.parametrize( + "status, message", + [ + pytest.param("timeout", "Worker timed out", id="timeout"), + pytest.param("fail", "Failed while executing", id="fail"), + ], +) +def test_build_failure_reply(status: "Status", message: str): + "Ensures failure replies can be built successfully" + response = build_failure_reply(status, message) + display_name = response.schema.node.displayName # type: ignore + class_name = display_name.split(":")[-1] + assert class_name == "Response" + assert response.status == status + assert response.message == message + + +def test_build_failure_reply_fails(): + "Ensures ValueError is raised if a Status Enum is not used" + with pytest.raises(ValueError) as ex: + build_failure_reply("not a status enum", "message") + + assert "Error assigning status to response" in ex.value.args[0] diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index 501279c85f..3dd37d4afe 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -30,7 +30,6 @@ import copy import textwrap import typing as t -from os import path as osp from .._core.generation.operations.operations import FileSysOperationSet from .._core.utils.helpers import expand_exe_path @@ -217,7 +216,8 @@ def key_prefixing_enabled(self, value: bool) -> None: self.key_prefixing_enabled = copy.deepcopy(value) def as_executable_sequence(self) -> t.Sequence[str]: - """Converts the executable and its arguments into a sequence of program arguments. + """Converts the executable and its arguments into a sequence of program + arguments. :return: a sequence of strings representing the executable and its arguments """ @@ -250,7 +250,8 @@ def _build_exe_args(exe_args: t.Union[str, t.Sequence[str], None]) -> t.List[str def __str__(self) -> str: # pragma: no cover exe_args_str = "\n".join(self.exe_args) entities_str = "\n".join(str(entity) for entity in self.incoming_entities) - return textwrap.dedent(f"""\ + return textwrap.dedent( + f"""\ Name: {self.name} Type: {self.type} Executable: @@ -260,4 +261,5 @@ def __str__(self) -> str: # pragma: no cover Incoming Entities: {entities_str} Key Prefixing Enabled: {self.key_prefixing_enabled} - """) + """ + ) diff --git a/smartsim/entity/dbnode.py b/smartsim/entity/dbnode.py index 60a69b5222..c2e25199ef 100644 --- a/smartsim/entity/dbnode.py +++ b/smartsim/entity/dbnode.py @@ -65,6 +65,7 @@ def __init__( ) -> None: """Initialize a feature store node within an feature store.""" super().__init__(name) + self.path = path self.run_settings = run_settings self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] self.exe_args = exe_args or [] diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index 3f5a9eabd0..561cb73683 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -111,7 +111,8 @@ def __init__( @abc.abstractmethod def as_executable_sequence(self) -> t.Sequence[str]: - """Converts the executable and its arguments into a sequence of program arguments. + """Converts the executable and its arguments into a sequence of + program arguments. :return: a sequence of strings representing the executable and its arguments """ diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 4db503819a..9e6a657d90 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -46,13 +46,12 @@ from smartsim.status import TERMINAL_STATUSES, InvalidJobStatus, JobStatus from ._core import Generator, Manifest -from ._core.generation.generator import Job_Path +from ._core.generation.generator import JobPath from .entity import TelemetryConfiguration from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer if t.TYPE_CHECKING: - from smartsim.launchable.job import Job from smartsim.types import LaunchedJobID logger = get_logger(__name__) @@ -210,18 +209,18 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: args = job.launch_settings.launch_args env = job.launch_settings.env_vars exe = job.entity.as_executable_sequence() - dispatch = dispatcher.get_dispatch(args) + dispatch_item = dispatcher.get_dispatch(args) try: # Check to see if one of the existing launchers can be # configured to handle the launch arguments ... - launch_config = dispatch.configure_first_compatible_launcher( + launch_config = dispatch_item.configure_first_compatible_launcher( from_available_launchers=self._launch_history.iter_past_launchers(), with_arguments=args, ) except errors.LauncherNotFoundError: # ... otherwise create a new launcher that _can_ handle the # launch arguments and configure _that_ one - launch_config = dispatch.create_new_launcher_configuration( + launch_config = dispatch_item.create_new_launcher_configuration( for_experiment=self, with_arguments=args ) # Generate the job directory and return the generated job path @@ -361,7 +360,7 @@ def is_finished( return final @_contextualize - def _generate(self, generator: Generator, job: Job, job_index: int) -> Job_Path: + def _generate(self, generator: Generator, job: Job, job_index: int) -> JobPath: """Generate the directory structure and files for a ``Job`` If files or directories are attached to an ``Application`` object @@ -490,8 +489,8 @@ def _append_to_fs_identifier_list(self, fs_identifier: str) -> None: """Check if fs_identifier already exists when calling create_feature_store""" if fs_identifier in self._fs_identifiers: logger.warning( - f"A feature store with the identifier {fs_identifier} has already been made " - "An error will be raised if multiple Feature Stores are started " + f"A feature store with the identifier {fs_identifier} has already " + "been made. An error will be raised if multiple Feature Stores are " "with the same identifier" ) # Otherwise, add diff --git a/smartsim/launchable/base_job_group.py b/smartsim/launchable/base_job_group.py index 9031705f39..523e605b21 100644 --- a/smartsim/launchable/base_job_group.py +++ b/smartsim/launchable/base_job_group.py @@ -48,14 +48,13 @@ def jobs(self) -> t.List[BaseJob]: It represents the collection of jobs associated with an instance of the BaseJobGroup abstract class. """ - pass - def insert(self, idx: int, value: BaseJob) -> None: + def insert(self, index: int, value: BaseJob) -> None: """Inserts the given value at the specified index (idx) in the list of jobs. If the index is out of bounds, the method prints an error message. """ - self.jobs.insert(idx, value) + self.jobs.insert(index, value) def __iter__(self) -> t.Iterator[BaseJob]: """Allows iteration over the jobs in the collection.""" diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index 6082ba61d7..132dd7c05c 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -57,8 +57,9 @@ def __init__( ): """Initialize a ``Job`` - Jobs require a SmartSimEntity and a LaunchSettings. Optionally, users may provide - a name. To create a simple Job that echos `Hello World!`, consider the example below: + Jobs require a SmartSimEntity and a LaunchSettings. Optionally, + users may provide a name. To create a simple Job that echos `Hello World!`, + consider the example below: .. highlight:: python .. code-block:: python @@ -118,6 +119,7 @@ def entity(self, value: SmartSimEntity) -> None: :param value: the SmartSimEntity :raises Type Error: if entity is not SmartSimEntity """ + # pylint: disable=import-outside-toplevel from smartsim.entity.entity import SmartSimEntity if not isinstance(value, SmartSimEntity): diff --git a/smartsim/launchable/launchable.py b/smartsim/launchable/launchable.py index 7a8af2c19a..d3146587b7 100644 --- a/smartsim/launchable/launchable.py +++ b/smartsim/launchable/launchable.py @@ -28,11 +28,7 @@ class SmartSimObject: """Base Class for SmartSim Objects""" - ... - class Launchable(SmartSimObject): """Base Class for anything than can be passed into Experiment.start()""" - - ... diff --git a/smartsim/launchable/mpmd_job.py b/smartsim/launchable/mpmd_job.py index e526f10746..2b59746c22 100644 --- a/smartsim/launchable/mpmd_job.py +++ b/smartsim/launchable/mpmd_job.py @@ -60,7 +60,7 @@ def _check_entity(mpmd_pairs: t.List[MPMDPair]) -> None: ret: SmartSimEntity | None = None for mpmd_pair in mpmd_pairs: if flag == 1: - if type(ret) == type(mpmd_pair.entity): + if isinstance(ret,type(mpmd_pair.entity)): flag = 0 else: raise SSUnsupportedError( @@ -110,9 +110,13 @@ def get_launch_steps(self) -> LaunchCommands: def __str__(self) -> str: # pragma: no cover """returns A user-readable string of a MPMD Job""" - fmt = lambda mpmd_pair: textwrap.dedent(f"""\ - == MPMD Pair == - {mpmd_pair.entity} - {mpmd_pair.launch_settings} - """) - return "\n".join(map(fmt, self.mpmd_pairs)) + + def _fmt(mpmd_pair: MPMDPair) -> str: + return textwrap.dedent( + f"""\ + == MPMD Pair == + {mpmd_pair.entity} + {mpmd_pair.launch_settings} + """ + ) + return "\n".join(map(_fmt, self.mpmd_pairs)) diff --git a/smartsim/ml/data.py b/smartsim/ml/data.py index 21e4e33a5d..dcdc886f44 100644 --- a/smartsim/ml/data.py +++ b/smartsim/ml/data.py @@ -38,6 +38,12 @@ logger = get_logger(__name__) +# *************************************** +# TODO: Remove pylint disable after merge +# *************************************** + +# pylint: disable=unsubscriptable-object + def form_name(*args: t.Any) -> str: return "_".join(str(arg) for arg in args if arg is not None) @@ -81,7 +87,6 @@ def publish(self) -> None: :param client: Client to connect to Feature Store """ - ... def download(self) -> None: """Download DataInfo information from FeatureStore @@ -128,6 +133,7 @@ class TrainingDataUploader: """ + # pylint: disable=unused-argument,unsubscriptable-object def __init__( self, list_name: str = "training_data", @@ -187,7 +193,7 @@ def put_batch( if self.verbose: logger.info(f"Added dataset to list {self.list_name}") - logger.info(f"List length") + logger.info("List length") self.batch_idx += 1 diff --git a/smartsim/settings/arguments/batch/lsf.py b/smartsim/settings/arguments/batch/lsf.py index 23f948bd09..98a4451684 100644 --- a/smartsim/settings/arguments/batch/lsf.py +++ b/smartsim/settings/arguments/batch/lsf.py @@ -31,7 +31,6 @@ from smartsim.log import get_logger from ...batch_command import BatchSchedulerType -from ...common import StringArgument from ..batch_arguments import BatchArguments logger = get_logger(__name__) @@ -47,7 +46,7 @@ def scheduler_str(self) -> str: :returns: The string representation of the scheduler """ - return BatchSchedulerType.Lsf.value + return BatchSchedulerType.LSF.value def set_walltime(self, walltime: str) -> None: """Set the walltime diff --git a/smartsim/settings/arguments/batch/pbs.py b/smartsim/settings/arguments/batch/pbs.py index 1262076656..6f61e13773 100644 --- a/smartsim/settings/arguments/batch/pbs.py +++ b/smartsim/settings/arguments/batch/pbs.py @@ -33,7 +33,6 @@ from ....error import SSConfigError from ...batch_command import BatchSchedulerType -from ...common import StringArgument from ..batch_arguments import BatchArguments logger = get_logger(__name__) @@ -49,7 +48,7 @@ def scheduler_str(self) -> str: :returns: The string representation of the scheduler """ - return BatchSchedulerType.Pbs.value + return BatchSchedulerType.PBS.value def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job diff --git a/smartsim/settings/arguments/batch/slurm.py b/smartsim/settings/arguments/batch/slurm.py index 26f9cf8549..d26d59fb05 100644 --- a/smartsim/settings/arguments/batch/slurm.py +++ b/smartsim/settings/arguments/batch/slurm.py @@ -32,7 +32,6 @@ from smartsim.log import get_logger from ...batch_command import BatchSchedulerType -from ...common import StringArgument from ..batch_arguments import BatchArguments logger = get_logger(__name__) @@ -48,7 +47,7 @@ def scheduler_str(self) -> str: :returns: The string representation of the scheduler """ - return BatchSchedulerType.Slurm.value + return BatchSchedulerType.SLURM.value def set_walltime(self, walltime: str) -> None: """Set the walltime of the job diff --git a/smartsim/settings/arguments/batch_arguments.py b/smartsim/settings/arguments/batch_arguments.py index 0fa8d39640..e285168192 100644 --- a/smartsim/settings/arguments/batch_arguments.py +++ b/smartsim/settings/arguments/batch_arguments.py @@ -51,7 +51,6 @@ def __init__(self, batch_args: t.Dict[str, str | None] | None) -> None: @abstractmethod def scheduler_str(self) -> str: """Get the string representation of the launcher""" - pass @abstractmethod def set_account(self, account: str) -> None: @@ -59,7 +58,6 @@ def set_account(self, account: str) -> None: :param account: account id """ - pass @abstractmethod def set_queue(self, queue: str) -> None: @@ -69,7 +67,6 @@ def set_queue(self, queue: str) -> None: :param queue: the partition to run the batch job on """ - pass @abstractmethod def set_walltime(self, walltime: str) -> None: @@ -77,7 +74,6 @@ def set_walltime(self, walltime: str) -> None: :param walltime: wall time """ - pass @abstractmethod def set_nodes(self, num_nodes: int) -> None: @@ -85,7 +81,6 @@ def set_nodes(self, num_nodes: int) -> None: :param num_nodes: number of nodes """ - pass @abstractmethod def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: @@ -94,7 +89,6 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: :param host_list: hosts to launch on :raises TypeError: if not str or list of str """ - pass @abstractmethod def format_batch_args(self) -> t.List[str]: @@ -102,7 +96,6 @@ def format_batch_args(self) -> t.List[str]: :return: batch arguments for Sbatch """ - pass def __str__(self) -> str: # pragma: no-cover string = f"\nScheduler Arguments:\n{fmt_dict(self._batch_args)}" diff --git a/smartsim/settings/arguments/launch/alps.py b/smartsim/settings/arguments/launch/alps.py index 356a443d65..379c7002df 100644 --- a/smartsim/settings/arguments/launch/alps.py +++ b/smartsim/settings/arguments/launch/alps.py @@ -40,9 +40,11 @@ _as_aprun_command = make_shell_format_fn(run_command="aprun") + @dispatch(with_format=_as_aprun_command, to_launcher=ShellLauncher) class AprunLaunchArguments(ShellLaunchArguments): - def _reserved_launch_args(self) -> set[str]: + @staticmethod + def _reserved_launch_args() -> set[str]: """Return reserved launch arguments. :returns: The set of reserved launcher arguments @@ -54,7 +56,7 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Alps.value + return LauncherType.ALPS.value def set_cpus_per_task(self, cpus_per_task: int) -> None: """Set the number of cpus to use per task @@ -207,22 +209,22 @@ def format_launch_args(self) -> t.List[str]: args += ["=".join((prefix + opt, str(value)))] return args - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._reserved_launch_args(): + set_check_input(arg, val) + if arg in self._reserved_launch_args(): logger.warning( ( - f"Could not set argument '{key}': " + f"Could not set argument '{arg}': " f"it is a reserved argument of '{type(self).__name__}'" ) ) return - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val diff --git a/smartsim/settings/arguments/launch/dragon.py b/smartsim/settings/arguments/launch/dragon.py index d8044267e6..e58cf69b1b 100644 --- a/smartsim/settings/arguments/launch/dragon.py +++ b/smartsim/settings/arguments/launch/dragon.py @@ -45,7 +45,7 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Dragon.value + return LauncherType.DRAGON.value def set_nodes(self, nodes: int) -> None: """Set the number of nodes @@ -62,17 +62,17 @@ def set_tasks_per_node(self, tasks_per_node: int) -> None: self.set("tasks_per_node", str(tasks_per_node)) @override - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + set_check_input(arg, val) + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: """Specify the node feature for this job diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index 2c589cb48d..2ffb9be5e2 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -33,7 +33,7 @@ from smartsim._core.shell.shell_launcher import ShellLauncher, make_shell_format_fn from smartsim.log import get_logger -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launch_command import LauncherType logger = get_logger(__name__) @@ -47,7 +47,7 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Local.value + return LauncherType.LOCAL.value def format_env_vars(self, env_vars: t.Mapping[str, str | None]) -> list[str]: """Build bash compatible sequence of strings to specify an environment @@ -74,14 +74,14 @@ def format_launch_args(self) -> t.List[str]: formatted.append(str(value)) return formatted - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + set_check_input(arg, val) + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index ed24271985..4a605de5d0 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -41,6 +41,7 @@ logger = get_logger(__name__) + def _as_jsrun_command( args: ShellLaunchArguments, exe: t.Sequence[str], @@ -69,9 +70,10 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Lsf.value + return LauncherType.LSF.value - def _reserved_launch_args(self) -> set[str]: + @staticmethod + def _reserved_launch_args() -> set[str]: """Return reserved launch arguments. :returns: The set of reserved launcher arguments @@ -131,22 +133,22 @@ def format_launch_args(self) -> t.List[str]: args += ["=".join((prefix + opt, str(value)))] return args - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._reserved_launch_args(): + set_check_input(arg, val) + if arg in self._reserved_launch_args(): logger.warning( ( - f"Could not set argument '{key}': " + f"Could not set argument '{arg}': " f"it is a reserved argument of '{type(self).__name__}'" ) ) return - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val diff --git a/smartsim/settings/arguments/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py index ce8c43aa5c..d2f20331c7 100644 --- a/smartsim/settings/arguments/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -37,13 +37,16 @@ from ...launch_command import LauncherType logger = get_logger(__name__) + + _as_mpirun_command = make_shell_format_fn("mpirun") _as_mpiexec_command = make_shell_format_fn("mpiexec") _as_orterun_command = make_shell_format_fn("orterun") class _BaseMPILaunchArguments(ShellLaunchArguments): - def _reserved_launch_args(self) -> set[str]: + @staticmethod + def _reserved_launch_args() -> set[str]: """Return reserved launch arguments. :returns: The set of reserved launcher arguments @@ -204,25 +207,25 @@ def format_launch_args(self) -> t.List[str]: args += [prefix + opt, str(value)] return args - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._reserved_launch_args(): + set_check_input(arg, val) + if arg in self._reserved_launch_args(): logger.warning( ( - f"Could not set argument '{key}': " + f"Could not set argument '{arg}': " f"it is a reserved argument of '{type(self).__name__}'" ) ) return - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val @dispatch(with_format=_as_mpirun_command, to_launcher=ShellLauncher) @@ -232,7 +235,7 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Mpirun.value + return LauncherType.MPIRUN.value @dispatch(with_format=_as_mpiexec_command, to_launcher=ShellLauncher) @@ -242,7 +245,7 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Mpiexec.value + return LauncherType.MPIEXEC.value @dispatch(with_format=_as_orterun_command, to_launcher=ShellLauncher) @@ -252,4 +255,4 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Orterun.value + return LauncherType.ORTERUN.value diff --git a/smartsim/settings/arguments/launch/pals.py b/smartsim/settings/arguments/launch/pals.py index d48dc799b9..a60485e5d5 100644 --- a/smartsim/settings/arguments/launch/pals.py +++ b/smartsim/settings/arguments/launch/pals.py @@ -40,6 +40,7 @@ _as_pals_command = make_shell_format_fn(run_command="mpiexec") + @dispatch(with_format=_as_pals_command, to_launcher=ShellLauncher) class PalsMpiexecLaunchArguments(ShellLaunchArguments): def launcher_str(self) -> str: @@ -47,9 +48,10 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Pals.value + return LauncherType.PALS.value - def _reserved_launch_args(self) -> set[str]: + @staticmethod + def _reserved_launch_args() -> set[str]: """Return reserved launch arguments. :returns: The set of reserved launcher arguments @@ -143,20 +145,20 @@ def format_launch_args(self) -> t.List[str]: return args - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._reserved_launch_args(): + set_check_input(arg, val) + if arg in self._reserved_launch_args(): logger.warning( - f"Could not set argument '{key}': " + f"Could not set argument '{arg}': " f"it is a reserved argument of '{type(self).__name__}'" ) return - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index c5dceff628..66f8e59c90 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -71,9 +71,10 @@ def launcher_str(self) -> str: :returns: The string representation of the launcher """ - return LauncherType.Slurm.value + return LauncherType.SLURM.value - def _reserved_launch_args(self) -> set[str]: + @staticmethod + def _reserved_launch_args() -> set[str]: """Return reserved launch arguments. :returns: The set of reserved launcher arguments @@ -311,7 +312,8 @@ def format_comma_sep_env_vars( return fmt_exported_env, compound_env - def _check_env_vars(self, env_vars: t.Mapping[str, str | None]) -> None: + @staticmethod + def _check_env_vars(env_vars: t.Mapping[str, str | None]) -> None: """Warn a user trying to set a variable which is set in the environment Given Slurm's env var precedence, trying to export a variable which is already @@ -332,22 +334,22 @@ def _check_env_vars(self, env_vars: t.Mapping[str, str | None]) -> None: ) logger.warning(msg) - def set(self, key: str, value: str | None) -> None: + def set(self, arg: str, val: str | None) -> None: """Set an arbitrary launch argument - :param key: The launch argument - :param value: A string representation of the value for the launch + :param arg: The launch argument + :param val: A string representation of the value for the launch argument (if applicable), otherwise `None` """ - set_check_input(key, value) - if key in self._reserved_launch_args(): + set_check_input(arg, val) + if arg in self._reserved_launch_args(): logger.warning( ( - f"Could not set argument '{key}': " + f"Could not set argument '{arg}': " f"it is a reserved argument of '{type(self).__name__}'" ) ) return - if key in self._launch_args and key != self._launch_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value + if arg in self._launch_args and arg != self._launch_args[arg]: + logger.warning(f"Overwritting argument '{arg}' with value '{val}'") + self._launch_args[arg] = val diff --git a/smartsim/settings/batch_command.py b/smartsim/settings/batch_command.py index a96492d398..05a7ff71eb 100644 --- a/smartsim/settings/batch_command.py +++ b/smartsim/settings/batch_command.py @@ -30,6 +30,6 @@ class BatchSchedulerType(Enum): """Schedulers supported by SmartSim.""" - Slurm = "slurm" - Pbs = "pbs" - Lsf = "lsf" + SLURM = "slurm" + PBS = "pbs" + LSF = "lsf" diff --git a/smartsim/settings/batch_settings.py b/smartsim/settings/batch_settings.py index 734e919ce3..61b69ca8e7 100644 --- a/smartsim/settings/batch_settings.py +++ b/smartsim/settings/batch_settings.py @@ -48,8 +48,8 @@ class BatchSettings(BaseSettings): used to inject scheduler-specific behavior into a job. BatchSettings is designed to be extended by a BatchArguments child class that - corresponds to the scheduler provided during initialization. The supported schedulers - are Slurm, PBS, and LSF. Using the BatchSettings class, users can: + corresponds to the scheduler provided during initialization. The supported + schedulers are Slurm, PBS, and LSF. Using the BatchSettings class, users can: - Set the scheduler type of a batch job. - Configure batch arguments and environment variables. @@ -66,14 +66,15 @@ def __init__( self, batch_scheduler: t.Union[BatchSchedulerType, str], batch_args: StringArgument | None = None, + # batch_args: StringArgument = None, env_vars: StringArgument | None = None, ) -> None: """Initialize a BatchSettings instance. The "batch_scheduler" of SmartSim BatchSettings will determine the child type assigned to the BatchSettings.batch_args attribute. - For example, to configure a job for SLURM batch jobs, assign BatchSettings.batch_scheduler - to "slurm" or BatchSchedulerType.Slurm: + For example, to configure a job for SLURM batch jobs, assign + BatchSettings.batch_scheduler to "slurm" or BatchSchedulerType.Slurm: .. highlight:: python .. code-block:: python @@ -82,9 +83,9 @@ def __init__( # OR sbatch_settings = BatchSettings(batch_scheduler=BatchSchedulerType.Slurm) - This will assign a SlurmBatchArguments object to ``sbatch_settings.batch_args``. - Using the object, users may access the child class functions to set - batch configurations. For example: + This will assign a SlurmBatchArguments object to + ``sbatch_settings.batch_args``. Using the object, users may access the child + class functions to set batch configurations. For example: .. highlight:: python .. code-block:: python @@ -103,10 +104,11 @@ def __init__( If the key already exists in the existing batch arguments, the value will be overwritten. - :param batch_scheduler: The type of scheduler to initialize (e.g., Slurm, PBS, LSF) - :param batch_args: A dictionary of arguments for the scheduler, where the keys - are strings and the values can be either strings or None. This argument is optional - and defaults to None. + :param batch_scheduler: The type of scheduler to initialize + (e.g., Slurm, PBS, LSF) + :param batch_args: A dictionary of arguments for the scheduler, where + the keys are strings and the values can be either strings or None. + This argument is optional and defaults to None. :param env_vars: Environment variables for the batch settings, where the keys are strings and the values can be either strings or None. This argument is also optional and defaults to None. @@ -151,14 +153,13 @@ def _get_arguments(self, batch_args: StringArgument | None) -> BatchArguments: :returns: The appropriate type for the settings instance. :raises ValueError: An invalid scheduler type was provided. """ - if self._batch_scheduler == BatchSchedulerType.Slurm: + if self._batch_scheduler == BatchSchedulerType.SLURM: return SlurmBatchArguments(batch_args) - elif self._batch_scheduler == BatchSchedulerType.Lsf: + if self._batch_scheduler == BatchSchedulerType.LSF: return BsubBatchArguments(batch_args) - elif self._batch_scheduler == BatchSchedulerType.Pbs: + if self._batch_scheduler == BatchSchedulerType.PBS: return QsubBatchArguments(batch_args) - else: - raise ValueError(f"Invalid scheduler type: {self._batch_scheduler}") + raise ValueError(f"Invalid scheduler type: {self._batch_scheduler}") def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments to preview diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index edca5fd52b..df7eb243aa 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -45,5 +45,6 @@ def set_check_input(key: str, value: t.Optional[str]) -> None: key = key.lstrip("-") logger.warning( "One or more leading `-` characters were provided to the run argument.\n" - "Leading dashes were stripped and the arguments were passed to the run_command." + "Leading dashes were stripped and the arguments were passed to the \n" + "run_command." ) diff --git a/smartsim/settings/launch_command.py b/smartsim/settings/launch_command.py index b848e35e1f..595ae0e6aa 100644 --- a/smartsim/settings/launch_command.py +++ b/smartsim/settings/launch_command.py @@ -30,12 +30,12 @@ class LauncherType(Enum): """Launchers supported by SmartSim.""" - Dragon = "dragon" - Slurm = "slurm" - Pals = "pals" - Alps = "alps" - Local = "local" - Mpiexec = "mpiexec" - Mpirun = "mpirun" - Orterun = "orterun" - Lsf = "lsf" + DRAGON = "dragon" + SLURM = "slurm" + PALS = "pals" + ALPS = "alps" + LOCAL = "local" + MPIEXEC = "mpiexec" + MPIRUN = "mpirun" + ORTERUN = "orterun" + LSF = "lsf" diff --git a/smartsim/settings/launch_settings.py b/smartsim/settings/launch_settings.py index 7b60830228..136de7638b 100644 --- a/smartsim/settings/launch_settings.py +++ b/smartsim/settings/launch_settings.py @@ -114,8 +114,8 @@ def __init__( :param launcher: The type of launcher to initialize (e.g., Dragon, Slurm, PALS, ALPS, Local, Mpiexec, Mpirun, Orterun, LSF) :param launch_args: A dictionary of arguments for the launcher, where the keys - are strings and the values can be either strings or None. This argument is optional - and defaults to None. + are strings and the values can be either strings or None. This argument is + optional and defaults to None. :param env_vars: Environment variables for the launch settings, where the keys are strings and the values can be either strings or None. This argument is also optional and defaults to None. @@ -124,8 +124,8 @@ def __init__( try: self._launcher = LauncherType(launcher) """The launcher type""" - except ValueError: - raise ValueError(f"Invalid launcher type: {launcher}") + except ValueError as exc: + raise ValueError(f"Invalid launcher type: {launcher}") from exc self._arguments = self._get_arguments(launch_args) """The LaunchSettings child class based on launcher type""" self.env_vars = env_vars or {} @@ -176,26 +176,25 @@ def _get_arguments(self, launch_args: StringArgument | None) -> LaunchArguments: :returns: The appropriate type for the settings instance. :raises ValueError: An invalid launcher type was provided. """ - if self._launcher == LauncherType.Slurm: + if self._launcher == LauncherType.SLURM: return SlurmLaunchArguments(launch_args) - elif self._launcher == LauncherType.Mpiexec: + if self._launcher == LauncherType.MPIEXEC: return MpiexecLaunchArguments(launch_args) - elif self._launcher == LauncherType.Mpirun: + if self._launcher == LauncherType.MPIRUN: return MpirunLaunchArguments(launch_args) - elif self._launcher == LauncherType.Orterun: + if self._launcher == LauncherType.ORTERUN: return OrterunLaunchArguments(launch_args) - elif self._launcher == LauncherType.Alps: + if self._launcher == LauncherType.ALPS: return AprunLaunchArguments(launch_args) - elif self._launcher == LauncherType.Lsf: + if self._launcher == LauncherType.LSF: return JsrunLaunchArguments(launch_args) - elif self._launcher == LauncherType.Pals: + if self._launcher == LauncherType.PALS: return PalsMpiexecLaunchArguments(launch_args) - elif self._launcher == LauncherType.Dragon: + if self._launcher == LauncherType.DRAGON: return DragonLaunchArguments(launch_args) - elif self._launcher == LauncherType.Local: + if self._launcher == LauncherType.LOCAL: return LocalLaunchArguments(launch_args) - else: - raise ValueError(f"Invalid launcher type: {self._launcher}") + raise ValueError(f"Invalid launcher type: {self._launcher}") def update_env(self, env_vars: t.Dict[str, str | None]) -> None: """Update the job environment variables diff --git a/smartsim/settings/sge_settings.py b/smartsim/settings/sge_settings.py index a5cd3f2b0f..dbe9ce3b7b 100644 --- a/smartsim/settings/sge_settings.py +++ b/smartsim/settings/sge_settings.py @@ -26,12 +26,19 @@ import typing as t +from smartsim.settings.batch_settings import BatchSettings + from ..error import LauncherUnsupportedFeature, SSConfigError from ..log import get_logger -from .base import BatchSettings logger = get_logger(__name__) +# *************************************** +# TODO: Remove pylint disable after merge +# *************************************** +# pylint: disable=no-member +# TODO: 'BsubBatchArguments','SlurmBatchArguments', +# 'QsubBatchArguments' has no 'items' member class SgeQsubBatchSettings(BatchSettings): def __init__( @@ -55,7 +62,10 @@ def __init__( :param resources: overrides for resource arguments :param batch_args: overrides for SGE batch arguments """ - + if batch_args is None: + batch_args = {} + self.account = account + self.time = time if "nodes" in kwargs: kwargs["nodes"] = 0 @@ -70,8 +80,6 @@ def __init__( super().__init__( "qsub", batch_args=batch_args, - account=account, - time=time, **kwargs, ) @@ -87,12 +95,14 @@ def resources(self, resources: t.Dict[str, t.Union[str, int]]) -> None: self._sanity_check_resources(resources) self._resources = resources.copy() - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: + @staticmethod + def set_hostlist(host_list: t.Union[str, t.List[str]]) -> None: raise LauncherUnsupportedFeature( "SGE does not support requesting specific hosts in batch jobs" ) - def set_queue(self, queue: str) -> None: + @staticmethod + def set_queue(queue: str) -> None: raise LauncherUnsupportedFeature("SGE does not support specifying queues") def set_shebang(self, shebang: str) -> None: @@ -117,7 +127,8 @@ def set_walltime(self, walltime: str) -> None: if walltime: self.set_resource("h_rt", walltime) - def set_nodes(self, num_nodes: t.Optional[int]) -> None: + @staticmethod + def set_nodes(num_nodes: t.Optional[int]) -> None: """Set the number of nodes, invalid for SGE :param nodes: Number of nodes, any integer other than 0 is invalid diff --git a/tests/_legacy/install/test_builder.py b/tests/_legacy/install/test_builder.py index e0518a96d8..feaf7e54fe 100644 --- a/tests/_legacy/install/test_builder.py +++ b/tests/_legacy/install/test_builder.py @@ -1,404 +1,404 @@ -# # BSD 2-Clause License -# # -# # Copyright (c) 2021-2024, Hewlett Packard Enterprise -# # All rights reserved. -# # -# # Redistribution and use in source and binary forms, with or without -# # modification, are permitted provided that the following conditions are met: -# # -# # 1. Redistributions of source code must retain the above copyright notice, this -# # list of conditions and the following disclaimer. -# # -# # 2. Redistributions in binary form must reproduce the above copyright notice, -# # this list of conditions and the following disclaimer in the documentation -# # and/or other materials provided with the distribution. -# # -# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -# import functools -# import pathlib -# import textwrap -# import time - -# import pytest - -# import smartsim._core._install.builder as build -# from smartsim._core._install.buildenv import RedisAIVersion - -# # The tests in this file belong to the group_a group -# pytestmark = pytest.mark.group_a - -# RAI_VERSIONS = RedisAIVersion("1.2.7") - -# for_each_device = pytest.mark.parametrize( -# "device", [build.Device.CPU, build.Device.GPU] -# ) - -# _toggle_build_optional_backend = lambda backend: pytest.mark.parametrize( -# f"build_{backend}", -# [ -# pytest.param(switch, id=f"with{'' if switch else 'out'}-{backend}") -# for switch in (True, False) -# ], -# ) -# toggle_build_tf = _toggle_build_optional_backend("tf") -# toggle_build_pt = _toggle_build_optional_backend("pt") -# toggle_build_ort = _toggle_build_optional_backend("ort") - - -# @pytest.mark.parametrize( -# "mock_os", [pytest.param(os_, id=f"os='{os_}'") for os_ in ("Windows", "Java", "")] -# ) -# def test_os_enum_raises_on_unsupported(mock_os): -# with pytest.raises(build.BuildError, match="operating system") as err_info: -# build.OperatingSystem.from_str(mock_os) - - -# @pytest.mark.parametrize( -# "mock_arch", -# [ -# pytest.param(arch_, id=f"arch='{arch_}'") -# for arch_ in ("i386", "i686", "i86pc", "aarch64", "armv7l", "") -# ], -# ) -# def test_arch_enum_raises_on_unsupported(mock_arch): -# with pytest.raises(build.BuildError, match="architecture"): -# build.Architecture.from_str(mock_arch) - - -# @pytest.fixture -# def p_test_dir(test_dir): -# yield pathlib.Path(test_dir).resolve() - - -# @for_each_device -# def test_rai_builder_raises_if_attempting_to_place_deps_when_build_dir_dne( -# monkeypatch, p_test_dir, device -# ): -# monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) -# monkeypatch.setattr( -# build.RedisAIBuilder, -# "rai_build_path", -# property(lambda self: p_test_dir / "path/to/dir/that/dne"), -# ) -# rai_builder = build.RedisAIBuilder() -# with pytest.raises(build.BuildError, match=r"build directory not found"): -# rai_builder._fetch_deps_for(device) - - -# @for_each_device -# def test_rai_builder_raises_if_attempting_to_place_deps_in_nonempty_dir( -# monkeypatch, p_test_dir, device -# ): -# (p_test_dir / "some_file.txt").touch() -# monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) -# monkeypatch.setattr( -# build.RedisAIBuilder, "rai_build_path", property(lambda self: p_test_dir) -# ) -# monkeypatch.setattr( -# build.RedisAIBuilder, "get_deps_dir_path_for", lambda *a, **kw: p_test_dir -# ) -# rai_builder = build.RedisAIBuilder() - -# with pytest.raises(build.BuildError, match=r"is not empty"): -# rai_builder._fetch_deps_for(device) - - -# invalid_build_arm64 = [ -# dict(build_tf=True, build_onnx=True), -# dict(build_tf=False, build_onnx=True), -# dict(build_tf=True, build_onnx=False), -# ] -# invalid_build_ids = [ -# ",".join([f"{key}={value}" for key, value in d.items()]) -# for d in invalid_build_arm64 -# ] - - -# @pytest.mark.parametrize("build_options", invalid_build_arm64, ids=invalid_build_ids) -# def test_rai_builder_raises_if_unsupported_deps_on_arm64(build_options): -# with pytest.raises(build.BuildError, match=r"are not supported on.*ARM64"): -# build.RedisAIBuilder( -# _os=build.OperatingSystem.DARWIN, -# architecture=build.Architecture.ARM64, -# **build_options, -# ) - - -# def _confirm_inst_presence(type_, should_be_present, seq): -# expected_num_occurrences = 1 if should_be_present else 0 -# occurrences = filter(lambda item: isinstance(item, type_), seq) -# return expected_num_occurrences == len(tuple(occurrences)) - - -# # Helper functions to check for the presence (or absence) of a -# # ``_RAIBuildDependency`` dependency in a list of dependencies that need to be -# # fetched by a ``RedisAIBuilder`` instance -# dlpack_dep_presence = functools.partial( -# _confirm_inst_presence, build._DLPackRepository, True -# ) -# pt_dep_presence = functools.partial(_confirm_inst_presence, build._PTArchive) -# tf_dep_presence = functools.partial(_confirm_inst_presence, build._TFArchive) -# ort_dep_presence = functools.partial(_confirm_inst_presence, build._ORTArchive) - - -# @for_each_device -# @toggle_build_tf -# @toggle_build_pt -# @toggle_build_ort -# def test_rai_builder_will_add_dep_if_backend_requested_wo_duplicates( -# monkeypatch, device, build_tf, build_pt, build_ort -# ): -# monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) - -# rai_builder = build.RedisAIBuilder( -# build_tf=build_tf, build_torch=build_pt, build_onnx=build_ort -# ) -# requested_backends = rai_builder._get_deps_to_fetch_for(build.Device(device)) -# assert dlpack_dep_presence(requested_backends) -# assert tf_dep_presence(build_tf, requested_backends) -# assert pt_dep_presence(build_pt, requested_backends) -# assert ort_dep_presence(build_ort, requested_backends) - - -# @for_each_device -# @toggle_build_tf -# @toggle_build_pt -# def test_rai_builder_will_not_add_dep_if_custom_dep_path_provided( -# monkeypatch, device, p_test_dir, build_tf, build_pt -# ): -# monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) -# mock_ml_lib = p_test_dir / "some/ml/lib" -# mock_ml_lib.mkdir(parents=True) -# rai_builder = build.RedisAIBuilder( -# build_tf=build_tf, -# build_torch=build_pt, -# build_onnx=False, -# libtf_dir=str(mock_ml_lib if build_tf else ""), -# torch_dir=str(mock_ml_lib if build_pt else ""), -# ) -# requested_backends = rai_builder._get_deps_to_fetch_for(device) -# assert dlpack_dep_presence(requested_backends) -# assert tf_dep_presence(False, requested_backends) -# assert pt_dep_presence(False, requested_backends) -# assert ort_dep_presence(False, requested_backends) -# assert len(requested_backends) == 1 - - -# def test_rai_builder_raises_if_it_fetches_an_unexpected_number_of_ml_deps( -# monkeypatch, p_test_dir -# ): -# monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) -# monkeypatch.setattr( -# build.RedisAIBuilder, "rai_build_path", property(lambda self: p_test_dir) -# ) -# monkeypatch.setattr( -# build, -# "_place_rai_dep_at", -# lambda target, verbose: lambda dep: target -# / "whoops_all_ml_deps_extract_to_a_dir_with_this_name", -# ) -# rai_builder = build.RedisAIBuilder(build_tf=True, build_torch=True, build_onnx=True) -# with pytest.raises( -# build.BuildError, -# match=r"Expected to place \d+ dependencies, but only found \d+", -# ): -# rai_builder._fetch_deps_for(build.Device.CPU) - - -# def test_threaded_map(): -# def _some_io_op(x): -# return x * x - -# assert (0, 1, 4, 9, 16) == tuple(build._threaded_map(_some_io_op, range(5))) - - -# def test_threaded_map_returns_early_if_nothing_to_map(): -# sleep_duration = 60 - -# def _some_long_io_op(_): -# time.sleep(sleep_duration) - -# start = time.time() -# build._threaded_map(_some_long_io_op, []) -# end = time.time() -# assert end - start < sleep_duration - - -# def test_correct_pt_variant_os(): -# # Check that all Linux variants return Linux -# for linux_variant in build.OperatingSystem.LINUX.value: -# os_ = build.OperatingSystem.from_str(linux_variant) -# assert build._choose_pt_variant(os_) == build._PTArchiveLinux - -# # Check that ARM64 and X86_64 Mac OSX return the Mac variant -# all_archs = (build.Architecture.ARM64, build.Architecture.X64) -# for arch in all_archs: -# os_ = build.OperatingSystem.DARWIN -# assert build._choose_pt_variant(os_) == build._PTArchiveMacOSX - - -# def test_PTArchiveMacOSX_url(): -# arch = build.Architecture.X64 -# pt_version = RAI_VERSIONS.torch - -# pt_linux_cpu = build._PTArchiveLinux( -# build.Architecture.X64, build.Device.CPU, pt_version, False -# ) -# x64_prefix = "https://download.pytorch.org/libtorch/" -# assert x64_prefix in pt_linux_cpu.url - -# pt_macosx_cpu = build._PTArchiveMacOSX( -# build.Architecture.ARM64, build.Device.CPU, pt_version, False -# ) -# arm64_prefix = "https://github.com/CrayLabs/ml_lib_builder/releases/download/" -# assert arm64_prefix in pt_macosx_cpu.url - - -# def test_PTArchiveMacOSX_gpu_error(): -# with pytest.raises(build.BuildError, match="support GPU on Mac OSX"): -# build._PTArchiveMacOSX( -# build.Architecture.ARM64, build.Device.GPU, RAI_VERSIONS.torch, False -# ).url - - -# def test_valid_platforms(): -# assert build.RedisAIBuilder( -# _os=build.OperatingSystem.LINUX, -# architecture=build.Architecture.X64, -# build_tf=True, -# build_torch=True, -# build_onnx=True, -# ) -# assert build.RedisAIBuilder( -# _os=build.OperatingSystem.DARWIN, -# architecture=build.Architecture.X64, -# build_tf=True, -# build_torch=True, -# build_onnx=False, -# ) -# assert build.RedisAIBuilder( -# _os=build.OperatingSystem.DARWIN, -# architecture=build.Architecture.X64, -# build_tf=False, -# build_torch=True, -# build_onnx=False, -# ) - - -# @pytest.mark.parametrize( -# "plat,cmd,expected_cmd", -# [ -# # Bare Word -# pytest.param( -# build.Platform(build.OperatingSystem.LINUX, build.Architecture.X64), -# ["git", "clone", "my-repo"], -# ["git", "clone", "my-repo"], -# id="git-Linux-X64", -# ), -# pytest.param( -# build.Platform(build.OperatingSystem.LINUX, build.Architecture.ARM64), -# ["git", "clone", "my-repo"], -# ["git", "clone", "my-repo"], -# id="git-Linux-Arm64", -# ), -# pytest.param( -# build.Platform(build.OperatingSystem.DARWIN, build.Architecture.X64), -# ["git", "clone", "my-repo"], -# ["git", "clone", "my-repo"], -# id="git-Darwin-X64", -# ), -# pytest.param( -# build.Platform(build.OperatingSystem.DARWIN, build.Architecture.ARM64), -# ["git", "clone", "my-repo"], -# [ -# "git", -# "clone", -# "--config", -# "core.autocrlf=false", -# "--config", -# "core.eol=lf", -# "my-repo", -# ], -# id="git-Darwin-Arm64", -# ), -# # Abs path -# pytest.param( -# build.Platform(build.OperatingSystem.LINUX, build.Architecture.X64), -# ["/path/to/git", "clone", "my-repo"], -# ["/path/to/git", "clone", "my-repo"], -# id="Abs-Linux-X64", -# ), -# pytest.param( -# build.Platform(build.OperatingSystem.LINUX, build.Architecture.ARM64), -# ["/path/to/git", "clone", "my-repo"], -# ["/path/to/git", "clone", "my-repo"], -# id="Abs-Linux-Arm64", -# ), -# pytest.param( -# build.Platform(build.OperatingSystem.DARWIN, build.Architecture.X64), -# ["/path/to/git", "clone", "my-repo"], -# ["/path/to/git", "clone", "my-repo"], -# id="Abs-Darwin-X64", -# ), -# pytest.param( -# build.Platform(build.OperatingSystem.DARWIN, build.Architecture.ARM64), -# ["/path/to/git", "clone", "my-repo"], -# [ -# "/path/to/git", -# "clone", -# "--config", -# "core.autocrlf=false", -# "--config", -# "core.eol=lf", -# "my-repo", -# ], -# id="Abs-Darwin-Arm64", -# ), -# ], -# ) -# def test_git_commands_are_configered_correctly_for_platforms(plat, cmd, expected_cmd): -# assert build.config_git_command(plat, cmd) == expected_cmd - - -# def test_modify_source_files(p_test_dir): -# def make_text_blurb(food): -# return textwrap.dedent(f"""\ -# My favorite food is {food} -# {food} is an important part of a healthy breakfast -# {food} {food} {food} {food} -# This line should be unchanged! -# --> {food} <-- -# """) - -# original_word = "SPAM" -# mutated_word = "EGGS" - -# source_files = [] -# for i in range(3): -# source_file = p_test_dir / f"test_{i}" -# source_file.touch() -# source_file.write_text(make_text_blurb(original_word)) -# source_files.append(source_file) -# # Modify a single file -# build._modify_source_files(source_files[0], original_word, mutated_word) -# assert source_files[0].read_text() == make_text_blurb(mutated_word) -# assert source_files[1].read_text() == make_text_blurb(original_word) -# assert source_files[2].read_text() == make_text_blurb(original_word) - -# # Modify multiple files -# build._modify_source_files( -# (source_files[1], source_files[2]), original_word, mutated_word -# ) -# assert source_files[1].read_text() == make_text_blurb(mutated_word) -# assert source_files[2].read_text() == make_text_blurb(mutated_word) +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import functools +import pathlib +import textwrap +import time + +import pytest + +import smartsim._core._install.builder as build +from smartsim._core._install.buildenv import RedisAIVersion + +# The tests in this file belong to the group_a group +pytestmark = pytest.mark.group_a + +RAI_VERSIONS = RedisAIVersion("1.2.7") + +for_each_device = pytest.mark.parametrize( + "device", [build.Device.CPU, build.Device.GPU] +) + +_toggle_build_optional_backend = lambda backend: pytest.mark.parametrize( + f"build_{backend}", + [ + pytest.param(switch, id=f"with{'' if switch else 'out'}-{backend}") + for switch in (True, False) + ], +) +toggle_build_tf = _toggle_build_optional_backend("tf") +toggle_build_pt = _toggle_build_optional_backend("pt") +toggle_build_ort = _toggle_build_optional_backend("ort") + + +@pytest.mark.parametrize( + "mock_os", [pytest.param(os_, id=f"os='{os_}'") for os_ in ("Windows", "Java", "")] +) +def test_os_enum_raises_on_unsupported(mock_os): + with pytest.raises(build.BuildError, match="operating system") as err_info: + build.OperatingSystem.from_str(mock_os) + + +@pytest.mark.parametrize( + "mock_arch", + [ + pytest.param(arch_, id=f"arch='{arch_}'") + for arch_ in ("i386", "i686", "i86pc", "aarch64", "armv7l", "") + ], +) +def test_arch_enum_raises_on_unsupported(mock_arch): + with pytest.raises(build.BuildError, match="architecture"): + build.Architecture.from_str(mock_arch) + + +@pytest.fixture +def p_test_dir(test_dir): + yield pathlib.Path(test_dir).resolve() + + +@for_each_device +def test_rai_builder_raises_if_attempting_to_place_deps_when_build_dir_dne( + monkeypatch, p_test_dir, device +): + monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) + monkeypatch.setattr( + build.RedisAIBuilder, + "rai_build_path", + property(lambda self: p_test_dir / "path/to/dir/that/dne"), + ) + rai_builder = build.RedisAIBuilder() + with pytest.raises(build.BuildError, match=r"build directory not found"): + rai_builder._fetch_deps_for(device) + + +@for_each_device +def test_rai_builder_raises_if_attempting_to_place_deps_in_nonempty_dir( + monkeypatch, p_test_dir, device +): + (p_test_dir / "some_file.txt").touch() + monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) + monkeypatch.setattr( + build.RedisAIBuilder, "rai_build_path", property(lambda self: p_test_dir) + ) + monkeypatch.setattr( + build.RedisAIBuilder, "get_deps_dir_path_for", lambda *a, **kw: p_test_dir + ) + rai_builder = build.RedisAIBuilder() + + with pytest.raises(build.BuildError, match=r"is not empty"): + rai_builder._fetch_deps_for(device) + + +invalid_build_arm64 = [ + dict(build_tf=True, build_onnx=True), + dict(build_tf=False, build_onnx=True), + dict(build_tf=True, build_onnx=False), +] +invalid_build_ids = [ + ",".join([f"{key}={value}" for key, value in d.items()]) + for d in invalid_build_arm64 +] + + +@pytest.mark.parametrize("build_options", invalid_build_arm64, ids=invalid_build_ids) +def test_rai_builder_raises_if_unsupported_deps_on_arm64(build_options): + with pytest.raises(build.BuildError, match=r"are not supported on.*ARM64"): + build.RedisAIBuilder( + _os=build.OperatingSystem.DARWIN, + architecture=build.Architecture.ARM64, + **build_options, + ) + + +def _confirm_inst_presence(type_, should_be_present, seq): + expected_num_occurrences = 1 if should_be_present else 0 + occurrences = filter(lambda item: isinstance(item, type_), seq) + return expected_num_occurrences == len(tuple(occurrences)) + + +# Helper functions to check for the presence (or absence) of a +# ``_RAIBuildDependency`` dependency in a list of dependencies that need to be +# fetched by a ``RedisAIBuilder`` instance +dlpack_dep_presence = functools.partial( + _confirm_inst_presence, build._DLPackRepository, True +) +pt_dep_presence = functools.partial(_confirm_inst_presence, build._PTArchive) +tf_dep_presence = functools.partial(_confirm_inst_presence, build._TFArchive) +ort_dep_presence = functools.partial(_confirm_inst_presence, build._ORTArchive) + + +@for_each_device +@toggle_build_tf +@toggle_build_pt +@toggle_build_ort +def test_rai_builder_will_add_dep_if_backend_requested_wo_duplicates( + monkeypatch, device, build_tf, build_pt, build_ort +): + monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) + + rai_builder = build.RedisAIBuilder( + build_tf=build_tf, build_torch=build_pt, build_onnx=build_ort + ) + requested_backends = rai_builder._get_deps_to_fetch_for(build.Device(device)) + assert dlpack_dep_presence(requested_backends) + assert tf_dep_presence(build_tf, requested_backends) + assert pt_dep_presence(build_pt, requested_backends) + assert ort_dep_presence(build_ort, requested_backends) + + +@for_each_device +@toggle_build_tf +@toggle_build_pt +def test_rai_builder_will_not_add_dep_if_custom_dep_path_provided( + monkeypatch, device, p_test_dir, build_tf, build_pt +): + monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) + mock_ml_lib = p_test_dir / "some/ml/lib" + mock_ml_lib.mkdir(parents=True) + rai_builder = build.RedisAIBuilder( + build_tf=build_tf, + build_torch=build_pt, + build_onnx=False, + libtf_dir=str(mock_ml_lib if build_tf else ""), + torch_dir=str(mock_ml_lib if build_pt else ""), + ) + requested_backends = rai_builder._get_deps_to_fetch_for(device) + assert dlpack_dep_presence(requested_backends) + assert tf_dep_presence(False, requested_backends) + assert pt_dep_presence(False, requested_backends) + assert ort_dep_presence(False, requested_backends) + assert len(requested_backends) == 1 + + +def test_rai_builder_raises_if_it_fetches_an_unexpected_number_of_ml_deps( + monkeypatch, p_test_dir +): + monkeypatch.setattr(build.RedisAIBuilder, "_validate_platform", lambda a: None) + monkeypatch.setattr( + build.RedisAIBuilder, "rai_build_path", property(lambda self: p_test_dir) + ) + monkeypatch.setattr( + build, + "_place_rai_dep_at", + lambda target, verbose: lambda dep: target + / "whoops_all_ml_deps_extract_to_a_dir_with_this_name", + ) + rai_builder = build.RedisAIBuilder(build_tf=True, build_torch=True, build_onnx=True) + with pytest.raises( + build.BuildError, + match=r"Expected to place \d+ dependencies, but only found \d+", + ): + rai_builder._fetch_deps_for(build.Device.CPU) + + +def test_threaded_map(): + def _some_io_op(x): + return x * x + + assert (0, 1, 4, 9, 16) == tuple(build._threaded_map(_some_io_op, range(5))) + + +def test_threaded_map_returns_early_if_nothing_to_map(): + sleep_duration = 60 + + def _some_long_io_op(_): + time.sleep(sleep_duration) + + start = time.time() + build._threaded_map(_some_long_io_op, []) + end = time.time() + assert end - start < sleep_duration + + +def test_correct_pt_variant_os(): + # Check that all Linux variants return Linux + for linux_variant in build.OperatingSystem.LINUX.value: + os_ = build.OperatingSystem.from_str(linux_variant) + assert build._choose_pt_variant(os_) == build._PTArchiveLinux + + # Check that ARM64 and X86_64 Mac OSX return the Mac variant + all_archs = (build.Architecture.ARM64, build.Architecture.X64) + for arch in all_archs: + os_ = build.OperatingSystem.DARWIN + assert build._choose_pt_variant(os_) == build._PTArchiveMacOSX + + +def test_PTArchiveMacOSX_url(): + arch = build.Architecture.X64 + pt_version = RAI_VERSIONS.torch + + pt_linux_cpu = build._PTArchiveLinux( + build.Architecture.X64, build.Device.CPU, pt_version, False + ) + x64_prefix = "https://download.pytorch.org/libtorch/" + assert x64_prefix in pt_linux_cpu.url + + pt_macosx_cpu = build._PTArchiveMacOSX( + build.Architecture.ARM64, build.Device.CPU, pt_version, False + ) + arm64_prefix = "https://github.com/CrayLabs/ml_lib_builder/releases/download/" + assert arm64_prefix in pt_macosx_cpu.url + + +def test_PTArchiveMacOSX_gpu_error(): + with pytest.raises(build.BuildError, match="support GPU on Mac OSX"): + build._PTArchiveMacOSX( + build.Architecture.ARM64, build.Device.GPU, RAI_VERSIONS.torch, False + ).url + + +def test_valid_platforms(): + assert build.RedisAIBuilder( + _os=build.OperatingSystem.LINUX, + architecture=build.Architecture.X64, + build_tf=True, + build_torch=True, + build_onnx=True, + ) + assert build.RedisAIBuilder( + _os=build.OperatingSystem.DARWIN, + architecture=build.Architecture.X64, + build_tf=True, + build_torch=True, + build_onnx=False, + ) + assert build.RedisAIBuilder( + _os=build.OperatingSystem.DARWIN, + architecture=build.Architecture.X64, + build_tf=False, + build_torch=True, + build_onnx=False, + ) + + +@pytest.mark.parametrize( + "plat,cmd,expected_cmd", + [ + # Bare Word + pytest.param( + build.Platform(build.OperatingSystem.LINUX, build.Architecture.X64), + ["git", "clone", "my-repo"], + ["git", "clone", "my-repo"], + id="git-Linux-X64", + ), + pytest.param( + build.Platform(build.OperatingSystem.LINUX, build.Architecture.ARM64), + ["git", "clone", "my-repo"], + ["git", "clone", "my-repo"], + id="git-Linux-Arm64", + ), + pytest.param( + build.Platform(build.OperatingSystem.DARWIN, build.Architecture.X64), + ["git", "clone", "my-repo"], + ["git", "clone", "my-repo"], + id="git-Darwin-X64", + ), + pytest.param( + build.Platform(build.OperatingSystem.DARWIN, build.Architecture.ARM64), + ["git", "clone", "my-repo"], + [ + "git", + "clone", + "--config", + "core.autocrlf=false", + "--config", + "core.eol=lf", + "my-repo", + ], + id="git-Darwin-Arm64", + ), + # Abs path + pytest.param( + build.Platform(build.OperatingSystem.LINUX, build.Architecture.X64), + ["/path/to/git", "clone", "my-repo"], + ["/path/to/git", "clone", "my-repo"], + id="Abs-Linux-X64", + ), + pytest.param( + build.Platform(build.OperatingSystem.LINUX, build.Architecture.ARM64), + ["/path/to/git", "clone", "my-repo"], + ["/path/to/git", "clone", "my-repo"], + id="Abs-Linux-Arm64", + ), + pytest.param( + build.Platform(build.OperatingSystem.DARWIN, build.Architecture.X64), + ["/path/to/git", "clone", "my-repo"], + ["/path/to/git", "clone", "my-repo"], + id="Abs-Darwin-X64", + ), + pytest.param( + build.Platform(build.OperatingSystem.DARWIN, build.Architecture.ARM64), + ["/path/to/git", "clone", "my-repo"], + [ + "/path/to/git", + "clone", + "--config", + "core.autocrlf=false", + "--config", + "core.eol=lf", + "my-repo", + ], + id="Abs-Darwin-Arm64", + ), + ], +) +def test_git_commands_are_configered_correctly_for_platforms(plat, cmd, expected_cmd): + assert build.config_git_command(plat, cmd) == expected_cmd + + +def test_modify_source_files(p_test_dir): + def make_text_blurb(food): + return textwrap.dedent(f"""\ + My favorite food is {food} + {food} is an important part of a healthy breakfast + {food} {food} {food} {food} + This line should be unchanged! + --> {food} <-- + """) + + original_word = "SPAM" + mutated_word = "EGGS" + + source_files = [] + for i in range(3): + source_file = p_test_dir / f"test_{i}" + source_file.touch() + source_file.write_text(make_text_blurb(original_word)) + source_files.append(source_file) + # Modify a single file + build._modify_source_files(source_files[0], original_word, mutated_word) + assert source_files[0].read_text() == make_text_blurb(mutated_word) + assert source_files[1].read_text() == make_text_blurb(original_word) + assert source_files[2].read_text() == make_text_blurb(original_word) + + # Modify multiple files + build._modify_source_files( + (source_files[1], source_files[2]), original_word, mutated_word + ) + assert source_files[1].read_text() == make_text_blurb(mutated_word) + assert source_files[2].read_text() == make_text_blurb(mutated_word) diff --git a/tests/_legacy/test_colo_model_local.py b/tests/_legacy/test_colo_model_local.py index 1ab97c4cc3..54848907d3 100644 --- a/tests/_legacy/test_colo_model_local.py +++ b/tests/_legacy/test_colo_model_local.py @@ -29,7 +29,7 @@ import pytest from smartsim import Experiment -from smartsim.entity import Application +from smartsim._core.utils.helpers import _create_pinning_string from smartsim.error import SSUnsupportedError from smartsim.status import JobStatus @@ -116,7 +116,7 @@ def test_unsupported_custom_pinning(fileutils, test_dir, coloutils, custom_pinni ], ) def test_create_pinning_string(pin_list, num_cpus, expected): - assert Application._create_pinning_string(pin_list, num_cpus) == expected + assert _create_pinning_string(pin_list, num_cpus) == expected @pytest.mark.parametrize("fs_type", supported_fss) diff --git a/tests/_legacy/test_controller.py b/tests/_legacy/test_controller.py index ad0c98fe88..dcb713cb0d 100644 --- a/tests/_legacy/test_controller.py +++ b/tests/_legacy/test_controller.py @@ -31,7 +31,7 @@ from smartsim._core.control.controller import Controller from smartsim._core.launcher.step import Step from smartsim.builders.ensemble import Ensemble -from smartsim.database.orchestrator import FeatureStore +from smartsim.database.feature_store import FeatureStore from smartsim.settings.slurmSettings import SbatchSettings, SrunSettings controller = Controller() diff --git a/tests/_legacy/test_dragon_run_request.py b/tests/_legacy/test_dragon_run_request.py index 94e7a5dd97..a1c1e495f3 100644 --- a/tests/_legacy/test_dragon_run_request.py +++ b/tests/_legacy/test_dragon_run_request.py @@ -626,7 +626,8 @@ def test_view(monkeypatch: pytest.MonkeyPatch) -> None: hosts = dragon_backend.hosts dragon_backend._prioritizer.increment(hosts[0]) - expected_msg = textwrap.dedent(f"""\ + expected_msg = textwrap.dedent( + f"""\ Dragon server backend update | Host | Status | |--------|----------| @@ -639,7 +640,8 @@ def test_view(monkeypatch: pytest.MonkeyPatch) -> None: | del999-2 | Cancelled | {hosts[1]} | -9 | 1 | | c101vz-3 | Completed | {hosts[1]},{hosts[2]} | 0 | 2 | | 0ghjk1-4 | Failed | {hosts[2]} | -1 | 1 | - | ljace0-5 | NeverStarted | | | 0 |""") + | ljace0-5 | NeverStarted | | | 0 |""" + ) # get rid of white space to make the comparison easier actual_msg = dragon_backend.status_message.replace(" ", "") diff --git a/tests/_legacy/test_dragon_runsettings.py b/tests/_legacy/test_dragon_runsettings.py index 8c7600c74c..34e8510e82 100644 --- a/tests/_legacy/test_dragon_runsettings.py +++ b/tests/_legacy/test_dragon_runsettings.py @@ -96,122 +96,3 @@ def test_dragon_runsettings_gpu_affinity(): # ensure the value is not changed when we extend the list rs.run_args["gpu-affinity"] = "7,8,9" assert rs.run_args["gpu-affinity"] != ",".join(str(val) for val in exp_value) - - -def test_dragon_runsettings_hostlist_null(): - """Verify that passing a null hostlist is treated as a failure""" - rs = DragonRunSettings(exe="sleep", exe_args=["1"]) - - # baseline check that no host list exists - stored_list = rs.run_args.get("host-list", None) - assert stored_list is None - - with pytest.raises(ValueError) as ex: - rs.set_hostlist(None) - - assert "empty hostlist" in ex.value.args[0] - - -def test_dragon_runsettings_hostlist_empty(): - """Verify that passing an empty hostlist is treated as a failure""" - rs = DragonRunSettings(exe="sleep", exe_args=["1"]) - - # baseline check that no host list exists - stored_list = rs.run_args.get("host-list", None) - assert stored_list is None - - with pytest.raises(ValueError) as ex: - rs.set_hostlist([]) - - assert "empty hostlist" in ex.value.args[0] - - -@pytest.mark.parametrize("hostlist_csv", [" ", " , , , ", ",", ",,,"]) -def test_dragon_runsettings_hostlist_whitespace_handling(hostlist_csv: str): - """Verify that passing a hostlist with emptystring host names is treated as a failure""" - rs = DragonRunSettings(exe="sleep", exe_args=["1"]) - - # baseline check that no host list exists - stored_list = rs.run_args.get("host-list", None) - assert stored_list is None - - # empty string as hostname in list - with pytest.raises(ValueError) as ex: - rs.set_hostlist(hostlist_csv) - - assert "invalid names" in ex.value.args[0] - - -@pytest.mark.parametrize( - "hostlist_csv", [[" "], [" ", "", " ", " "], ["", " "], ["", "", "", ""]] -) -def test_dragon_runsettings_hostlist_whitespace_handling_list(hostlist_csv: str): - """Verify that passing a hostlist with emptystring host names contained in a list - is treated as a failure""" - rs = DragonRunSettings(exe="sleep", exe_args=["1"]) - - # baseline check that no host list exists - stored_list = rs.run_args.get("host-list", None) - assert stored_list is None - - # empty string as hostname in list - with pytest.raises(ValueError) as ex: - rs.set_hostlist(hostlist_csv) - - assert "invalid names" in ex.value.args[0] - - -def test_dragon_runsettings_hostlist_as_csv(): - """Verify that a hostlist is stored properly when passing in a CSV string""" - rs = DragonRunSettings(exe="sleep", exe_args=["1"]) - - # baseline check that no host list exists - stored_list = rs.run_args.get("host-list", None) - assert stored_list is None - - hostnames = ["host0", "host1", "host2", "host3", "host4"] - - # set the host list with ideal comma separated values - input0 = ",".join(hostnames) - - # set the host list with a string of comma separated values - # including extra whitespace - input1 = ", ".join(hostnames) - - for hosts_input in [input0, input1]: - rs.set_hostlist(hosts_input) - - stored_list = rs.run_args.get("host-list", None) - assert stored_list - - # confirm that all values from the original list are retrieved - split_stored_list = stored_list.split(",") - assert set(hostnames) == set(split_stored_list) - - -def test_dragon_runsettings_hostlist_as_csv(): - """Verify that a hostlist is stored properly when passing in a CSV string""" - rs = DragonRunSettings(exe="sleep", exe_args=["1"]) - - # baseline check that no host list exists - stored_list = rs.run_args.get("host-list", None) - assert stored_list is None - - hostnames = ["host0", "host1", "host2", "host3", "host4"] - - # set the host list with ideal comma separated values - input0 = ",".join(hostnames) - - # set the host list with a string of comma separated values - # including extra whitespace - input1 = ", ".join(hostnames) - - for hosts_input in [input0, input1]: - rs.set_hostlist(hosts_input) - - stored_list = rs.run_args.get("host-list", None) - assert stored_list - - # confirm that all values from the original list are retrieved - split_stored_list = stored_list.split(",") - assert set(hostnames) == set(split_stored_list) diff --git a/tests/_legacy/test_output_files.py b/tests/_legacy/test_output_files.py index 55ecfd90a5..cf093b0b1c 100644 --- a/tests/_legacy/test_output_files.py +++ b/tests/_legacy/test_output_files.py @@ -34,7 +34,7 @@ from smartsim._core.control.controller import Controller, _AnonymousBatchJob from smartsim._core.launcher.step import Step from smartsim.builders.ensemble import Ensemble -from smartsim.database.orchestrator import FeatureStore +from smartsim.database.feature_store import FeatureStore from smartsim.entity.application import Application from smartsim.settings.base import RunSettings from smartsim.settings.slurmSettings import SbatchSettings, SrunSettings diff --git a/tests/_legacy/test_preview.py b/tests/_legacy/test_preview.py index 25a51671d0..6f029aab8f 100644 --- a/tests/_legacy/test_preview.py +++ b/tests/_legacy/test_preview.py @@ -983,7 +983,7 @@ def test_preview_active_infrastructure_feature_store_error( exp = Experiment(exp_name, exp_path=test_dir, launcher=test_launcher) monkeypatch.setattr( - smartsim.database.orchestrator.FeatureStore, "is_active", lambda x: True + smartsim.database.feature_store.FeatureStore, "is_active", lambda x: True ) orc = exp.create_feature_store( diff --git a/tests/_legacy/test_serialize.py b/tests/_legacy/test_serialize.py index eb56d75540..7fd596c2ba 100644 --- a/tests/_legacy/test_serialize.py +++ b/tests/_legacy/test_serialize.py @@ -36,7 +36,7 @@ from smartsim._core._cli import utils from smartsim._core.control.manifest import LaunchedManifestBuilder from smartsim._core.utils import serialize -from smartsim.database.orchestrator import FeatureStore +from smartsim.database.feature_store import FeatureStore _CFG_TM_ENABLED_ATTR = "telemetry_enabled" diff --git a/tests/_legacy/test_symlinking.py b/tests/_legacy/test_symlinking.py index 95aa187e6b..5892abd4c1 100644 --- a/tests/_legacy/test_symlinking.py +++ b/tests/_legacy/test_symlinking.py @@ -33,7 +33,7 @@ from smartsim._core.config import CONFIG from smartsim._core.control.controller import Controller, _AnonymousBatchJob from smartsim.builders.ensemble import Ensemble -from smartsim.database.orchestrator import FeatureStore +from smartsim.database.feature_store import FeatureStore from smartsim.entity.application import Application from smartsim.settings.base import RunSettings from smartsim.settings.slurmSettings import SbatchSettings, SrunSettings diff --git a/tests/dragon_wlm/test_reply_building.py b/tests/dragon_wlm/test_reply_building.py index 48493b3c4d..1b0074ca0e 100644 --- a/tests/dragon_wlm/test_reply_building.py +++ b/tests/dragon_wlm/test_reply_building.py @@ -24,6 +24,17 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +<<<<<<< HEAD:smartsim/entity/_mock.py +"""This module contains stubs of functionality that is not currently +implemented. + +THIS WHOLE MODULE SHOULD BE REMOVED IN FUTURE!! +""" + +from __future__ import annotations + +import typing as t +======= import typing as t import pytest @@ -34,10 +45,22 @@ if t.TYPE_CHECKING: from smartsim._core.mli.mli_schemas.response.response_capnp import Status +>>>>>>> 5bdafc5f93fd56bf94ca5a7979a28f185c7c7ebf:tests/dragon_wlm/test_reply_building.py # The tests in this file belong to the dragon group pytestmark = pytest.mark.dragon +<<<<<<< HEAD:smartsim/entity/_mock.py +class Mock: + """Base mock class""" + + def __init__(self, *_: t.Any, **__: t.Any): ... + def __getattr__(self, _: str) -> Mock: + return type(self)() + + def __deepcopy__(self, _: dict[t.Any, t.Any]) -> Mock: + return type(self)() +======= @pytest.mark.parametrize( "status, message", @@ -62,3 +85,4 @@ def test_build_failure_reply_fails(): response = build_failure_reply("not a status enum", "message") assert "Error assigning status to response" in ex.value.args[0] +>>>>>>> 5bdafc5f93fd56bf94ca5a7979a28f185c7c7ebf:tests/dragon_wlm/test_reply_building.py diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 45f3ecf8e5..57529ac815 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -40,7 +40,7 @@ from smartsim._core import dispatch from smartsim._core.control.launch_history import LaunchHistory -from smartsim._core.generation.generator import Job_Path +from smartsim._core.generation.generator import JobPath from smartsim._core.utils.launcher import LauncherProtocol, create_job_id from smartsim.builders.ensemble import Ensemble from smartsim.entity import entity @@ -72,7 +72,7 @@ def experiment(monkeypatch, test_dir, dispatcher): monkeypatch.setattr( exp, "_generate", - lambda generator, job, idx: Job_Path( + lambda generator, job, idx: JobPath( "/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt" ), )