diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 3a729d70e..3230d8da8 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -117,7 +117,7 @@ jobs: - name: Install SmartSim (with ML backends) run: | python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis - python -m pip install .[dev,ml] + python -m pip install .[dev,mypy,ml] - name: Install ML Runtimes with Smart (with pt, tf, and onnx support) if: contains( matrix.os, 'ubuntu' ) || contains( matrix.os, 'macos-12') @@ -129,7 +129,6 @@ jobs: - name: Run mypy run: | - python -m pip install .[mypy] make check-mypy # TODO: Re-enable static analysis once API is firmed up @@ -174,7 +173,7 @@ jobs: retention-days: 5 - name: Upload Pytest coverage to Codecov - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v4.5.0 with: fail_ci_if_error: false files: ./coverage.xml diff --git a/doc/changelog.md b/doc/changelog.md index 6efeedfaf..740197ce5 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -15,6 +15,9 @@ To be released at some future point in time Description +- Update codecov to 4.5.0 +- Remove build of Redis from setup.py +- Mitigate dependency installation issues - Fix internal host name representation for Dragon backend - Make dependencies more discoverable in setup.py - Add hardware pinning capability when using dragon @@ -27,6 +30,19 @@ Description Detailed Notes +- Update codecov to 4.5.0 to mitigate GitHub action failure + ([SmartSim-PR657](https://github.com/CrayLabs/SmartSim/pull/657)) +- The builder module was included in setup.py to allow us to ship the + main Redis binaries (not RedisAI) with installs from PyPI. To + allow easier maintenance of this file and enable future complexity + this has been removed. The Redis binaries will thus be built + by users during the `smart build` step +- Installation of mypy or dragon in separate build actions caused + some dependencies (typing_extensions, numpy) to be upgraded and + caused runtime failures. The build actions were tweaked to include + all optional dependencies to be considered by pip during resolution. + Additionally, the numpy version was capped on dragon installations. + ([SmartSim-PR653](https://github.com/CrayLabs/SmartSim/pull/653)) - setup.py used to define dependencies in a way that was not amenable to code scanning tools. Direct dependencies now appear directly in the setup call and the definition of the SmartRedis version diff --git a/setup.py b/setup.py index 4c2479f71..328bf1ffb 100644 --- a/setup.py +++ b/setup.py @@ -77,9 +77,6 @@ from pathlib import Path from setuptools import setup -from setuptools.command.build_py import build_py -from setuptools.command.install import install -from setuptools.dist import Distribution # Some necessary evils we have to do to be able to use # the _install tools in smartsim/smartsim/_core/_install @@ -95,12 +92,6 @@ buildenv = importlib.util.module_from_spec(buildenv_spec) buildenv_spec.loader.exec_module(buildenv) -# import builder module -builder_path = _install_dir.joinpath("builder.py") -builder_spec = importlib.util.spec_from_file_location("builder", str(builder_path)) -builder = importlib.util.module_from_spec(builder_spec) -builder_spec.loader.exec_module(builder) - # helper classes for building dependencies that are # also utilized by the Smart CLI build_env = buildenv.BuildEnv(checks=False) @@ -128,41 +119,7 @@ class BuildError(Exception): pass - -# Hacky workaround for solving CI build "purelib" issue -# see https://github.com/google/or-tools/issues/616 -class InstallPlatlib(install): - def finalize_options(self): - super().finalize_options() - if self.distribution.has_ext_modules(): - self.install_lib = self.install_platlib - - -class SmartSimBuild(build_py): - def run(self): - feature_store_builder = builder.FeatureStoreBuilder( - build_env(), build_env.MALLOC, build_env.JOBS - ) - if not feature_store_builder.is_built: - feature_store_builder.build_from_git(versions.REDIS_URL, versions.REDIS) - - feature_store_builder.cleanup() - - # run original build_py command - super().run() - - -# Tested with wheel v0.29.0 -class BinaryDistribution(Distribution): - """Distribution which always forces a binary package with platform name - - We use this because we want to pre-package Redis for certain - platforms to use. - """ - - def has_ext_modules(_placeholder): - return True - +# Define needed dependencies for the installation extras_require = { "dev": [ @@ -182,6 +139,7 @@ def has_ext_modules(_placeholder): "types-tqdm", "types-tensorflow==2.12.0.9", "types-setuptools", + "typing_extensions>=4.1.0", ], "docs": [ "Sphinx==6.2.1", @@ -225,15 +183,9 @@ def has_ext_modules(_placeholder): "pygithub>=2.3.0", "numpy<2", "smartredis>=0.5,<0.6", - "typing_extensions>=4.1.0,<4.6", ], - cmdclass={ - "build_py": SmartSimBuild, - "install": InstallPlatlib, - }, zip_safe=False, extras_require=extras_require, - distclass=BinaryDistribution, entry_points={ "console_scripts": [ "smart = smartsim._core._cli.__main__:main", diff --git a/smartsim/_core/_cli/scripts/dragon_install.py b/smartsim/_core/_cli/scripts/dragon_install.py index 466c390bd..a2e8ed36f 100644 --- a/smartsim/_core/_cli/scripts/dragon_install.py +++ b/smartsim/_core/_cli/scripts/dragon_install.py @@ -182,7 +182,7 @@ def install_package(asset_dir: pathlib.Path) -> int: logger.info(f"Installing package: {wheel_path.absolute()}") try: - pip("install", "--force-reinstall", str(wheel_path)) + pip("install", "--force-reinstall", str(wheel_path), "numpy<2") wheel_path = next(wheels, None) except Exception: logger.error(f"Unable to install from {asset_dir}") diff --git a/smartsim/_core/control/launch_history.py b/smartsim/_core/control/launch_history.py index 28b8a6f86..b8b9f4c7e 100644 --- a/smartsim/_core/control/launch_history.py +++ b/smartsim/_core/control/launch_history.py @@ -32,7 +32,7 @@ from smartsim._core.utils import helpers as _helpers if t.TYPE_CHECKING: - from smartsim.settings.dispatch import LauncherProtocol + from smartsim._core.dispatch import LauncherProtocol from smartsim.types import LaunchedJobID diff --git a/smartsim/settings/dispatch.py b/smartsim/_core/dispatch.py similarity index 100% rename from smartsim/settings/dispatch.py rename to smartsim/_core/dispatch.py diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index e71e694ee..1d2b94423 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -356,12 +356,13 @@ def _assert_schema_type(obj: object, typ: t.Type[_SchemaT], /) -> _SchemaT: return obj +from smartsim._core.dispatch import ExecutableProtocol, dispatch + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # TODO: Remove this registry and move back to builder file after fixing # circular import caused by `DragonLauncher.supported_rs` # ----------------------------------------------------------------------------- from smartsim.settings.arguments.launch.dragon import DragonLaunchArguments -from smartsim.settings.dispatch import ExecutableProtocol, dispatch def _as_run_request_args_and_policy( diff --git a/smartsim/experiment.py b/smartsim/experiment.py index 50b06cabd..00e25ba6d 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -40,10 +40,10 @@ from tabulate import tabulate +from smartsim._core import dispatch from smartsim._core.config import CONFIG from smartsim._core.control.launch_history import LaunchHistory as _LaunchHistory from smartsim.error import errors -from smartsim.settings import dispatch from smartsim.status import InvalidJobStatus, JobStatus from ._core import Controller, Generator, Manifest, previewrenderer @@ -59,8 +59,8 @@ from .log import ctx_exp_path, get_logger, method_contextualizer if t.TYPE_CHECKING: + from smartsim._core.dispatch import ExecutableProtocol from smartsim.launchable.job import Job - from smartsim.settings.dispatch import ExecutableProtocol from smartsim.types import LaunchedJobID logger = get_logger(__name__) diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index 4c29c3c9f..a433319ac 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -26,18 +26,23 @@ from __future__ import annotations +import os import typing as t from copy import deepcopy from smartsim._core.commands.launchCommands import LaunchCommands from smartsim._core.utils.helpers import check_name from smartsim.launchable.basejob import BaseJob +from smartsim.log import get_logger from smartsim.settings import LaunchSettings +logger = get_logger(__name__) + if t.TYPE_CHECKING: from smartsim.entity.entity import SmartSimEntity +@t.final class Job(BaseJob): """A Job holds a reference to a SmartSimEntity and associated LaunchSettings prior to launch. It is responsible for turning @@ -65,10 +70,11 @@ def name(self) -> str: return self._name @name.setter - def name(self, name: str | None) -> None: + def name(self, name: str) -> None: """Sets the name of the Job.""" - self._name = name if name else self._entity.name - check_name(self._name) + check_name(name) + logger.debug(f'Overwriting the Job name from "{self._name}" to "{name}"') + self._name = name @property def entity(self) -> SmartSimEntity: diff --git a/smartsim/launchable/jobGroup.py b/smartsim/launchable/jobGroup.py index d2e64c454..3de767711 100644 --- a/smartsim/launchable/jobGroup.py +++ b/smartsim/launchable/jobGroup.py @@ -29,14 +29,19 @@ import typing as t from copy import deepcopy +from smartsim.log import get_logger + from .._core.utils.helpers import check_name from .basejob import BaseJob from .baseJobGroup import BaseJobGroup +logger = get_logger(__name__) + if t.TYPE_CHECKING: from typing_extensions import Self +@t.final class JobGroup(BaseJobGroup): """A job group holds references to multiple jobs that will be executed all at the same time when resources @@ -62,6 +67,7 @@ def name(self) -> str: def name(self, name: str) -> None: """Sets the name of the JobGroup.""" check_name(name) + logger.debug(f'Overwriting Job name from "{self._name}" to "{name}"') self._name = name @property diff --git a/smartsim/settings/arguments/launch/alps.py b/smartsim/settings/arguments/launch/alps.py index 1879dd102..6375a4141 100644 --- a/smartsim/settings/arguments/launch/alps.py +++ b/smartsim/settings/arguments/launch/alps.py @@ -28,8 +28,8 @@ import typing as t +from smartsim._core.dispatch import ShellLauncher, dispatch, make_shell_format_fn from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/arguments/launch/local.py b/smartsim/settings/arguments/launch/local.py index 0bbba2584..97b300bce 100644 --- a/smartsim/settings/arguments/launch/local.py +++ b/smartsim/settings/arguments/launch/local.py @@ -28,8 +28,8 @@ import typing as t +from smartsim._core.dispatch import ShellLauncher, dispatch, make_shell_format_fn from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/arguments/launch/lsf.py b/smartsim/settings/arguments/launch/lsf.py index e99c39af7..3696ac5bc 100644 --- a/smartsim/settings/arguments/launch/lsf.py +++ b/smartsim/settings/arguments/launch/lsf.py @@ -30,14 +30,15 @@ import subprocess import typing as t -from smartsim.log import get_logger -from smartsim.settings.dispatch import ( +from smartsim._core.dispatch import ( ExecutableProtocol, ShellLauncher, ShellLauncherCommand, _EnvironMappingType, dispatch, + make_shell_format_fn, ) +from smartsim.log import get_logger from ...common import set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/arguments/launch/mpi.py b/smartsim/settings/arguments/launch/mpi.py index 85fd38145..04ae55b57 100644 --- a/smartsim/settings/arguments/launch/mpi.py +++ b/smartsim/settings/arguments/launch/mpi.py @@ -28,8 +28,8 @@ import typing as t +from smartsim._core.dispatch import ShellLauncher, dispatch, make_shell_format_fn from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/arguments/launch/pals.py b/smartsim/settings/arguments/launch/pals.py index 3132f1b02..2727e47d5 100644 --- a/smartsim/settings/arguments/launch/pals.py +++ b/smartsim/settings/arguments/launch/pals.py @@ -28,8 +28,8 @@ import typing as t +from smartsim._core.dispatch import ShellLauncher, dispatch, make_shell_format_fn from smartsim.log import get_logger -from smartsim.settings.dispatch import ShellLauncher, dispatch, make_shell_format_fn from ...common import set_check_input from ...launchCommand import LauncherType diff --git a/smartsim/settings/arguments/launch/slurm.py b/smartsim/settings/arguments/launch/slurm.py index f8e514ab2..c32e4b6bc 100644 --- a/smartsim/settings/arguments/launch/slurm.py +++ b/smartsim/settings/arguments/launch/slurm.py @@ -32,14 +32,15 @@ import subprocess import typing as t -from smartsim.log import get_logger -from smartsim.settings.dispatch import ( +from smartsim._core.dispatch import ( ExecutableProtocol, ShellLauncher, ShellLauncherCommand, _EnvironMappingType, dispatch, + make_shell_format_fn, ) +from smartsim.log import get_logger from ...common import set_check_input from ...launchCommand import LauncherType diff --git a/tests/temp_tests/test_settings/conftest.py b/tests/temp_tests/test_settings/conftest.py index 834fdf987..d5e66e94d 100644 --- a/tests/temp_tests/test_settings/conftest.py +++ b/tests/temp_tests/test_settings/conftest.py @@ -26,7 +26,7 @@ import pytest -from smartsim.settings import dispatch +from smartsim._core import dispatch from smartsim.settings.arguments import launchArguments as launch diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index ed18d5810..3bec0eb30 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -29,12 +29,12 @@ import pytest +from smartsim._core.dispatch import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.alps import ( AprunLaunchArguments, _as_aprun_command, ) -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a diff --git a/tests/temp_tests/test_settings/test_dispatch.py b/tests/temp_tests/test_settings/test_dispatch.py index cbb8ca3c4..db346ab98 100644 --- a/tests/temp_tests/test_settings/test_dispatch.py +++ b/tests/temp_tests/test_settings/test_dispatch.py @@ -32,8 +32,8 @@ import pytest +from smartsim._core import dispatch from smartsim.error import errors -from smartsim.settings import dispatch pytestmark = pytest.mark.group_a diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 65bcb6bc9..4b5970d19 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -29,12 +29,12 @@ import pytest +from smartsim._core.dispatch import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.local import ( LocalLaunchArguments, _as_local_command, ) -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index eff6b3ca2..c43d1e264 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -31,6 +31,7 @@ import pytest +from smartsim._core.dispatch import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.mpi import ( MpiexecLaunchArguments, @@ -40,7 +41,6 @@ _as_mpirun_command, _as_orterun_command, ) -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 261c04c17..acdf6c6a1 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -30,12 +30,12 @@ import pytest +from smartsim._core.dispatch import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.pals import ( PalsMpiexecLaunchArguments, _as_pals_command, ) -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 4160e60f1..206d4965a 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -27,12 +27,12 @@ import pytest +from smartsim._core.dispatch import ShellLauncherCommand from smartsim.settings import LaunchSettings from smartsim.settings.arguments.launch.slurm import ( SlurmLaunchArguments, _as_srun_command, ) -from smartsim.settings.dispatch import ShellLauncherCommand from smartsim.settings.launchCommand import LauncherType pytestmark = pytest.mark.group_a diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 3f1195e9c..ff5417462 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -34,11 +34,12 @@ import pytest +from smartsim._core import dispatch from smartsim._core.control.launch_history import LaunchHistory from smartsim.entity import _mock, entity from smartsim.experiment import Experiment from smartsim.launchable import job -from smartsim.settings import dispatch, launchSettings +from smartsim.settings import launchSettings from smartsim.settings.arguments import launchArguments from smartsim.status import InvalidJobStatus, JobStatus diff --git a/tests/test_generator.py b/tests/test_generator.py index 3b2c95cf2..695c84d2e 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -258,7 +258,7 @@ def test_generate_ensemble_directory(wlmutils, generator_instance): def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): monkeypatch.setattr( - "smartsim.settings.dispatch._LauncherAdapter.start", + "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble("ensemble-name", "echo", replicas=2) @@ -278,7 +278,7 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): monkeypatch.setattr( - "smartsim.settings.dispatch._LauncherAdapter.start", + "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( @@ -300,7 +300,7 @@ def test_generate_ensemble_symlink( test_dir, wlmutils, monkeypatch, get_gen_symlink_dir ): monkeypatch.setattr( - "smartsim.settings.dispatch._LauncherAdapter.start", + "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), ) ensemble = Ensemble( @@ -327,7 +327,7 @@ def test_generate_ensemble_configure( test_dir, wlmutils, monkeypatch, get_gen_configure_dir ): monkeypatch.setattr( - "smartsim.settings.dispatch._LauncherAdapter.start", + "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), ) params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} diff --git a/tests/test_launch_history.py b/tests/test_launch_history.py index fb0274cc2..d076e41a3 100644 --- a/tests/test_launch_history.py +++ b/tests/test_launch_history.py @@ -30,7 +30,7 @@ import pytest from smartsim._core.control.launch_history import LaunchHistory -from smartsim.settings.dispatch import LauncherProtocol, create_job_id +from smartsim._core.dispatch import LauncherProtocol, create_job_id pytestmark = pytest.mark.group_a diff --git a/tests/test_shell_launcher.py b/tests/test_shell_launcher.py index d9143c52a..5ce397e2a 100644 --- a/tests/test_shell_launcher.py +++ b/tests/test_shell_launcher.py @@ -32,11 +32,11 @@ import psutil import pytest +from smartsim._core.dispatch import ShellLauncher, ShellLauncherCommand, sp from smartsim._core.utils import helpers from smartsim._core.utils.shell import * from smartsim.entity import _mock, entity from smartsim.error.errors import LauncherJobNotFound -from smartsim.settings.dispatch import ShellLauncher, ShellLauncherCommand, sp from smartsim.status import JobStatus # TODO tests bad vars in Popen call at beginning @@ -112,7 +112,7 @@ def test_shell_launcher_init(): def test_shell_launcher_start_calls_popen(shell_cmd: ShellLauncherCommand): """Test that the process leading up to the shell launcher popen call was correct""" shell_launcher = ShellLauncher() - with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: + with unittest.mock.patch("smartsim._core.dispatch.sp.Popen") as mock_open: _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once() @@ -120,7 +120,7 @@ def test_shell_launcher_start_calls_popen(shell_cmd: ShellLauncherCommand): def test_shell_launcher_start_calls_popen_with_value(shell_cmd: ShellLauncherCommand): """Test that popen was called with correct values""" shell_launcher = ShellLauncher() - with unittest.mock.patch("smartsim.settings.dispatch.sp.Popen") as mock_open: + with unittest.mock.patch("smartsim._core.dispatch.sp.Popen") as mock_open: _ = shell_launcher.start(shell_cmd) mock_open.assert_called_once_with( shell_cmd.command_tuple,