post-merge fixes & move tests/dragon

Makefile

@@ -174,12 +174,12 @@ test-verbose:
 # help: test-debug                     - Run all tests with debug output
 .PHONY: test-debug
 test-debug:
-	@SMARTSIM_LOG_LEVEL=developer python -m pytest -s -o log_cli=true -vv --ignore=tests/full_wlm/ --ignore=tests/dragon
+	@SMARTSIM_LOG_LEVEL=developer python -m pytest -s -o log_cli=true -vv --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-cov                       - Run all tests with coverage
 .PHONY: test-cov
 test-cov:
-	@python -m pytest -vv --cov=./smartsim --cov-config=${COV_FILE} --ignore=tests/full_wlm/ --ignore=tests/dragon
+	@python -m pytest -vv --cov=./smartsim --cov-config=${COV_FILE} --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 
 # help: test-full                      - Run all WLM tests with Python coverage (full test suite)
@@ -196,4 +196,4 @@ test-wlm:
 # help: test-dragon                   - Run dragon-specific tests
 .PHONY: test-dragon
 test-dragon:
-	@dragon pytest tests/dragon
+	@dragon pytest tests/dragon_wlm

doc/changelog.md

@@ -13,6 +13,7 @@ Jump to:
 
 Description
 
+- Merge core refactor into MLI feature branch
 - Implement asynchronous notifications for shared data
 - Quick bug fix in _validate
 - Add helper methods to MLI classes

smartsim/_core/_cli/build.py

@@ -27,8 +27,8 @@
 import argparse
 import importlib.metadata
 import operator
-import platform
 import os
+import platform
 import re
 import shutil
 import textwrap
@@ -44,19 +44,10 @@
     display_post_install_logs,
     install_dragon,
 )
-from smartsim._core._cli.scripts.dragon_install import install_dragon
 from smartsim._core._cli.utils import SMART_LOGGER_FORMAT, pip
 from smartsim._core._install import builder
-from smartsim._core._install.buildenv import (
-    BuildEnv,
-    SetupError,
-    Version_,
-    VersionConflictError,
-    Versioner,
-)
-from smartsim._core._cli.utils import SMART_LOGGER_FORMAT
-from smartsim._core._install import builder
-from smartsim._core._install.buildenv import BuildEnv, DbEngine, Version_, Versioner
+from smartsim._core._install.buildenv import BuildEnv, SetupError, Version_, Versioner
+from smartsim._core._install.builder import BuildError
 from smartsim._core._install.mlpackages import (
     DEFAULT_MLPACKAGE_PATH,
     DEFAULT_MLPACKAGES,
@@ -79,6 +70,7 @@
 # NOTE: all smartsim modules need full paths as the smart cli
 #       may be installed into a different directory.
 
+
 def parse_requirement(
     requirement: str,
 ) -> t.Tuple[str, t.Optional[str], t.Callable[[Version_], bool]]:
@@ -176,10 +168,6 @@ def execute(
     dragon_repo = args.dragon_repo
     dragon_version = args.dragon_version
 
-    if Path(CONFIG.build_path).exists():
-        logger.warning(f"Build path already exists, removing: {CONFIG.build_path}")
-        shutil.rmtree(CONFIG.build_path)
-
     # The user should never have to specify the OS and Architecture
     current_platform = Platform(
         OperatingSystem.autodetect(), Architecture.autodetect(), device
@@ -242,14 +230,6 @@ def execute(
         else:
             logger.warning("Dragon installation failed")
 
-    try:
-        if not args.only_python_packages:
-            ...
-
-    except (SetupError, BuildError) as e:
-        logger.error(str(e))
-        return os.EX_SOFTWARE
-
     backends = []
     backends_str = ", ".join(s.capitalize() for s in backends) if backends else "No"
     logger.info(f"{backends_str} backend(s) available")
@@ -345,9 +325,3 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
         default=False,
         help="Build KeyDB instead of Redis",
     )
-    parser.add_argument(
-        "--no_torch_with_mkl",
-        dest="torch_with_mkl",
-        action="store_false",
-        help="Do not build Torch with Intel MKL",
-    )

smartsim/_core/_cli/info.py

@@ -6,9 +6,7 @@
 
 from tabulate import tabulate
 
-import smartsim._core._cli.utils as _utils
 import smartsim._core.utils.helpers as _helpers
-from smartsim._core._cli.scripts.dragon_install import dragon_pin
 from smartsim._core._install.buildenv import BuildEnv as _BuildEnv
 
 _MISSING_DEP = _helpers.colorize("Not Installed", "red")
@@ -30,7 +28,11 @@ def execute(
     )
 
     print("Dragon Installation:")
-    dragon_version = dragon_pin()
+    # dragon_version = dragon_pin()
+    # todo: fix dragon_pin removal
+    # note: smartsim/config/dragon/.env should have an env var
+    #  that we can take the version from
+    dragon_version = "0.10"
 
     fs_table = [["Version", str(dragon_version)]]
     print(tabulate(fs_table, tablefmt="fancy_outline"), end="\n\n")

smartsim/_core/_cli/validate.py

@@ -34,7 +34,7 @@
 from types import TracebackType
 
 from smartsim._core._cli.utils import SMART_LOGGER_FORMAT
-from smartsim._core._install.builder import Device
+from smartsim._core._install.platform import Device
 from smartsim.log import get_logger
 
 logger = get_logger("Smart", fmt=SMART_LOGGER_FORMAT)

smartsim/_core/_install/buildenv.py

@@ -37,6 +37,8 @@
 
 from packaging.version import InvalidVersion, Version, parse
 
+DbEngine = t.Literal["REDIS", "KEYDB"]
+
 
 class SetupError(Exception):
     """A simple exception class for errors in _install.buildenv file.
@@ -161,25 +163,25 @@ class Versioner:
     SMARTSIM = Version_(get_env("SMARTSIM_VERSION", "0.8.0"))
     SMARTSIM_SUFFIX = get_env("SMARTSIM_SUFFIX", "")
 
-    # # Redis
-    # REDIS = Version_(get_env("SMARTSIM_REDIS", "7.2.4"))
-    # REDIS_URL = get_env("SMARTSIM_REDIS_URL", "https://github.com/redis/redis.git")
-    # REDIS_BRANCH = get_env("SMARTSIM_REDIS_BRANCH", REDIS)
-
-    # # RedisAI
-    # REDISAI = "1.2.7"
-    # REDISAI_URL = get_env(
-    #     "SMARTSIM_REDISAI_URL", "https://github.com/RedisAI/RedisAI.git"
-    # )
-    # REDISAI_BRANCH = get_env("SMARTSIM_REDISAI_BRANCH", f"v{REDISAI}")
-
-    # def as_dict(self, db_name: DbEngine = "REDIS") -> t.Dict[str, t.Tuple[str, ...]]:
-    #     pkg_map = {
-    #         "SMARTSIM": self.SMARTSIM,
-    #         db_name: self.REDIS,
-    #         "REDISAI": self.REDISAI,
-    #     }
-    #     return {"Packages": tuple(pkg_map), "Versions": tuple(pkg_map.values())}
+    # Redis
+    REDIS = Version_(get_env("SMARTSIM_REDIS", "7.2.4"))
+    REDIS_URL = get_env("SMARTSIM_REDIS_URL", "https://github.com/redis/redis.git")
+    REDIS_BRANCH = get_env("SMARTSIM_REDIS_BRANCH", REDIS)
+
+    # RedisAI
+    REDISAI = "1.2.7"
+    REDISAI_URL = get_env(
+        "SMARTSIM_REDISAI_URL", "https://github.com/RedisAI/RedisAI.git"
+    )
+    REDISAI_BRANCH = get_env("SMARTSIM_REDISAI_BRANCH", f"v{REDISAI}")
+
+    def as_dict(self, db_name: DbEngine = "REDIS") -> t.Dict[str, t.Tuple[str, ...]]:
+        pkg_map = {
+            "SMARTSIM": self.SMARTSIM,
+            db_name: self.REDIS,
+            "REDISAI": self.REDISAI,
+        }
+        return {"Packages": tuple(pkg_map), "Versions": tuple(pkg_map.values())}
 
     @staticmethod
     def get_sha(setup_py_dir: Path) -> str:

smartsim/_core/mli/comm/channel/dragon_fli.py

@@ -25,7 +25,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # isort: off
-from dragon import fli
+
+import dragon
+import dragon.fli as fli
 from dragon.channels import Channel
 
 # isort: on

smartsim/_core/mli/infrastructure/control/dragon_util.py

@@ -0,0 +1,79 @@
+# BSD 2-Clause License
+#
+# Copyright (c) 2021-2024, Hewlett Packard Enterprise
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import annotations
+
+import os
+import socket
+import typing as t
+
+import pytest
+
+dragon = pytest.importorskip("dragon")
+
+# isort: off
+
+import dragon.infrastructure.policy as dragon_policy
+import dragon.infrastructure.process_desc as dragon_process_desc
+import dragon.native.process as dragon_process
+
+# isort: on
+
+from smartsim.log import get_logger
+
+logger = get_logger(__name__)
+
+
+def function_as_dragon_proc(
+    entrypoint_fn: t.Callable[[t.Any], None],
+    args: t.List[t.Any],
+    cpu_affinity: t.List[int],
+    gpu_affinity: t.List[int],
+) -> dragon_process.Process:
+    """Execute a function as an independent dragon process.
+
+    :param entrypoint_fn: The function to execute
+    :param args: The arguments for the entrypoint function
+    :param cpu_affinity: The cpu affinity for the process
+    :param gpu_affinity: The gpu affinity for the process
+    :returns: The dragon process handle
+    """
+    options = dragon_process_desc.ProcessOptions(make_inf_channels=True)
+    local_policy = dragon_policy.Policy(
+        placement=dragon_policy.Policy.Placement.HOST_NAME,
+        host_name=socket.gethostname(),
+        cpu_affinity=cpu_affinity,
+        gpu_affinity=gpu_affinity,
+    )
+    return dragon_process.Process(
+        target=entrypoint_fn,
+        args=args,
+        cwd=os.getcwd(),
+        policy=local_policy,
+        options=options,
+        stderr=dragon_process.Popen.STDOUT,
+        stdout=dragon_process.Popen.STDOUT,
+    )

smartsim/_core/utils/helpers.py

@@ -34,17 +34,17 @@
 import functools
 import itertools
 import os
-from pathlib import Path
 import signal
 import subprocess
 import sys
 import typing as t
 import uuid
 import warnings
 from datetime import datetime
+from pathlib import Path
 from shutil import which
 
-import deprecated
+from deprecated import deprecated
 from typing_extensions import TypeAlias
 
 if t.TYPE_CHECKING:

tests/dragon/conftest.py → tests/dragon_wlm/conftest.py

@@ -27,10 +27,7 @@
 from __future__ import annotations
 
 import os
-import pathlib
 import socket
-import subprocess
-import sys
 import typing as t
 
 import pytest

tests/dragon/test_dragon_backend.py → tests/dragon_wlm/test_dragon_backend.py

@@ -33,7 +33,7 @@
 dragon = pytest.importorskip("dragon")
 
 
-from smartsim._core.launcher.dragon.dragonBackend import DragonBackend
+from smartsim._core.launcher.dragon.dragon_backend import DragonBackend
 from smartsim._core.mli.comm.channel.dragon_channel import DragonCommChannel
 from smartsim._core.mli.infrastructure.comm.event import (
     OnCreateConsumer,