From 2df210ee9164a491bf53882b7e2c74af7f4e69cd Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 22 Aug 2024 15:43:30 -0600 Subject: [PATCH 01/71] Initial attempt to fix Nick's problem --- lib/pavilion/commands/_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pavilion/commands/_run.py b/lib/pavilion/commands/_run.py index b06bf4079..820754b28 100644 --- a/lib/pavilion/commands/_run.py +++ b/lib/pavilion/commands/_run.py @@ -234,6 +234,7 @@ def _run(self, test: TestRun): except TestRunError as err: # An unexpected TestRunError test.status.set(STATES.RUN_ERROR, err) + return except TimeoutError: # This is expected pass From a48009fe628054f7f96799190260d402afd84b8d Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 23 Aug 2024 12:30:52 -0600 Subject: [PATCH 02/71] Exit immediately on raising error The _run error handling in the _run function was causing it to continue onto gather results even after TestRun.run had failed, but was trying to use a variable that hadn't been created due to the error handing. This commit simply changes _run to exit on an error instead of continuing on to resul parsing. --- lib/pavilion/commands/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/commands/_run.py b/lib/pavilion/commands/_run.py index 820754b28..dd7223720 100644 --- a/lib/pavilion/commands/_run.py +++ b/lib/pavilion/commands/_run.py @@ -237,7 +237,7 @@ def _run(self, test: TestRun): return except TimeoutError: # This is expected - pass + return except Exception: # Some other unexpected exception. test.status.set( From caa5f24a3ac121d5363282a1b16d3ef0f37c36c9 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 27 Sep 2024 12:58:46 -0600 Subject: [PATCH 03/71] Implement PR feedback --- lib/pavilion/commands/_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pavilion/commands/_run.py b/lib/pavilion/commands/_run.py index dd7223720..f1bd1f824 100644 --- a/lib/pavilion/commands/_run.py +++ b/lib/pavilion/commands/_run.py @@ -237,6 +237,8 @@ def _run(self, test: TestRun): return except TimeoutError: # This is expected + test.status.set(STATES.RUN_ERROR, + f"Timed out waiting for test {test.name} to complete.") return except Exception: # Some other unexpected exception. From c8e6d93d360d4aac4d44d7fc3f7613b49f4e0e70 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 27 Sep 2024 14:28:01 -0600 Subject: [PATCH 04/71] Increase timeout for test_owner --- test/tests/utils_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 0e502b607..0610313fc 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -66,7 +66,7 @@ def test_owner(self): # Try to set the permissions of the file to an unknown user. proc = sp.Popen(['sudo', '-n', 'chown', '12341', path.as_posix()], stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE) - if proc.wait(1) == 0: + if proc.wait(2) == 0: self.assertEqual(utils.owner(path), "") def test_relative_to(self): From 8d4feb2104e2dac95e3de8a1968b574fc3626496 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 8 Aug 2024 15:34:37 -0600 Subject: [PATCH 05/71] Add suites directory --- lib/pavilion/builder.py | 8 +++---- lib/pavilion/commands/config.py | 2 +- lib/pavilion/common.py | 24 +++++++++++++++++++ lib/pavilion/config.py | 32 +++++++++++++++---------- lib/pavilion/create_files.py | 7 +++--- lib/pavilion/test_config/file_format.py | 10 ++++---- lib/pavilion/test_run/test_run.py | 6 ++++- 7 files changed, 63 insertions(+), 26 deletions(-) create mode 100644 lib/pavilion/common.py diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 7cdf3acef..4b5d8a59c 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -223,7 +223,7 @@ def _create_build_hash(self) -> str: # Hash extra files. for extra_file in self._config.get('extra_files', []): extra_file = Path(extra_file) - full_path = self._pav_cfg.find_file(extra_file, Path('test_src')) + full_path = self._pav_cfg.find_file(extra_file, [Path('suites'), Path('test_src')]) if full_path is None: raise TestBuilderError( @@ -317,7 +317,7 @@ def _update_src(self): "The source path must be a valid unix path, either relative " "or absolute, got '{}'".format(src_path), err) - found_src_path = self._pav_cfg.find_file(src_path, 'test_src') + found_src_path = self._pav_cfg.find_file(src_path, ['suites', 'test_src']) src_url = self._config.get('source_url') src_download = self._config.get('source_download') @@ -670,7 +670,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): if raw_src_path is None: src_path = None else: - src_path = self._pav_cfg.find_file(Path(raw_src_path), 'test_src') + src_path = self._pav_cfg.find_file(Path(raw_src_path), ['suites', 'test_src']) if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) @@ -777,7 +777,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): # Now we just need to copy over all the extra files. for extra in self._config.get('extra_files', []): extra = Path(extra) - path = self._pav_cfg.find_file(extra, 'test_src') + path = self._pav_cfg.find_file(extra, ['test_src', 'suites']) final_dest = dest / path.name try: if path.is_dir(): diff --git a/lib/pavilion/commands/config.py b/lib/pavilion/commands/config.py index f3a5ceae9..81a86be6e 100644 --- a/lib/pavilion/commands/config.py +++ b/lib/pavilion/commands/config.py @@ -244,7 +244,7 @@ def create_config_dir(self, pav_cfg: config.PavConfig, path: Path, raise ConfigCmdError("Error writing config file at '{}'" .format(config_file_path), err) - for subdir in 'hosts', 'modes', 'tests', 'os', 'test_src', 'plugins', 'collections': + for subdir in 'hosts', 'modes', 'tests', 'os', 'test_src', 'plugins', 'collections', 'suites': subdir = path/subdir try: subdir.mkdir() diff --git a/lib/pavilion/common.py b/lib/pavilion/common.py new file mode 100644 index 000000000..aed9e8c8d --- /dev/null +++ b/lib/pavilion/common.py @@ -0,0 +1,24 @@ +from typing import List, Union, TypeVar, Iterator, Iterable, Callable, Optional + +T = TypeVar("T") + + +def enforce_list(val: Union[T, List[T]]) -> List[T]: + if isinstance(val, list): + return val + + return list(val) + +def replace(lst: Iterable[T], old: T, new: T) -> Iterator[T]: + return map(lambda x: new if x == old else x, lst) + +def remove_none(lst: Iterable[T]) -> Iterator[T]: + return filter(lambda x: x is not None, lst) + +def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: + filtered = list(filter(pred, lst)) + + if len(filtered) > 0: + return filtered[0] + + return None diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 6b4c286d9..0b535aa0a 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -11,11 +11,13 @@ import sys from collections import OrderedDict from pathlib import Path +from itertools import product, starmap from typing import List, Union, Dict, NewType import yaml_config as yc from pavilion import output from pavilion import errors +from pavilion.common import enforce_list, first # Figure out what directories we'll search for the base configuration. PAV_CONFIG_SEARCH_DIRS = [Path('./').resolve()] @@ -182,6 +184,7 @@ def as_dict(self) -> dict: return adict +Pathlike = Union[str, Path] class PavConfig(PavConfigDict): """Define types and attributes for Pavilion config options.""" @@ -217,14 +220,15 @@ def __init__(self, set_attrs=None): super().__init__(set_attrs) - def find_file(self, file: Path, sub_dir: Union[str, Path] = None) \ + + def find_file(self, file: Path, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ -> Union[Path, None]: """Look for the given file and return a full path to it. Relative paths are searched for in all config directories under 'sub_dir', if it exists. :param file: The path to the file. - :param sub_dir: The subdirectory in each config directory in which to - search. + :param sub_dirs: The subdirectory (or list of subdirectories) in which to + search in each directory. :returns: The full path to the found file, or None if no such file could be found.""" @@ -234,17 +238,21 @@ def find_file(self, file: Path, sub_dir: Union[str, Path] = None) \ else: return None - # Assemble a potential location from each config dir. - for config in self.configs.values(): - path = config['path'] - if sub_dir is not None: - path = path/sub_dir - path = path/file + sub_dirs = enforce_list(sub_dirs) + + cfg_paths = map(lambda x: x.get('path'), self.configs.values()) + path_comps = product(cfg_paths, sub_dirs, [file]) + + def make_path(path: Path, subdir: Pathlike, file: Path) -> Path: + if subdir is None: + return path / file + + return path / subdir / file - if path.exists(): - return path + paths = starmap(make_path, path_comps) - return None + # Return the first path to the file that exists (or None) + return first(lambda x: x.exists(), paths) class ExPathElem(yc.PathElem): diff --git a/lib/pavilion/create_files.py b/lib/pavilion/create_files.py index 9b7ddb8cb..73cf30b91 100644 --- a/lib/pavilion/create_files.py +++ b/lib/pavilion/create_files.py @@ -64,11 +64,12 @@ def resolve_template(pav_cfg: pavilion.config.PavConfig, template: str, var_man: variables.VariableSetManager) -> List[str]: """Resolve each of the template files specified in the test config.""" - tmpl_path = pav_cfg.find_file(Path(template), 'test_src') + tmpl_path = pav_cfg.find_file(Path(template), ['suites', 'test_src']) + if tmpl_path is None: raise TestConfigError("Template file '{}' from 'templates' does not exist in " - "any 'test_src' dir (Note that it must be in a Pavilion config " - "area's test_src directory - NOT the build directory.)" + "any 'suites' dir (Note that it must be in a Pavilion config " + "area's suites directory - NOT the build directory.)" .format(template)) try: diff --git a/lib/pavilion/test_config/file_format.py b/lib/pavilion/test_config/file_format.py index 6e7cdb240..fe48d0374 100644 --- a/lib/pavilion/test_config/file_format.py +++ b/lib/pavilion/test_config/file_format.py @@ -623,7 +623,7 @@ class TestConfigLoader(yc.YamlConfigLoader): sub_elem=yc.StrElem(), help_text="Template files to resolve using Pavilion test variables. The " "key is the path to the template file (typically with a " - "'.pav' extension), in the 'test_src' directory. The value is the " + "'.pav' extension), in the 'suites' directory. The value is the " "output file location, relative to the test's build/run " "directory."), EnvCatElem( @@ -657,7 +657,7 @@ class TestConfigLoader(yc.YamlConfigLoader): "uncompressed archive (zip/tar), and is handled " "according to the internal (file-magic) type. " "For relative paths Pavilion looks in the " - "test_src directory " + "suites directory " "within all known config directories. If this " "is left blank, Pavilion will always assume " "there is no source to build."), @@ -669,7 +669,7 @@ class TestConfigLoader(yc.YamlConfigLoader): 'can\'t otherwise be found. You must give a ' 'source path so Pavilion knows where to store ' 'the file (relative paths will be stored ' - 'relative to the local test_src directory.'), + 'relative to the local suites directory.'), yc.StrElem( 'source_download', choices=['never', 'missing', 'latest'], default='missing', @@ -773,7 +773,7 @@ class TestConfigLoader(yc.YamlConfigLoader): "file. Pavilion test variables will be resolved in this lines " "before they are written."), - # Note - Template have to come from the test_src directory (or elsewhere on + # Note - Template have to come from the suites directory (or elsewhere on # the filesystem, because we have to be able to process them before # we can create a build hash. PathCategoryElem( @@ -782,7 +782,7 @@ class TestConfigLoader(yc.YamlConfigLoader): sub_elem=yc.StrElem(), help_text="Template files to resolve using Pavilion test variables. The " "key is the path to the template file (typically with a " - "'.pav' extension), in the 'test_src' directory. The value is the " + "'.pav' extension), in the 'suites' directory. The value is the " "output file location, relative to the test's build/run " "directory."), EnvCatElem( diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 9b093a5cc..3673b6c95 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -337,7 +337,11 @@ def _make_builder(self) -> builder.TestBuilder: spack_config = (self.config.get('spack_config', {}) if self.spack_enabled() else None) if self.suite_path != Path('..') and self.suite_path is not None: - download_dest = self.suite_path.parents[1] / 'test_src' + download_dest = self.suite_path.parents[1] / 'suites' + + # Check the deprecated directory + if not download_dest.exists(): + download_dest = self.suite_path.parents[1] / 'test_src' else: download_dest = None From 0972b87f5440d02e466c4bfd8af0d605d9e9ca2e Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 16 Aug 2024 11:52:03 -0600 Subject: [PATCH 06/71] Progress towards solution --- lib/pavilion/builder.py | 9 ++--- lib/pavilion/common.py | 27 +++++--------- lib/pavilion/config.py | 35 +++++++++++++++--- lib/pavilion/create_files.py | 13 ++++--- lib/pavilion/func_utils.py | 60 +++++++++++++++++++++++++++++++ lib/pavilion/resolve.py | 4 +-- lib/pavilion/test_run/test_run.py | 8 +++-- 7 files changed, 120 insertions(+), 36 deletions(-) create mode 100644 lib/pavilion/func_utils.py diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 4b5d8a59c..ad16c12ac 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -98,6 +98,7 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config self.name = build_name self.path = working_dir/'builds'/self.name # type: Path + self.suite_dir = Path('suites') / self.name current_status = status.current() @@ -223,7 +224,7 @@ def _create_build_hash(self) -> str: # Hash extra files. for extra_file in self._config.get('extra_files', []): extra_file = Path(extra_file) - full_path = self._pav_cfg.find_file(extra_file, [Path('suites'), Path('test_src')]) + full_path = self._pav_cfg.find_file(extra_file, [self.suite_dir, Path('test_src')]) if full_path is None: raise TestBuilderError( @@ -317,7 +318,7 @@ def _update_src(self): "The source path must be a valid unix path, either relative " "or absolute, got '{}'".format(src_path), err) - found_src_path = self._pav_cfg.find_file(src_path, ['suites', 'test_src']) + found_src_path = self._pav_cfg.find_file(src_path, [self.suite_dir, Path('test_src')]) src_url = self._config.get('source_url') src_download = self._config.get('source_download') @@ -670,7 +671,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): if raw_src_path is None: src_path = None else: - src_path = self._pav_cfg.find_file(Path(raw_src_path), ['suites', 'test_src']) + src_path = self._pav_cfg.find_file(Path(raw_src_path), [self.suite_dir, Path('test_src')]) if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) @@ -777,7 +778,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): # Now we just need to copy over all the extra files. for extra in self._config.get('extra_files', []): extra = Path(extra) - path = self._pav_cfg.find_file(extra, ['test_src', 'suites']) + path = self._pav_cfg.find_file(extra, [self.suite_dir, Path('test_src')]) final_dest = dest / path.name try: if path.is_dir(): diff --git a/lib/pavilion/common.py b/lib/pavilion/common.py index aed9e8c8d..3146894e0 100644 --- a/lib/pavilion/common.py +++ b/lib/pavilion/common.py @@ -1,24 +1,15 @@ -from typing import List, Union, TypeVar, Iterator, Iterable, Callable, Optional +from typing import TypeVar, Optional T = TypeVar("T") +def set_default(val: Optional[T], default: T) -> T: + """Set the input value to its default, if it is None.""" -def enforce_list(val: Union[T, List[T]]) -> List[T]: - if isinstance(val, list): - return val + if val is None: + return default - return list(val) + return val -def replace(lst: Iterable[T], old: T, new: T) -> Iterator[T]: - return map(lambda x: new if x == old else x, lst) - -def remove_none(lst: Iterable[T]) -> Iterator[T]: - return filter(lambda x: x is not None, lst) - -def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: - filtered = list(filter(pred, lst)) - - if len(filtered) > 0: - return filtered[0] - - return None +def get_nested(keys: Iterable[Hashable], dict: Dict) -> Dict: + """Safely get the hierarchical sequence of keys off the + dictionary. Guaranteed to return a dictionary.""" diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 0b535aa0a..af151b120 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -12,12 +12,12 @@ from collections import OrderedDict from pathlib import Path from itertools import product, starmap -from typing import List, Union, Dict, NewType +from typing import List, Union, Dict, NewType, Iterator import yaml_config as yc from pavilion import output from pavilion import errors -from pavilion.common import enforce_list, first +from pavilion.func_utils import first, flatten # Figure out what directories we'll search for the base configuration. PAV_CONFIG_SEARCH_DIRS = [Path('./').resolve()] @@ -220,7 +220,33 @@ def __init__(self, set_attrs=None): super().__init__(set_attrs) + @property + def cfg_paths(self) -> Iterator[Path]: + """Return an iterator of paths to all config directories""" + return (Path(cfg['path']) for cfg in self.configs.values()) + @property + def suites_dirs(self) -> Iterator[Path]: + """Return an iterator of paths to all suites directories""" + return (path / 'suites' for path in self.cfg_paths) + + @property + def suite_paths(self) -> Iterator[Path]: + """Return an iterator of paths to all test suites""" + return flatten(path.iterdir() for path in self.suites_dirs) + + @property + def suite_names(self) -> Iterator[str]: + """Return an iterator of suite names""" + + def is_suite(file: Path) -> bool: + return file.exists() and file.is_file() and file.suffix == '.yaml' + + test_files = map(is_suite, self.suite_paths) + + return map(lambda x: x.stem, self.suite_paths) + + def find_file(self, file: Path, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ -> Union[Path, None]: """Look for the given file and return a full path to it. Relative paths @@ -238,10 +264,9 @@ def find_file(self, file: Path, sub_dirs: Union[List[Pathlike], Pathlike] = None else: return None - sub_dirs = enforce_list(sub_dirs) + sub_dirs = list(sub_dirs) - cfg_paths = map(lambda x: x.get('path'), self.configs.values()) - path_comps = product(cfg_paths, sub_dirs, [file]) + path_comps = product(self.cfg_paths, sub_dirs, list(file)) def make_path(path: Path, subdir: Pathlike, file: Path) -> Path: if subdir is None: diff --git a/lib/pavilion/create_files.py b/lib/pavilion/create_files.py index 73cf30b91..b40686cb3 100644 --- a/lib/pavilion/create_files.py +++ b/lib/pavilion/create_files.py @@ -1,12 +1,14 @@ """Functions to dynamically generate test files.""" from pathlib import Path -from typing import List, Union, TextIO +from typing import List, Union, TextIO, Any import pavilion.config from pavilion import resolve from pavilion import utils from pavilion import variables +from pavilion.config import PavConfig +from pavilion.variables import VariableSetManager from pavilion.errors import TestConfigError @@ -60,11 +62,12 @@ def verify_path(dest, rel_path) -> Path: return file_path -def resolve_template(pav_cfg: pavilion.config.PavConfig, template: str, - var_man: variables.VariableSetManager) -> List[str]: - """Resolve each of the template files specified in the test config.""" +def resolve_template(pav_cfg: PavConfig, template: Path, var_man: VariableSetManager) -> Any: + """Resolve a single template file specified in the test config. Return a resolved + component.""" - tmpl_path = pav_cfg.find_file(Path(template), ['suites', 'test_src']) + # TODO: This needs to be the test-specific suites directory + tmpl_path = pav_cfg.find_file(template, ['suites', 'test_src']) if tmpl_path is None: raise TestConfigError("Template file '{}' from 'templates' does not exist in " diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py new file mode 100644 index 000000000..a482500d6 --- /dev/null +++ b/lib/pavilion/func_utils.py @@ -0,0 +1,60 @@ +"""A collection of utilities defined using functional methods""" + +from typing import (List, Union, TypeVar, Iterator, Iterable, Callable, Optional, + Hashable, Dict, Tuple) + + +T = TypeVar('T') +U = TypeVar('U') + + +def partition(pred: Callable[[T], bool], lst: Iterable[T]) -> Tuple[Iterator[T], Iterator[T]]: + """Partition the sequence into two sequences: one consisting of the elements + for which the given predicate is true and one consisting of those for + which it is false.""" + + f_true, f_false = tee(lst) + + return filter(pred, f_true), filterfalse(pred, f_false) + +def flatten(lst: Iterable[Iterable[T]]) -> Iterator[T]: + """Convert a singly nested iterable into an unnested iterable.""" + return chain.from_iterable(lst) + +def remove_all(lst: Iterable[T], item: T) -> Iterator[T]: + """Remove all instances of the given item from the iterable.""" + return filter(lambda x: x != item, lst) + +def unique(lst: Iterable[T]) -> List[T]: + """Return a list of the unique items in the original list.""" + return list(set(lst)) + +def replace(lst: Iterable[T], old: T, new: T) -> Iterator[T]: + """Replace all instances of old with new.""" + return map(lambda x: new if x == old else x, lst) + +def remove_none(lst: Iterable[T]) -> Iterator[T]: + """Remove all instances of None from the iterable.""" + return filter(lambda x: x is not None, lst) + +def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: + """Return the first item of the list that satisfies the given + predicate, or None if no item does.""" + + for item in filter(pred, lst): + return item + +def apply_to_first(func: Callable[[T], U], pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[U]: + """Apply the function to the first element of the list that satisfies + the given predicate. If no element satisfies the predicate, return None.""" + + fst = first(pred, lst) + + if fst is not None: + return func(fst) + +def get_nested(keys: Iterable[Hashable], nested_dict: Dict) -> Dict: + for key in keys: + nested_dict = nested_dict.get(key) + + return nested_dict diff --git a/lib/pavilion/resolve.py b/lib/pavilion/resolve.py index 8cba6162e..769b88498 100644 --- a/lib/pavilion/resolve.py +++ b/lib/pavilion/resolve.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Union, Tuple +from typing import Dict, List, Union, Tuple, Any from pavilion import parsers from pavilion import variables @@ -132,7 +132,7 @@ def section_values(component: Union[Dict, List, str], var_man: variables.VariableSetManager, allow_deferred: bool = False, deferred_only: bool = False, - key_parts: Union[None, Tuple[str]] = None): + key_parts: Union[None, Tuple[str]] = None) -> Any: """Recursively resolve the given config component's value strings using a variable manager. diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 3673b6c95..22b880242 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -36,6 +36,7 @@ from pavilion.test_config.file_format import NO_WORKING_DIR from pavilion.test_config.utils import parse_timeout from pavilion.types import ID_Pair +from pavilion.func_utils import get_nested from .test_attrs import TestAttributes @@ -370,9 +371,10 @@ def _make_builder(self) -> builder.TestBuilder: def _create_build_templates(self) -> Dict[Path, Path]: """Generate templated files for the builder to use.""" - templates = self.config.get('build', {}).get('templates', {}) + templates = get_nested(['build', 'templates'], self.config) tmpl_dir = self.path/self.BUILD_TEMPLATE_DIR - if templates: + + if templates != {}: if not tmpl_dir.exists(): try: tmpl_dir.mkdir(exist_ok=True) @@ -380,6 +382,7 @@ def _create_build_templates(self) -> Dict[Path, Path]: raise TestRunError("Could not create build template directory", err) tmpl_paths = {} + for tmpl_src, tmpl_dest in templates.items(): if not (tmpl_dir/tmpl_dest).exists(): try: @@ -387,6 +390,7 @@ def _create_build_templates(self) -> Dict[Path, Path]: create_files.create_file(tmpl_dest, tmpl_dir, tmpl, newlines='') except TestConfigError as err: raise TestRunError("Error resolving Build template files", err) + tmpl_paths[tmpl_dir/tmpl_dest] = tmpl_dest return tmpl_paths From 29f42f7a55dda21ce2dacbdf4688c97a947451af Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 16 Aug 2024 13:45:47 -0600 Subject: [PATCH 07/71] Further progress --- lib/pavilion/builder.py | 20 ++++++++++++++------ lib/pavilion/config.py | 6 ++++-- lib/pavilion/create_files.py | 6 +++--- lib/pavilion/func_utils.py | 2 +- lib/pavilion/test_run/test_run.py | 5 +++++ 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index ad16c12ac..d31dc7baf 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -77,7 +77,7 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config self._download_dest = download_dest self._templates: Dict[Path, Path] = templates or {} self._build_hash = None - + try: self._timeout = parse_timeout(config.get('timeout')) except ValueError: @@ -98,7 +98,6 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config self.name = build_name self.path = working_dir/'builds'/self.name # type: Path - self.suite_dir = Path('suites') / self.name current_status = status.current() @@ -135,6 +134,15 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config raise TestBuilderError("build.create_file has bad destination path '{}'" .format(dest), err) + @property + def suite_path(self) -> Path: + spath = self._config.get('suite_path') + + if spath in (None, ''): + return Path('..').resolve() + + return Path(spath) + def exists(self): """Return True if the given build exists.""" return self.path.exists() @@ -224,7 +232,7 @@ def _create_build_hash(self) -> str: # Hash extra files. for extra_file in self._config.get('extra_files', []): extra_file = Path(extra_file) - full_path = self._pav_cfg.find_file(extra_file, [self.suite_dir, Path('test_src')]) + full_path = self._pav_cfg.find_file(extra_file, [self.suite_path, Path('test_src')]) if full_path is None: raise TestBuilderError( @@ -318,7 +326,7 @@ def _update_src(self): "The source path must be a valid unix path, either relative " "or absolute, got '{}'".format(src_path), err) - found_src_path = self._pav_cfg.find_file(src_path, [self.suite_dir, Path('test_src')]) + found_src_path = self._pav_cfg.find_file(src_path, [self.suite_path, Path('test_src')]) src_url = self._config.get('source_url') src_download = self._config.get('source_download') @@ -671,7 +679,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): if raw_src_path is None: src_path = None else: - src_path = self._pav_cfg.find_file(Path(raw_src_path), [self.suite_dir, Path('test_src')]) + src_path = self._pav_cfg.find_file(Path(raw_src_path), [self.suite_path, Path('test_src')]) if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) @@ -778,7 +786,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): # Now we just need to copy over all the extra files. for extra in self._config.get('extra_files', []): extra = Path(extra) - path = self._pav_cfg.find_file(extra, [self.suite_dir, Path('test_src')]) + path = self._pav_cfg.find_file(extra, [self.suite_path, Path('test_src')]) final_dest = dest / path.name try: if path.is_dir(): diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index af151b120..690ed3127 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -247,7 +247,7 @@ def is_suite(file: Path) -> bool: return map(lambda x: x.stem, self.suite_paths) - def find_file(self, file: Path, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ + def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ -> Union[Path, None]: """Look for the given file and return a full path to it. Relative paths are searched for in all config directories under 'sub_dir', if it exists. @@ -258,6 +258,8 @@ def find_file(self, file: Path, sub_dirs: Union[List[Pathlike], Pathlike] = None :returns: The full path to the found file, or None if no such file could be found.""" + file = Path(file) + if file.is_absolute(): if file.exists(): return file @@ -266,7 +268,7 @@ def find_file(self, file: Path, sub_dirs: Union[List[Pathlike], Pathlike] = None sub_dirs = list(sub_dirs) - path_comps = product(self.cfg_paths, sub_dirs, list(file)) + path_comps = product(self.cfg_paths, sub_dirs, [file]) def make_path(path: Path, subdir: Pathlike, file: Path) -> Path: if subdir is None: diff --git a/lib/pavilion/create_files.py b/lib/pavilion/create_files.py index b40686cb3..57eb02230 100644 --- a/lib/pavilion/create_files.py +++ b/lib/pavilion/create_files.py @@ -62,18 +62,18 @@ def verify_path(dest, rel_path) -> Path: return file_path -def resolve_template(pav_cfg: PavConfig, template: Path, var_man: VariableSetManager) -> Any: +def resolve_template(pav_cfg: PavConfig, template_fname: str, var_man: VariableSetManager) -> Any: """Resolve a single template file specified in the test config. Return a resolved component.""" # TODO: This needs to be the test-specific suites directory - tmpl_path = pav_cfg.find_file(template, ['suites', 'test_src']) + tmpl_path = pav_cfg.find_file(template_fname, ['suites', 'test_src']) if tmpl_path is None: raise TestConfigError("Template file '{}' from 'templates' does not exist in " "any 'suites' dir (Note that it must be in a Pavilion config " "area's suites directory - NOT the build directory.)" - .format(template)) + .format(template_fname)) try: with tmpl_path.open() as tmpl_file: diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py index a482500d6..c89f58a2b 100644 --- a/lib/pavilion/func_utils.py +++ b/lib/pavilion/func_utils.py @@ -55,6 +55,6 @@ def apply_to_first(func: Callable[[T], U], pred: Callable[[T], bool], lst: Itera def get_nested(keys: Iterable[Hashable], nested_dict: Dict) -> Dict: for key in keys: - nested_dict = nested_dict.get(key) + nested_dict = nested_dict.get(key, {}) return nested_dict diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 22b880242..7923cd34b 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -252,6 +252,11 @@ def is_empty(section: str) -> bool: return not all(map(is_empty, sections)) + @property + def suite_name(self) -> str: + """Return the name of the suite associated with the test.""" + return self.suite_path.stem + @property def id_pair(self) -> ID_Pair: """Returns an ID_pair (a tuple of the working dir and test id).""" From 7a17eba80a07e6a4acea750913f9a78672111080 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 16 Aug 2024 14:07:09 -0600 Subject: [PATCH 08/71] Fix style issues --- lib/pavilion/builder.py | 4 ++-- lib/pavilion/commands/config.py | 3 ++- lib/pavilion/config.py | 1 - lib/pavilion/func_utils.py | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index d31dc7baf..87943d95a 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -77,7 +77,7 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config self._download_dest = download_dest self._templates: Dict[Path, Path] = templates or {} self._build_hash = None - + try: self._timeout = parse_timeout(config.get('timeout')) except ValueError: @@ -679,7 +679,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): if raw_src_path is None: src_path = None else: - src_path = self._pav_cfg.find_file(Path(raw_src_path), [self.suite_path, Path('test_src')]) + src_path = self._pav_cfg.find_file(raw_src_path, [self.suite_path, Path('test_src')]) if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) diff --git a/lib/pavilion/commands/config.py b/lib/pavilion/commands/config.py index 81a86be6e..2f7f5d639 100644 --- a/lib/pavilion/commands/config.py +++ b/lib/pavilion/commands/config.py @@ -244,7 +244,8 @@ def create_config_dir(self, pav_cfg: config.PavConfig, path: Path, raise ConfigCmdError("Error writing config file at '{}'" .format(config_file_path), err) - for subdir in 'hosts', 'modes', 'tests', 'os', 'test_src', 'plugins', 'collections', 'suites': + for subdir in 'hosts', 'modes', 'tests', 'os', 'test_src', + 'plugins', 'collections', 'suites': subdir = path/subdir try: subdir.mkdir() diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 690ed3127..de1f3ecb0 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -246,7 +246,6 @@ def is_suite(file: Path) -> bool: return map(lambda x: x.stem, self.suite_paths) - def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ -> Union[Path, None]: """Look for the given file and return a full path to it. Relative paths diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py index c89f58a2b..18b9628b0 100644 --- a/lib/pavilion/func_utils.py +++ b/lib/pavilion/func_utils.py @@ -44,7 +44,8 @@ def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: for item in filter(pred, lst): return item -def apply_to_first(func: Callable[[T], U], pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[U]: +def apply_to_first(func: Callable[[T], U], pred: Callable[[T], bool], + lst: Iterable[T]) -> Optional[U]: """Apply the function to the first element of the list that satisfies the given predicate. If no element satisfies the predicate, return None.""" From ec9c5724ace3672affa82a20da985050bf68e36b Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 16 Aug 2024 14:10:43 -0600 Subject: [PATCH 09/71] Increase timeout for test_owner Test had been passing on develop, but no longer is. Increasing the timeout from 1 to 2 seconds causes it to pass. Something has probable changed that is slowing the filesystem down. --- test/tests/utils_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/tests/utils_tests.py b/test/tests/utils_tests.py index 0e502b607..84652527d 100644 --- a/test/tests/utils_tests.py +++ b/test/tests/utils_tests.py @@ -58,6 +58,7 @@ def test_owner(self): """Check that the owner function works.""" path = Path(tempfile.mktemp()) + with path.open('w') as file: file.write('hi there') @@ -66,7 +67,7 @@ def test_owner(self): # Try to set the permissions of the file to an unknown user. proc = sp.Popen(['sudo', '-n', 'chown', '12341', path.as_posix()], stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE) - if proc.wait(1) == 0: + if proc.wait(2) == 0: self.assertEqual(utils.owner(path), "") def test_relative_to(self): From ce6e19641430e94894d5aa37d95f837c7b3c0fdd Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 16 Aug 2024 15:08:56 -0600 Subject: [PATCH 10/71] Add suites tests --- test/tests/suites_tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/tests/suites_tests.py diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py new file mode 100644 index 000000000..0c14198b4 --- /dev/null +++ b/test/tests/suites_tests.py @@ -0,0 +1,19 @@ +from pavilion.unittest import PavTestCase + + +class SuitesTests(PavTestCase): + + def test_suite_directory_created(self): + config = { + 'name': 'suite_dir_test', + 'scheduler': 'raw', + 'build': { + 'cmds': ['echo', 'Hello'] + } + } + + test = self._quick_test(config, 'suite_dir_test', build=False) + test.build() + + cfg_dir = test.working_dir / 'pav_cfgs' + configs = list(cfg_dir.iterdir()) From d6a540a27bb67baba3119dcfff9461a40a4aa804 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 12:19:42 -0600 Subject: [PATCH 11/71] Fix mistake --- lib/pavilion/commands/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/config.py b/lib/pavilion/commands/config.py index 2f7f5d639..3d0866b83 100644 --- a/lib/pavilion/commands/config.py +++ b/lib/pavilion/commands/config.py @@ -244,8 +244,8 @@ def create_config_dir(self, pav_cfg: config.PavConfig, path: Path, raise ConfigCmdError("Error writing config file at '{}'" .format(config_file_path), err) - for subdir in 'hosts', 'modes', 'tests', 'os', 'test_src', - 'plugins', 'collections', 'suites': + for subdir in ('hosts', 'modes', 'tests', 'os', 'test_src', + 'plugins', 'collections', 'suites'): subdir = path/subdir try: subdir.mkdir() From efcb41ec2a452bb1e38bdacbf078a2346e9f79b3 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 12:21:21 -0600 Subject: [PATCH 12/71] Make resolver check both tests and suites dirs --- lib/pavilion/resolver/resolver.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index cc6567f3c..fb81c27e4 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -36,6 +36,7 @@ KEY_NAME_RE) from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary +from pavilion.func_utils import apply_to_first from yaml_config import RequiredError from .proto_test import RawProtoTest, ProtoTest @@ -102,7 +103,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, self._suites: Dict[Dict] = {} CONF_TYPE_DIRNAMES = { - 'suite': 'tests', + 'suite': ['suites', 'tests'], # Still check tests until removal 'series': 'series', 'OS': 'os', 'host': 'hosts', @@ -111,7 +112,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } - def find_config(self, conf_type, conf_name) -> Tuple[str, Path]: + def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: """Search all of the known configuration directories for a config of the given type and name. @@ -121,11 +122,15 @@ def find_config(self, conf_type, conf_name) -> Tuple[str, Path]: and the path to that config. If nothing was found, returns (None, None). """ - conf_type = self.CONF_TYPE_DIRNAMES[conf_type] + conf_type = list(self.CONF_TYPE_DIRNAMES[conf_type]) + + def make_path(conf_type: str) -> Path: + return config['path']/conf_type/'{}.yaml'.format(conf_name) for label, config in self.pav_cfg.configs.items(): - path = config['path']/conf_type/'{}.yaml'.format(conf_name) - if path.exists(): + path = apply_to_first(make_path, lambda x: make_path(x).exists(), conf_type) + + if path is not None: return label, path return '', None @@ -133,13 +138,16 @@ def find_config(self, conf_type, conf_name) -> Tuple[str, Path]: def find_similar_configs(self, conf_type, conf_name) -> List[str]: """Find configs with a name similar to the one specified.""" - conf_type = self.CONF_TYPE_DIRNAMES[conf_type] + conf_type = list(self.CONF_TYPE_DIRNAMES[conf_type]) + + def make_path(conf_type: str) -> Path: + return config['path']/conf_type for label, config in self.pav_cfg.configs.items(): - type_path = config['path']/conf_type + type_path = apply_to_first(make_path, lambda x: make_path(x).exists(), conf_type) names = [] - if type_path.exists(): + if type_path is not None and type_path.exists(): for file in type_path.iterdir(): if file.name.endswith('.yaml') and not file.is_dir(): names.append(file.name[:-5]) From 25f6b4789c4650fda34b18496fd23ae23155d626 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 12:21:55 -0600 Subject: [PATCH 13/71] Improve suites test --- .../suites/basic_suite_test/suite.yaml | 4 +++ test/tests/suites_tests.py | 32 ++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 test/data/pav_config_dir/suites/basic_suite_test/suite.yaml diff --git a/test/data/pav_config_dir/suites/basic_suite_test/suite.yaml b/test/data/pav_config_dir/suites/basic_suite_test/suite.yaml new file mode 100644 index 000000000..24205bb82 --- /dev/null +++ b/test/data/pav_config_dir/suites/basic_suite_test/suite.yaml @@ -0,0 +1,4 @@ +basic_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index 0c14198b4..c00810073 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -1,19 +1,27 @@ +from pavilion import arguments +from pavilion import commands +from pavilion import plugins from pavilion.unittest import PavTestCase class SuitesTests(PavTestCase): - def test_suite_directory_created(self): - config = { - 'name': 'suite_dir_test', - 'scheduler': 'raw', - 'build': { - 'cmds': ['echo', 'Hello'] - } - } + def setUp(self): + plugins.initialize_plugins(self.pav_cfg) + run_cmd = commands.get_command('run') + # run_cmd.silence() - test = self._quick_test(config, 'suite_dir_test', build=False) - test.build() + def test_run_from_suite_directory(self): + """Test that Pavilion can find and run a test from + suites directory.""" - cfg_dir = test.working_dir / 'pav_cfgs' - configs = list(cfg_dir.iterdir()) + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + '-H', 'this', + 'basic_suite_test' + ]) + + run_cmd = commands.get_command(args.command_name) + + self.assertEqual(run_cmd.run(self.pav_cfg, args), 0) From ffc16b6546531ff8886ddb11ac65d129b566126c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 15:06:05 -0600 Subject: [PATCH 14/71] Pass basic suites test --- lib/pavilion/resolver/resolver.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index fb81c27e4..20a6860f5 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -36,7 +36,7 @@ KEY_NAME_RE) from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary -from pavilion.func_utils import apply_to_first +from pavilion.func_utils import first from yaml_config import RequiredError from .proto_test import RawProtoTest, ProtoTest @@ -103,7 +103,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, self._suites: Dict[Dict] = {} CONF_TYPE_DIRNAMES = { - 'suite': ['suites', 'tests'], # Still check tests until removal + 'suite': 'tests', 'series': 'series', 'OS': 'os', 'host': 'hosts', @@ -112,6 +112,11 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } + def _make_suite_paths(self, config: Dict, conf_name: str) -> List[Path]: + root = config['path'] / 'suites' + + return [root / f'{conf_name}.yaml', root / conf_name / 'suite.yaml'] + def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: """Search all of the known configuration directories for a config of the given type and name. @@ -121,14 +126,16 @@ def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: :return: A tuple of the config label under which a matching config was found and the path to that config. If nothing was found, returns (None, None). """ + conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] - conf_type = list(self.CONF_TYPE_DIRNAMES[conf_type]) + for label, config in self.pav_cfg.configs.items(): + cfg_path = config['path']/conf_type/'{}.yaml'.format(conf_name) + cfg_paths = [cfg_path] - def make_path(conf_type: str) -> Path: - return config['path']/conf_type/'{}.yaml'.format(conf_name) + if conf_type == 'suite': + cfg_paths.extend(self._make_suite_paths(config, conf_name)) - for label, config in self.pav_cfg.configs.items(): - path = apply_to_first(make_path, lambda x: make_path(x).exists(), conf_type) + path = first(lambda x: x.exists(), cfg_paths) if path is not None: return label, path @@ -138,12 +145,13 @@ def make_path(conf_type: str) -> Path: def find_similar_configs(self, conf_type, conf_name) -> List[str]: """Find configs with a name similar to the one specified.""" - conf_type = list(self.CONF_TYPE_DIRNAMES[conf_type]) + # TODO: modify this for new suites directory - def make_path(conf_type: str) -> Path: - return config['path']/conf_type + conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] for label, config in self.pav_cfg.configs.items(): + conf_dirs = [conf_dir] + type_path = apply_to_first(make_path, lambda x: make_path(x).exists(), conf_type) names = [] From 673b519bfce90e02c5bbbdb738bab125ddfae460 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 15:44:28 -0600 Subject: [PATCH 15/71] Fix regression --- lib/pavilion/resolver/resolver.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 20a6860f5..c436d9a19 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -36,7 +36,7 @@ KEY_NAME_RE) from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary -from pavilion.func_utils import first +from pavilion.func_utils import first, apply_to_first from yaml_config import RequiredError from .proto_test import RawProtoTest, ProtoTest @@ -126,10 +126,11 @@ def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: :return: A tuple of the config label under which a matching config was found and the path to that config. If nothing was found, returns (None, None). """ + conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] for label, config in self.pav_cfg.configs.items(): - cfg_path = config['path']/conf_type/'{}.yaml'.format(conf_name) + cfg_path = config['path'] / conf_dir /'{}.yaml'.format(conf_name) cfg_paths = [cfg_path] if conf_type == 'suite': @@ -142,7 +143,7 @@ def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: return '', None - def find_similar_configs(self, conf_type, conf_name) -> List[str]: + def find_similar_configs(self, conf_type: str, conf_name: str) -> List[str]: """Find configs with a name similar to the one specified.""" # TODO: modify this for new suites directory @@ -150,11 +151,10 @@ def find_similar_configs(self, conf_type, conf_name) -> List[str]: conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] for label, config in self.pav_cfg.configs.items(): - conf_dirs = [conf_dir] - - type_path = apply_to_first(make_path, lambda x: make_path(x).exists(), conf_type) + type_path = config['path'] / conf_type names = [] + if type_path is not None and type_path.exists(): for file in type_path.iterdir(): if file.name.endswith('.yaml') and not file.is_dir(): From 15e198eb6ed6e8a075259cf1ff20b3ba4da06963 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 15:49:13 -0600 Subject: [PATCH 16/71] Add bare YAML suites test --- test/data/pav_config_dir/suites/bare_yaml.yaml | 4 ++++ test/tests/suites_tests.py | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 test/data/pav_config_dir/suites/bare_yaml.yaml diff --git a/test/data/pav_config_dir/suites/bare_yaml.yaml b/test/data/pav_config_dir/suites/bare_yaml.yaml new file mode 100644 index 000000000..8aa715440 --- /dev/null +++ b/test/data/pav_config_dir/suites/bare_yaml.yaml @@ -0,0 +1,4 @@ +bare_yaml: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index c00810073..f0369a529 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -13,7 +13,7 @@ def setUp(self): def test_run_from_suite_directory(self): """Test that Pavilion can find and run a test from - suites directory.""" + a named suites directory.""" arg_parser = arguments.get_parser() args = arg_parser.parse_args([ @@ -25,3 +25,18 @@ def test_run_from_suite_directory(self): run_cmd = commands.get_command(args.command_name) self.assertEqual(run_cmd.run(self.pav_cfg, args), 0) + + def test_run_from_bare_yaml(self): + """Test that Pavilion can find and run a test from + a bare yaml file in the suites directory.""" + + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + '-H', 'this', + 'bare_yaml' + ]) + + run_cmd = commands.get_command(args.command_name) + + self.assertEqual(run_cmd.run(self.pav_cfg, args), 0) From 99fd8e906693136e3ec530cfa9f19ed105528dfc Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sat, 17 Aug 2024 15:57:01 -0600 Subject: [PATCH 17/71] Stub out additional suites tests --- test/tests/suites_tests.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index f0369a529..881e95d3d 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -11,7 +11,7 @@ def setUp(self): run_cmd = commands.get_command('run') # run_cmd.silence() - def test_run_from_suite_directory(self): + def test_suite_run_from_suite_directory(self): """Test that Pavilion can find and run a test from a named suites directory.""" @@ -26,7 +26,7 @@ def test_run_from_suite_directory(self): self.assertEqual(run_cmd.run(self.pav_cfg, args), 0) - def test_run_from_bare_yaml(self): + def test_suite_run_from_bare_yaml(self): """Test that Pavilion can find and run a test from a bare yaml file in the suites directory.""" @@ -40,3 +40,27 @@ def test_run_from_bare_yaml(self): run_cmd = commands.get_command(args.command_name) self.assertEqual(run_cmd.run(self.pav_cfg, args), 0) + + def test_suites_host_config(self): + """Test that Pavilion loads host configs from the + suites directory""" + + self.assertTrue(False) + + def test_suites_mode_config(self): + """Test that Pavilion loads mode configs from the + suites directory""" + + self.assertTrue(False) + + def test_suites_os_config(self): + """Test that Pavilion loads OS configs from the + suites directory""" + + self.assertTrue(False) + + def test_suites_build_hash(self): + """Test that Pavilion ignores config files in the + suites directory when computing the build hash.""" + + self.assertTrue(False) From b2cce4d15f1c305f35ecd8f7aff4aecfae5b489e Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Sun, 18 Aug 2024 17:08:30 -0600 Subject: [PATCH 18/71] Refactor find_config --- lib/pavilion/resolver/resolver.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index c436d9a19..1f395d42e 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -112,10 +112,19 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } - def _make_suite_paths(self, config: Dict, conf_name: str) -> List[Path]: - root = config['path'] / 'suites' + def _get_conf_paths(self, config_path: Path, conf_type: str, conf_name: str) -> List[Path]: + """Return a list of all locations in which the given config type could + potentially be, given a particular config path.""" - return [root / f'{conf_name}.yaml', root / conf_name / 'suite.yaml'] + suite_dir = config_path / 'suites' / conf_name + conf_dir = config_path / self.CONF_TYPE_DIRNAMES[conf_type] + + paths = [suite_dir / f'{conf_type}.yaml', conf_dir / f'{conf_name}.yaml'] + + if conf_type == 'suite': + paths.insert(0, config_path / 'suites' / f'{conf_name}.yaml') + + return paths def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: """Search all of the known configuration directories for a config of the @@ -127,16 +136,9 @@ def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: and the path to that config. If nothing was found, returns (None, None). """ - conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] - for label, config in self.pav_cfg.configs.items(): - cfg_path = config['path'] / conf_dir /'{}.yaml'.format(conf_name) - cfg_paths = [cfg_path] - - if conf_type == 'suite': - cfg_paths.extend(self._make_suite_paths(config, conf_name)) - - path = first(lambda x: x.exists(), cfg_paths) + path = first(lambda x: x.exists(), + self._get_conf_paths(config['path'], conf_type, conf_name)) if path is not None: return label, path From b8fa001496077f5f79538c0202f94847623c8257 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 19 Aug 2024 10:25:29 -0600 Subject: [PATCH 19/71] Miscellaneous changes --- lib/pavilion/resolver/resolver.py | 32 +++++++++++++------ .../suites/hosts_suite_test/hosts.yaml | 7 ++++ .../suites/hosts_suite_test/suite.yaml | 4 +++ .../suites/modes_suite_test/modes.yaml | 7 ++++ .../suites/modes_suite_test/suite.yaml | 4 +++ .../suites/os_suite_test/os.yaml | 7 ++++ .../suites/os_suite_test/suite.yaml | 4 +++ test/tests/suites_tests.py | 18 +++++++++++ 8 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 test/data/pav_config_dir/suites/hosts_suite_test/hosts.yaml create mode 100644 test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml create mode 100644 test/data/pav_config_dir/suites/modes_suite_test/modes.yaml create mode 100644 test/data/pav_config_dir/suites/modes_suite_test/suite.yaml create mode 100644 test/data/pav_config_dir/suites/os_suite_test/os.yaml create mode 100644 test/data/pav_config_dir/suites/os_suite_test/suite.yaml diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 1f395d42e..c7eb75167 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -112,21 +112,33 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } - def _get_conf_paths(self, config_path: Path, conf_type: str, conf_name: str) -> List[Path]: + def _get_suite_dir(self, config_path: Path, suite_name: str) -> Path: + """Return the path to the individual suite.""" + + def _get_conf_paths(self, config_path: Path, suite_name: str, conf_type: str, conf_name: str) -> List[Path]: """Return a list of all locations in which the given config type could potentially be, given a particular config path.""" - suite_dir = config_path / 'suites' / conf_name - conf_dir = config_path / self.CONF_TYPE_DIRNAMES[conf_type] + paths = [] + + conf_dir_name = self.CONF_TYPE_DIRNAMES[conf_type] + + conf_dir_path = config_path / conf_dir_name + suite_dir = config_path / "suites" / suite_name + + paths.append(suite_dir / f"{conf_type}.yaml") - paths = [suite_dir / f'{conf_type}.yaml', conf_dir / f'{conf_name}.yaml'] + if conf_dir_name in ('hosts', 'modes', 'os'): + paths.append(suite_dir / f"{conf_dir_name}.yaml") + elif conf_type == 'suite': + paths.append(suite_dir / 'suite.yaml') - if conf_type == 'suite': - paths.insert(0, config_path / 'suites' / f'{conf_name}.yaml') + # Old suite path (deprecated) + paths.append(conf_dir_path / f"{conf_name}.yaml") return paths - def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: + def find_config(self, suite_name: str, conf_type: str, conf_name: str) -> Tuple[str, Path]: """Search all of the known configuration directories for a config of the given type and name. @@ -137,8 +149,8 @@ def find_config(self, conf_type: str, conf_name: str) -> Tuple[str, Path]: """ for label, config in self.pav_cfg.configs.items(): - path = first(lambda x: x.exists(), - self._get_conf_paths(config['path'], conf_type, conf_name)) + potential_paths = self._get_conf_paths(config['path'], suite_name, conf_type, conf_name) + path = first(lambda x: x.exists(), potential_paths) if path is not None: return label, path @@ -536,7 +548,7 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: return multiplied_tests def _load_raw_config(self, name: str, config_type: str, optional=False) \ - -> Tuple[Any, Union[Path, None], Union[str, None]]: + -> Tuple[Dict, Union[Path, None], Union[str, None]]: """Load the given raw test config file. It can be a host, mode, or suite file. Returns a tuple of the config, path, and config label (name of the config area). """ diff --git a/test/data/pav_config_dir/suites/hosts_suite_test/hosts.yaml b/test/data/pav_config_dir/suites/hosts_suite_test/hosts.yaml new file mode 100644 index 000000000..5fd87ad2e --- /dev/null +++ b/test/data/pav_config_dir/suites/hosts_suite_test/hosts.yaml @@ -0,0 +1,7 @@ +host1: + variables: + host1: True + +host2: + variables: + host2: True diff --git a/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml b/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml new file mode 100644 index 000000000..24205bb82 --- /dev/null +++ b/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml @@ -0,0 +1,4 @@ +basic_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/data/pav_config_dir/suites/modes_suite_test/modes.yaml b/test/data/pav_config_dir/suites/modes_suite_test/modes.yaml new file mode 100644 index 000000000..c14c31d04 --- /dev/null +++ b/test/data/pav_config_dir/suites/modes_suite_test/modes.yaml @@ -0,0 +1,7 @@ +mode1: + variables: + mode1: True + +mode2: + variables: + mode2: True diff --git a/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml b/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml new file mode 100644 index 000000000..1e33457b7 --- /dev/null +++ b/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml @@ -0,0 +1,4 @@ +mode_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/data/pav_config_dir/suites/os_suite_test/os.yaml b/test/data/pav_config_dir/suites/os_suite_test/os.yaml new file mode 100644 index 000000000..130918f31 --- /dev/null +++ b/test/data/pav_config_dir/suites/os_suite_test/os.yaml @@ -0,0 +1,7 @@ +os1: + variables: + os1: True + +os2: + variables: + os2: True diff --git a/test/data/pav_config_dir/suites/os_suite_test/suite.yaml b/test/data/pav_config_dir/suites/os_suite_test/suite.yaml new file mode 100644 index 000000000..24205bb82 --- /dev/null +++ b/test/data/pav_config_dir/suites/os_suite_test/suite.yaml @@ -0,0 +1,4 @@ +basic_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index 881e95d3d..c17ae18ae 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -9,6 +9,7 @@ class SuitesTests(PavTestCase): def setUp(self): plugins.initialize_plugins(self.pav_cfg) run_cmd = commands.get_command('run') + build_cmd = commands.get_command('build') # run_cmd.silence() def test_suite_run_from_suite_directory(self): @@ -45,6 +46,23 @@ def test_suites_host_config(self): """Test that Pavilion loads host configs from the suites directory""" + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + '-H', 'host1', + 'hosts_suite_test' + ]) + + run_cmd = commands.get_command(args.command_name) + # How do I resolve the configs without building the test? + ret = run_cmd.run(self.pav_cfg, args) + + self.assertEqual(ret, 0) + + last_test = run_cmd.last_tests[0] + + import pdb; pdb.set_trace() + self.assertTrue(False) def test_suites_mode_config(self): From ca32ae223da6a2246011a86b0e0248df973f931f Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 19 Aug 2024 11:49:58 -0600 Subject: [PATCH 20/71] Further progress --- lib/pavilion/resolver/resolver.py | 49 ++++++++++++++++--------------- test/tests/suites_tests.py | 2 -- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index c7eb75167..cb3d2b3e2 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -18,7 +18,7 @@ import sys from collections import defaultdict from pathlib import Path -from typing import List, IO, Dict, Tuple, NewType, Union, Any, Iterator, TextIO +from typing import List, IO, Dict, Tuple, NewType, Union, Any, Iterator, TextIO, Optional import similarity import yc_yaml @@ -115,7 +115,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, def _get_suite_dir(self, config_path: Path, suite_name: str) -> Path: """Return the path to the individual suite.""" - def _get_conf_paths(self, config_path: Path, suite_name: str, conf_type: str, conf_name: str) -> List[Path]: + def _get_conf_paths(self, config_path: Path, suite_name: Optional[str], conf_type: str, conf_name: str) -> List[Path]: """Return a list of all locations in which the given config type could potentially be, given a particular config path.""" @@ -124,21 +124,22 @@ def _get_conf_paths(self, config_path: Path, suite_name: str, conf_type: str, co conf_dir_name = self.CONF_TYPE_DIRNAMES[conf_type] conf_dir_path = config_path / conf_dir_name - suite_dir = config_path / "suites" / suite_name - paths.append(suite_dir / f"{conf_type}.yaml") + if suite_name is not None: + suite_dir = config_path / "suites" / suite_name + paths.append(suite_dir / f"{conf_type}.yaml") - if conf_dir_name in ('hosts', 'modes', 'os'): - paths.append(suite_dir / f"{conf_dir_name}.yaml") - elif conf_type == 'suite': - paths.append(suite_dir / 'suite.yaml') + if conf_dir_name in ('hosts', 'modes', 'os'): + paths.append(suite_dir / f"{conf_dir_name}.yaml") + elif conf_type == 'suite': + paths.append(suite_dir / 'suite.yaml') # Old suite path (deprecated) paths.append(conf_dir_path / f"{conf_name}.yaml") return paths - def find_config(self, suite_name: str, conf_type: str, conf_name: str) -> Tuple[str, Path]: + def find_config(self, conf_type: str, conf_name: str, suite_name: str = None) -> Tuple[str, Path]: """Search all of the known configuration directories for a config of the given type and name. @@ -547,7 +548,7 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: return multiplied_tests - def _load_raw_config(self, name: str, config_type: str, optional=False) \ + def _load_raw_config(self, config_name: str, config_type: str, suite_name: str = None, optional=False) \ -> Tuple[Dict, Union[Path, None], Union[str, None]]: """Load the given raw test config file. It can be a host, mode, or suite file. Returns a tuple of the config, path, and config label (name of the config area). @@ -560,30 +561,30 @@ def _load_raw_config(self, name: str, config_type: str, optional=False) \ else: raise RuntimeError("Unknown config type: '{}'".format(config_type)) - cfg_label, path = self.find_config(config_type, name) + cfg_label, path = self.find_config(config_type, config_name, suite_name) if path is None: if optional: return None, None, None # Give a special message if it looks like they got their commands mixed up. - if config_type == 'suite' and name == 'log': + if config_type == 'suite' and config_name == 'log': raise TestConfigError( "Could not find test suite 'log'. Were you trying to run `pav log run`?") - similar = self.find_similar_configs(config_type, name) + similar = self.find_similar_configs(config_type, config_name) show_type = 'test' if config_type == 'suite' else config_type if similar: raise TestConfigError( "Could not find {} config {}.yaml.\n" "Did you mean one of these? {}" - .format(config_type, name, ', '.join(similar))) + .format(config_type, config_name, ', '.join(similar))) else: raise TestConfigError( "Could not find {0} config file '{1}.yaml' in any of the " "Pavilion config directories.\n" "Run `pav show {2}` to get a list of available {0} files." - .format(config_type, name, show_type)) + .format(config_type, config_name, show_type)) try: with path.open() as cfg_file: raw_cfg = loader.load_raw(cfg_file) @@ -666,7 +667,7 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], # Apply modes. try: - test_cfg = self.apply_modes(test_cfg, modes) + test_cfg = self.apply_modes(test_cfg, modes, request.suite) except TestConfigError as err: err.request = request self.errors.append(err) @@ -764,13 +765,13 @@ def _load_base_config(self, op_sys, host) -> Dict: base_config = self.apply_os(base_config, op_sys) return self.apply_host(base_config, host) - def _load_suite_tests(self, suite: str): + def _load_suite_tests(self, suite_name: str): """Load the suite config, with standard info applied to """ - if suite in self._suites: - return self._suites[suite] + if suite_name in self._suites: + return self._suites[suite_name] - raw_suite_cfg, suite_path, cfg_label = self._load_raw_config(suite, 'suite') + raw_suite_cfg, suite_path, cfg_label = self._load_raw_config(suite_name, 'suite', suite_name) # Make sure each test has a dict as contents. for test_name, raw_test in raw_suite_cfg.items(): if raw_test is None: @@ -786,12 +787,12 @@ def _load_suite_tests(self, suite: str): test_cfg['cfg_label'] = cfg_label working_dir = self.pav_cfg['configs'][cfg_label]['working_dir'] test_cfg['working_dir'] = working_dir.as_posix() - test_cfg['suite'] = suite + test_cfg['suite'] = suite_name test_cfg['suite_path'] = suite_path.as_posix() test_cfg['host'] = self._host test_cfg['os'] = self._os - self._suites[suite] = suite_tests + self._suites[suite_name] = suite_tests return suite_tests def _reset_schedulers(self): @@ -907,7 +908,7 @@ def apply_os(self, test_cfg, op_sys): raise TestConfigError( "Error merging configuration for OS '{}'".format(os)) - def apply_modes(self, test_cfg, modes: List[str]): + def apply_modes(self, test_cfg, modes: List[str], suite_name: str): """Apply each of the mode files to the given test config. :param test_cfg: The raw test configuration. @@ -917,7 +918,7 @@ def apply_modes(self, test_cfg, modes: List[str]): loader = self._loader for mode in modes: - raw_mode_cfg, mode_cfg_path, _ = self._load_raw_config(mode, 'mode') + raw_mode_cfg, mode_cfg_path, _ = self._load_raw_config(mode, 'mode', suite_name) try: mode_cfg = loader.normalize( raw_mode_cfg, diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index c17ae18ae..d6b060371 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -61,8 +61,6 @@ def test_suites_host_config(self): last_test = run_cmd.last_tests[0] - import pdb; pdb.set_trace() - self.assertTrue(False) def test_suites_mode_config(self): From f09c8515274361895c431fa4dac7ff8a6dd70447 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 19 Aug 2024 13:15:00 -0600 Subject: [PATCH 21/71] Fix regression --- lib/pavilion/resolver/resolver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index cb3d2b3e2..bdc90b0e5 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -122,17 +122,17 @@ def _get_conf_paths(self, config_path: Path, suite_name: Optional[str], conf_typ paths = [] conf_dir_name = self.CONF_TYPE_DIRNAMES[conf_type] - conf_dir_path = config_path / conf_dir_name + suites_cfg_dir = config_path / "suites" if suite_name is not None: - suite_dir = config_path / "suites" / suite_name + suite_dir = suites_cfg_dir / suite_name paths.append(suite_dir / f"{conf_type}.yaml") if conf_dir_name in ('hosts', 'modes', 'os'): paths.append(suite_dir / f"{conf_dir_name}.yaml") elif conf_type == 'suite': - paths.append(suite_dir / 'suite.yaml') + paths.append(suites_cfg_dir / f"{suite_name}.yaml") # Old suite path (deprecated) paths.append(conf_dir_path / f"{conf_name}.yaml") @@ -151,6 +151,7 @@ def find_config(self, conf_type: str, conf_name: str, suite_name: str = None) -> for label, config in self.pav_cfg.configs.items(): potential_paths = self._get_conf_paths(config['path'], suite_name, conf_type, conf_name) + path = first(lambda x: x.exists(), potential_paths) if path is not None: From 38aa5386630a9d6b9544f2b89a089f602ab50ca2 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 20 Aug 2024 12:33:04 -0600 Subject: [PATCH 22/71] Progress towards proper parsing of host config --- lib/pavilion/resolver/resolver.py | 46 +++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index bdc90b0e5..c418ece1a 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -37,7 +37,7 @@ from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary from pavilion.func_utils import first, apply_to_first -from yaml_config import RequiredError +from yaml_config import RequiredError, YamlConfigLoader from .proto_test import RawProtoTest, ProtoTest from .request import TestRequest @@ -112,9 +112,6 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } - def _get_suite_dir(self, config_path: Path, suite_name: str) -> Path: - """Return the path to the individual suite.""" - def _get_conf_paths(self, config_path: Path, suite_name: Optional[str], conf_type: str, conf_name: str) -> List[Path]: """Return a list of all locations in which the given config type could potentially be, given a particular config path.""" @@ -346,10 +343,6 @@ def load_iter(self, tests: List[str], modes: List[str] = None, overrides: List[s if modes is None: modes = [] - if overrides is None: - overrides = [] - - if overrides is None: overrides = [] @@ -549,20 +542,28 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: return multiplied_tests + def _get_loader(self, cfg_path: Optional[Path], cfg_type: str) -> YamlConfigLoader: + """Return the appropriate loader for the given path and config type.""" + + if cfg_type == 'suite': + return TestSuiteLoader() + elif cfg_type.lower() in ('host', 'os', 'mode'): + if cfg_path is not None and cfg_path.parents[1].stem == 'suites': + return TestSuiteLoader() + + return self._loader + else: + raise RuntimeError("Unknown config type: '{}'".format(config_type)) + def _load_raw_config(self, config_name: str, config_type: str, suite_name: str = None, optional=False) \ -> Tuple[Dict, Union[Path, None], Union[str, None]]: """Load the given raw test config file. It can be a host, mode, or suite file. Returns a tuple of the config, path, and config label (name of the config area). """ - if config_type in ('host', 'OS', 'mode'): - loader = self._loader - elif config_type == 'suite': - loader = TestSuiteLoader() - else: - raise RuntimeError("Unknown config type: '{}'".format(config_type)) - cfg_label, path = self.find_config(config_type, config_name, suite_name) + loader = self._get_loader(path, config_type) + if path is None: if optional: @@ -669,6 +670,8 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], # Apply modes. try: test_cfg = self.apply_modes(test_cfg, modes, request.suite) + test_cfg = self.apply_host(test_cfg, self._host, request.suite) + test_cfg = self.apply_os(test_cfg, self._os, request.suite) except TestConfigError as err: err.request = request self.errors.append(err) @@ -863,12 +866,13 @@ def check_version_compatibility(self, test_cfg): "Incompatible with pavilion version '{}', compatible versions " "'{}'.".format(PavVars()['version'], comp_versions)) - def apply_host(self, test_cfg, host): + def apply_host(self, test_cfg, host, suite_name: str = None): """Apply the host configuration to the given config.""" - loader = self._loader + raw_host_cfg, host_cfg_path, _ = self._load_raw_config(host, 'host', suite_name, optional=True) + + loader = self._get_loader(host_cfg_path, 'host') - raw_host_cfg, host_cfg_path, _ = self._load_raw_config(host, 'host', optional=True) if raw_host_cfg is None: return test_cfg @@ -886,12 +890,12 @@ def apply_host(self, test_cfg, host): raise TestConfigError( "Error merging host configuration for host '{}'".format(host)) - def apply_os(self, test_cfg, op_sys): + def apply_os(self, test_cfg, op_sys, suite_name: str = None): """Apply the OS configuration to the given config.""" loader = self._loader - raw_os_cfg, os_cfg_path, _ = self._load_raw_config(op_sys, 'OS', optional=True) + raw_os_cfg, os_cfg_path, _ = self._load_raw_config(op_sys, 'OS', suite_name, optional=True) if raw_os_cfg is None: return test_cfg @@ -901,7 +905,7 @@ def apply_os(self, test_cfg, op_sys): root_name=f"the top level of the OS file.") except (KeyError, ValueError) as err: raise TestConfigError( - f"Error loading host config '{op_sys}' from file '{os_cfg_path}'.") + f"Error loading host config '{op_sys}' from file '{os_cfg_path}'") try: return loader.merge(test_cfg, os_cfg) From a752ace077ec127317086051cdb22df355d757e1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 22 Aug 2024 15:29:51 -0600 Subject: [PATCH 23/71] Progress towards solution --- lib/pavilion/resolver/resolver.py | 95 ++++++++++++++----------------- 1 file changed, 43 insertions(+), 52 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index c418ece1a..1f947ad2d 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -112,7 +112,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } - def _get_conf_paths(self, config_path: Path, suite_name: Optional[str], conf_type: str, conf_name: str) -> List[Path]: + def _get_cfg_paths(self, config_path: Path, suite_name: Optional[str], conf_type: str, conf_name: str) -> List[Path]: """Return a list of all locations in which the given config type could potentially be, given a particular config path.""" @@ -147,7 +147,7 @@ def find_config(self, conf_type: str, conf_name: str, suite_name: str = None) -> """ for label, config in self.pav_cfg.configs.items(): - potential_paths = self._get_conf_paths(config['path'], suite_name, conf_type, conf_name) + potential_paths = self._get_cfg_paths(config['path'], suite_name, conf_type, conf_name) path = first(lambda x: x.exists(), potential_paths) @@ -542,76 +542,48 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: return multiplied_tests - def _get_loader(self, cfg_path: Optional[Path], cfg_type: str) -> YamlConfigLoader: - """Return the appropriate loader for the given path and config type.""" - - if cfg_type == 'suite': - return TestSuiteLoader() - elif cfg_type.lower() in ('host', 'os', 'mode'): - if cfg_path is not None and cfg_path.parents[1].stem == 'suites': - return TestSuiteLoader() - - return self._loader - else: - raise RuntimeError("Unknown config type: '{}'".format(config_type)) - - def _load_raw_config(self, config_name: str, config_type: str, suite_name: str = None, optional=False) \ - -> Tuple[Dict, Union[Path, None], Union[str, None]]: - """Load the given raw test config file. It can be a host, mode, or suite file. - Returns a tuple of the config, path, and config label (name of the config area). - """ - - cfg_label, path = self.find_config(config_type, config_name, suite_name) - loader = self._get_loader(path, config_type) + def _load_raw_config(self, cfg_path: Optional[Path], cfg_type: str, loader: yc.YamlConfigLoader) -> Dict: + # TODO: docstring if path is None: + similar = self.find_similar_configs(cfg_type, config_name) - if optional: - return None, None, None - - # Give a special message if it looks like they got their commands mixed up. - if config_type == 'suite' and config_name == 'log': - raise TestConfigError( - "Could not find test suite 'log'. Were you trying to run `pav log run`?") - - similar = self.find_similar_configs(config_type, config_name) - show_type = 'test' if config_type == 'suite' else config_type if similar: raise TestConfigError( "Could not find {} config {}.yaml.\n" "Did you mean one of these? {}" - .format(config_type, config_name, ', '.join(similar))) + .format(cfg_type, config_name, ', '.join(similar))) else: raise TestConfigError( "Could not find {0} config file '{1}.yaml' in any of the " "Pavilion config directories.\n" "Run `pav show {2}` to get a list of available {0} files." - .format(config_type, config_name, show_type)) + .format(cfg_type, config_name, cfg_type)) try: with path.open() as cfg_file: raw_cfg = loader.load_raw(cfg_file) except (IOError, OSError) as err: raise TestConfigError("Could not open {} config '{}'" - .format(config_type, path), prior_error=err) + .format(cfg_type, path), prior_error=err) except ValueError as err: raise TestConfigError( "{} config '{}' has invalid value." - .format(config_type.capitalize(), path), prior_error=err) + .format(cfg_type.capitalize(), path), prior_error=err) except KeyError as err: raise TestConfigError( "{} config '{}' has an invalid key." - .format(config_type.capitalize(), path), prior_error=err) + .format(cfg_type.capitalize(), path), prior_error=err) except yc_yaml.YAMLError as err: raise TestConfigError( "{} config '{}' has a YAML Error" - .format(config_type.capitalize(), path), prior_error=err) + .format(cfg_type.capitalize(), path), prior_error=err) except TypeError as err: raise TestConfigError( "Structural issue with {} config '{}'" - .format(config_type, path), prior_error=err) + .format(cfg_type, path), prior_error=err) - return raw_cfg, path, cfg_label + return raw_cfg def _load_raw_configs(self, request: TestRequest, modes: List[str], @@ -703,6 +675,7 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], test_cfg['result_evaluate'][key] = '"{}"'.format(const) + import pdb; pdb.set_trace() test_cfg = self._validate(test_name, test_cfg) # Now that we've applied all general transforms to the config, make it into a ProtoTest. @@ -775,28 +748,33 @@ def _load_suite_tests(self, suite_name: str): if suite_name in self._suites: return self._suites[suite_name] - raw_suite_cfg, suite_path, cfg_label = self._load_raw_config(suite_name, 'suite', suite_name) + suite_cfg_path = self.find_config(suite_name, "suite", suite_name) + loader = TestSuiteLoader() + + raw_suite_cfg, = self._load_raw_config(suite_cfg_path, "suite") + # Make sure each test has a dict as contents. for test_name, raw_test in raw_suite_cfg.items(): if raw_test is None: raw_suite_cfg[test_name] = {} - suite_tests = self.resolve_inheritance(raw_suite_cfg, suite_path) + suite_tests = self.resolve_inheritance(raw_suite_cfg, suite_cfg_path) # Perform essential transformations to each test config. for test_cfg_name, test_cfg in list(suite_tests.items()): # Basic information that all test configs should have. test_cfg['name'] = test_cfg_name - test_cfg['cfg_label'] = cfg_label + # test_cfg['cfg_label'] = cfg_label working_dir = self.pav_cfg['configs'][cfg_label]['working_dir'] test_cfg['working_dir'] = working_dir.as_posix() test_cfg['suite'] = suite_name - test_cfg['suite_path'] = suite_path.as_posix() + test_cfg['suite_path'] = suite_cfg_path.as_posix() test_cfg['host'] = self._host test_cfg['os'] = self._os self._suites[suite_name] = suite_tests + return suite_tests def _reset_schedulers(self): @@ -866,12 +844,21 @@ def check_version_compatibility(self, test_cfg): "Incompatible with pavilion version '{}', compatible versions " "'{}'.".format(PavVars()['version'], comp_versions)) + def _get_loader(suite_name: Optional[str]) -> yc.TestConfigLoader: + """Given a suite_name, return the appropriate loader.""" + + if suite_name is None: + return TestConfigLoader() + + return loader = TestSuiteLoader() + def apply_host(self, test_cfg, host, suite_name: str = None): """Apply the host configuration to the given config.""" - raw_host_cfg, host_cfg_path, _ = self._load_raw_config(host, 'host', suite_name, optional=True) + host_cfg_path = self._get_cfg_paths(host, "host", suite_name) + loader = self._get_loader(suite_name) - loader = self._get_loader(host_cfg_path, 'host') + raw_host_cfg = self._load_raw_config(host_cfg_path, loader) if raw_host_cfg is None: return test_cfg @@ -893,9 +880,11 @@ def apply_host(self, test_cfg, host, suite_name: str = None): def apply_os(self, test_cfg, op_sys, suite_name: str = None): """Apply the OS configuration to the given config.""" - loader = self._loader + os_cfg_path = self._get_cfg_paths(op_sys, "OS", suite_name) + loader = self._get_loader(suite_name) + + raw_os_cfg = self._load_raw_config(os_cfg_path, loader) - raw_os_cfg, os_cfg_path, _ = self._load_raw_config(op_sys, 'OS', suite_name, optional=True) if raw_os_cfg is None: return test_cfg @@ -913,17 +902,19 @@ def apply_os(self, test_cfg, op_sys, suite_name: str = None): raise TestConfigError( "Error merging configuration for OS '{}'".format(os)) - def apply_modes(self, test_cfg, modes: List[str], suite_name: str): + def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): """Apply each of the mode files to the given test config. :param test_cfg: The raw test configuration. :param modes: A list of mode names. """ - loader = self._loader + loader = self._get_loader(suite_name) for mode in modes: - raw_mode_cfg, mode_cfg_path, _ = self._load_raw_config(mode, 'mode', suite_name) + mode_cfg_path = self._find_config(mode, "mode", suite_name) + raw_mode_cfg = self._load_raw_config(mode_cfg_path, loader) + try: mode_cfg = loader.normalize( raw_mode_cfg, From 731b3e288a076125c7a0c96d7d4d048549b6d424 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 23 Aug 2024 13:39:57 -0600 Subject: [PATCH 24/71] Further progress towards solution --- lib/pavilion/resolver/resolver.py | 62 ++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 1f947ad2d..8837e4732 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -51,6 +51,7 @@ TEST_VERS_RE = re.compile(r'^\d+(\.\d+){0,2}$') +TestConfig = Dict class TestConfigResolver: """Converts raw test configurations into their final, fully resolved @@ -541,24 +542,19 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: resolved_tests = remaining return multiplied_tests + + def _normalize_cfg_type(self, cfg_type: str) -> str: + cfg_type = cfg_type.lower() - def _load_raw_config(self, cfg_path: Optional[Path], cfg_type: str, loader: yc.YamlConfigLoader) -> Dict: - # TODO: docstring + if not cfg_type[-1] != 's' and cfg_type != 'os': + cfg_type += 's' - if path is None: - similar = self.find_similar_configs(cfg_type, config_name) + return cfg_type + + def _safe_load_config(self, path: Path, loader: yc.YamlConfigLoader) -> TestConfig: + """Given a path to a config, load the config, and raise an appropriate + error if it can't be loaded""" - if similar: - raise TestConfigError( - "Could not find {} config {}.yaml.\n" - "Did you mean one of these? {}" - .format(cfg_type, config_name, ', '.join(similar))) - else: - raise TestConfigError( - "Could not find {0} config file '{1}.yaml' in any of the " - "Pavilion config directories.\n" - "Run `pav show {2}` to get a list of available {0} files." - .format(cfg_type, config_name, cfg_type)) try: with path.open() as cfg_file: raw_cfg = loader.load_raw(cfg_file) @@ -577,7 +573,6 @@ def _load_raw_config(self, cfg_path: Optional[Path], cfg_type: str, loader: yc.Y raise TestConfigError( "{} config '{}' has a YAML Error" .format(cfg_type.capitalize(), path), prior_error=err) - except TypeError as err: raise TestConfigError( "Structural issue with {} config '{}'" @@ -585,6 +580,39 @@ def _load_raw_config(self, cfg_path: Optional[Path], cfg_type: str, loader: yc.Y return raw_cfg + def _load_from_suite(self, suite_dir: Path, cfg_type: str, cfg_name: str, loader: yc.YamlConfigLoader) -> TestConfig: + cfg_type = self._normalize_cfg_type(cfg_type) + cfg_path = suite_dir / f"{cfg_type}.yaml" + + raw_cfg = self._safe_load_config(cfg_path) + + try: + return raw_cfg[cfg_name] + except KeyError: + short_path = f"{cfg_path.parents[0].name}/{cfg_path.name}" + raise TestConfigError( + f"Config name {cfg_name} not found in {short_path}" + ) + + def _load_raw_config(self, cfg_path: Optional[Path], cfg_type: str, loader: yc.YamlConfigLoader) -> TestConfig: + # TODO: docstring + + if path is None: + similar = self.find_similar_configs(cfg_type, config_name) + + if similar: + raise TestConfigError( + "Could not find {} config {}.yaml.\n" + "Did you mean one of these? {}" + .format(cfg_type, config_name, ', '.join(similar))) + else: + raise TestConfigError( + "Could not find {0} config file '{1}.yaml' in any of the " + "Pavilion config directories.\n" + "Run `pav show {2}` to get a list of available {0} files." + .format(cfg_type, config_name, cfg_type)) + + return self._safe_load_config(cfg_path) def _load_raw_configs(self, request: TestRequest, modes: List[str], conditions: Dict, overrides: List[str]) \ @@ -912,7 +940,7 @@ def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): loader = self._get_loader(suite_name) for mode in modes: - mode_cfg_path = self._find_config(mode, "mode", suite_name) + mode_cfg_path = self.find_config(mode, "mode", suite_name) raw_mode_cfg = self._load_raw_config(mode_cfg_path, loader) try: From 12dc0ce2416507c4da4832b8d112a3d77ba3082c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 23 Aug 2024 15:00:44 -0600 Subject: [PATCH 25/71] =?UTF-8?q?Further=C2=A0progress?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pavilion/func_utils.py | 5 ++++ lib/pavilion/resolver/resolver.py | 47 ++++++++++++++----------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py index 18b9628b0..dcada9bdd 100644 --- a/lib/pavilion/func_utils.py +++ b/lib/pavilion/func_utils.py @@ -59,3 +59,8 @@ def get_nested(keys: Iterable[Hashable], nested_dict: Dict) -> Dict: nested_dict = nested_dict.get(key, {}) return nested_dict + +def listmap(func: Callable[[T], U], lst: Iterable[T]) -> List[U]: + """Map a function over an iterable, but return a list instead + of an iterator.""" + return list(map(func, lst)) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 8837e4732..c1cab5cc2 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -36,7 +36,7 @@ KEY_NAME_RE) from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary -from pavilion.func_utils import first, apply_to_first +from pavilion.func_utils import first, apply_to_first, listmap from yaml_config import RequiredError, YamlConfigLoader from .proto_test import RawProtoTest, ProtoTest @@ -72,6 +72,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, self._outfile = io.StringIO() if outfile is None else outfile self._verbosity = verbosity self._loader = TestConfigLoader() + self._suite_loader = TestSuiteLoader() self.errors = [] self._base_var_man = variables.VariableSetManager() @@ -113,29 +114,24 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, 'modes': 'modes', } - def _get_cfg_paths(self, config_path: Path, suite_name: Optional[str], conf_type: str, conf_name: str) -> List[Path]: - """Return a list of all locations in which the given config type could - potentially be, given a particular config path.""" + @property + def config_paths(self) -> Iterator[Path]: + return map(lambda x: x["path"], self.pav_cfg.configs.values()) - paths = [] - - conf_dir_name = self.CONF_TYPE_DIRNAMES[conf_type] - conf_dir_path = config_path / conf_dir_name - suites_cfg_dir = config_path / "suites" + @property + def suites_dirs(self) -> Iterator[Path]: + return map(lambda x: x / "suites", self.config_paths) - if suite_name is not None: - suite_dir = suites_cfg_dir / suite_name - paths.append(suite_dir / f"{conf_type}.yaml") + def _cfg_path_from_suite(self, suite_name: str, conf_type: str) -> Optional[Path]: + paths = [] - if conf_dir_name in ('hosts', 'modes', 'os'): - paths.append(suite_dir / f"{conf_dir_name}.yaml") - elif conf_type == 'suite': - paths.append(suites_cfg_dir / f"{suite_name}.yaml") + if conf_type == 'suite': + paths.append(listmap(lambda x: x / f"{suite_name}.yaml", self.suites_dirs)) - # Old suite path (deprecated) - paths.append(conf_dir_path / f"{conf_name}.yaml") + cfg_name = self._normalize_cfg_type(conf_type) + paths.append(listmap(lambda x: x / suite_name / f"{cfg_name}.yaml")) - return paths + return first(paths) def find_config(self, conf_type: str, conf_name: str, suite_name: str = None) -> Tuple[str, Path]: """Search all of the known configuration directories for a config of the @@ -147,10 +143,11 @@ def find_config(self, conf_type: str, conf_name: str, suite_name: str = None) -> and the path to that config. If nothing was found, returns (None, None). """ + suite_path = first(lambda x: x.exists(), self.suites_dirs) + for label, config in self.pav_cfg.configs.items(): potential_paths = self._get_cfg_paths(config['path'], suite_name, conf_type, conf_name) - path = first(lambda x: x.exists(), potential_paths) if path is not None: return label, path @@ -228,7 +225,7 @@ def find_all_tests(self): # may have been written to require a real host/mode file. with file.open('r') as suite_file: try: - suite_cfg = TestSuiteLoader().load(suite_file, partial=True) + suite_cfg = self._suite_loader.load(suite_file, partial=True) except (TypeError, KeyError, ValueError, @@ -777,9 +774,7 @@ def _load_suite_tests(self, suite_name: str): return self._suites[suite_name] suite_cfg_path = self.find_config(suite_name, "suite", suite_name) - loader = TestSuiteLoader() - - raw_suite_cfg, = self._load_raw_config(suite_cfg_path, "suite") + raw_suite_cfg = self._load_raw_config(suite_cfg_path, "suite") # Make sure each test has a dict as contents. for test_name, raw_test in raw_suite_cfg.items(): @@ -876,9 +871,9 @@ def _get_loader(suite_name: Optional[str]) -> yc.TestConfigLoader: """Given a suite_name, return the appropriate loader.""" if suite_name is None: - return TestConfigLoader() + return self._loader - return loader = TestSuiteLoader() + return self._suite_loader def apply_host(self, test_cfg, host, suite_name: str = None): """Apply the host configuration to the given config.""" From dd9a8382cef73a6ba132df6690f519d03d366ee4 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 26 Aug 2024 12:18:03 -0600 Subject: [PATCH 26/71] Reorganize resolver code --- lib/pavilion/func_utils.py | 17 +++ lib/pavilion/resolver/resolver.py | 219 ++++++++++++++++++------------ 2 files changed, 147 insertions(+), 89 deletions(-) diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py index dcada9bdd..d7d6e9d0e 100644 --- a/lib/pavilion/func_utils.py +++ b/lib/pavilion/func_utils.py @@ -1,5 +1,6 @@ """A collection of utilities defined using functional methods""" +from pathlib import Path from typing import (List, Union, TypeVar, Iterator, Iterable, Callable, Optional, Hashable, Dict, Tuple) @@ -64,3 +65,19 @@ def listmap(func: Callable[[T], U], lst: Iterable[T]) -> List[U]: """Map a function over an iterable, but return a list instead of an iterator.""" return list(map(func, lst)) + +def exists(path: Path) -> bool: + """Wraps Path.exists, which obviates the need for + a lambda function when mapping it.""" + + return path.exists() + +def append_path(suffix: Path) -> Callable[[Path], Path]: + """Constructs a function that appends the given suffix + to path. Intended for use with map.""" + + def f(path: Path) -> Path: + return path / suffix + + return f + diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index c1cab5cc2..1e9e9dad8 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -22,6 +22,7 @@ import similarity import yc_yaml +import yaml_config as yc from pavilion.enums import Verbose from pavilion import output, variables from pavilion import pavilion_variables @@ -36,7 +37,7 @@ KEY_NAME_RE) from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary -from pavilion.func_utils import first, apply_to_first, listmap +from pavilion.func_utils import first, listmap, append_path from yaml_config import RequiredError, YamlConfigLoader from .proto_test import RawProtoTest, ProtoTest @@ -53,6 +54,14 @@ TestConfig = Dict +class ConfigInfo: + def __init__(self, name: str, type: str, path: Path, label: str = None, from_suite: bool = False): + self.name = name + self.type = type + self.label = label + self.path = path + self.from_suite = from_suite + class TestConfigResolver: """Converts raw test configurations into their final, fully resolved form.""" @@ -104,62 +113,101 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, # Raw loaded test suites self._suites: Dict[Dict] = {} - CONF_TYPE_DIRNAMES = { - 'suite': 'tests', - 'series': 'series', - 'OS': 'os', - 'host': 'hosts', - 'hosts': 'hosts', - 'mode': 'modes', - 'modes': 'modes', - } + def _get_config_dirname(self, cfg_type: str, use_suites_dir: bool = False) -> str: + """Returns the canonical config directory name for a given config type.""" + dirname = cfg_type.lower() + + if cfg_type == "suite" and not use_suites_dir: + return "tests" + + if dirname[-1] != 's' and dirname != "os": + dirname += 's' + + return dirname @property def config_paths(self) -> Iterator[Path]: + """Return an iterator over all config paths.""" return map(lambda x: x["path"], self.pav_cfg.configs.values()) @property def suites_dirs(self) -> Iterator[Path]: - return map(lambda x: x / "suites", self.config_paths) + """Return an iterator over all suites directories.""" + return map(append_path("suites"), self.config_paths) + + @property + def config_labels(self) -> Iterator[str]: + """Return an iterator over all config labels.""" + return self.pav_cfg.configs.keys() + + def _get_test_config_path(self, cfg_name: str, cfg_type: str) -> Tuple[str, Optional[Path]]: + """Given a config name and type, find the path to that config, if it exists, + excluding configs in the suites directory. If no such config exists, + return None.""" + + cfg_dir = self._get_config_dirname(cfg_type) + paths = map(append_path(f"{cfg_dir}/{cfg_name}.yaml"), self.config_paths) + pairs = zip(self.config_labels, paths) + + res = first(lambda x: x[1].exists(), pairs) + + if res is None: + return '', None + + return res + + def _config_path_from_suite(self, suite_name: str, conf_type: str) -> Tuple[str, Optional[Path]]: + """Given a suite name, return the path to the config file of the specified + type, if one exists. If the file does not exist in any known suites directory, + returns None.""" - def _cfg_path_from_suite(self, suite_name: str, conf_type: str) -> Optional[Path]: paths = [] + labels = list(self.config_labels) + + if conf_type == "suite": + paths.extend(listmap(append_path(f"{suite_name}.yaml"), self.suites_dirs)) + labels *= 2 - if conf_type == 'suite': - paths.append(listmap(lambda x: x / f"{suite_name}.yaml", self.suites_dirs)) + paths.extend(listmap(append_path(f"{suite_name}/{conf_type}.yaml"), self.suites_dirs)) + pairs = zip(labels, paths) - cfg_name = self._normalize_cfg_type(conf_type) - paths.append(listmap(lambda x: x / suite_name / f"{cfg_name}.yaml")) + res = first(lambda x: x[1].exists(), pairs) - return first(paths) + if res is None: + return '', None - def find_config(self, conf_type: str, conf_name: str, suite_name: str = None) -> Tuple[str, Path]: + return res + + def find_config(self, cfg_type: str, cfg_name: str, suite_name: str = None) -> ConfigInfo: """Search all of the known configuration directories for a config of the - given type and name. + given type and name, and report whether it was found in the suites directory. :param str conf_type: 'host', 'os', 'mode', or 'test/suite' :param str conf_name: The name of the config (without a file extension). - :return: A tuple of the config label under which a matching config was found - and the path to that config. If nothing was found, returns (None, None). + :return: A tuple of the path to that config, if it exists, and a boolean + indicating whether it was found in the suites directory (True) or not (False). """ + + cfg_path = None - suite_path = first(lambda x: x.exists(), self.suites_dirs) + if suite_name is not None: + label, cfg_path = self._config_path_from_suite(suite_name, cfg_type) - for label, config in self.pav_cfg.configs.items(): - potential_paths = self._get_cfg_paths(config['path'], suite_name, conf_type, conf_name) - - - if path is not None: - return label, path + if cfg_path is not None: + from_suite = True + else: + label, cfg_path = self._get_test_config_path(cfg_name, cfg_type) + from_suite = False - return '', None + return ConfigInfo(cfg_name, cfg_type, cfg_path, label, from_suite) def find_similar_configs(self, conf_type: str, conf_name: str) -> List[str]: """Find configs with a name similar to the one specified.""" # TODO: modify this for new suites directory + # It will need to search inside suites config files to find names of modes, hosts, etc. - conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] + conf_dir = self._get_config_dirname(conf_type) for label, config in self.pav_cfg.configs.items(): type_path = config['path'] / conf_type @@ -277,7 +325,7 @@ def find_all_configs(self, conf_type): """ - conf_dir = self.CONF_TYPE_DIRNAMES[conf_type] + conf_dir = self._get_config_dirname(conf_type) configs = {} for config in self.pav_cfg.configs.values(): @@ -540,14 +588,6 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: return multiplied_tests - def _normalize_cfg_type(self, cfg_type: str) -> str: - cfg_type = cfg_type.lower() - - if not cfg_type[-1] != 's' and cfg_type != 'os': - cfg_type += 's' - - return cfg_type - def _safe_load_config(self, path: Path, loader: yc.YamlConfigLoader) -> TestConfig: """Given a path to a config, load the config, and raise an appropriate error if it can't be loaded""" @@ -577,39 +617,29 @@ def _safe_load_config(self, path: Path, loader: yc.YamlConfigLoader) -> TestConf return raw_cfg - def _load_from_suite(self, suite_dir: Path, cfg_type: str, cfg_name: str, loader: yc.YamlConfigLoader) -> TestConfig: - cfg_type = self._normalize_cfg_type(cfg_type) - cfg_path = suite_dir / f"{cfg_type}.yaml" - - raw_cfg = self._safe_load_config(cfg_path) - - try: - return raw_cfg[cfg_name] - except KeyError: - short_path = f"{cfg_path.parents[0].name}/{cfg_path.name}" - raise TestConfigError( - f"Config name {cfg_name} not found in {short_path}" - ) + def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, optional: bool = False) -> Optional[TestConfig]: + """Given a path to a config file and a loader, attempt to load the config, handle errors + appropriately.""" - def _load_raw_config(self, cfg_path: Optional[Path], cfg_type: str, loader: yc.YamlConfigLoader) -> TestConfig: - # TODO: docstring + if cfg_info.path is None and optional: + return None - if path is None: - similar = self.find_similar_configs(cfg_type, config_name) + if cfg_info.path is None and not optional: + similar = self.find_similar_configs(cfg_info.type, cfg_info.name) if similar: raise TestConfigError( "Could not find {} config {}.yaml.\n" "Did you mean one of these? {}" - .format(cfg_type, config_name, ', '.join(similar))) + .format(cfg_info.type, cfg_info.name, ', '.join(similar))) else: raise TestConfigError( "Could not find {0} config file '{1}.yaml' in any of the " "Pavilion config directories.\n" "Run `pav show {2}` to get a list of available {0} files." - .format(cfg_type, config_name, cfg_type)) + .format(cfg_info.type, cfg_info.name, cfg_info.type)) - return self._safe_load_config(cfg_path) + return self._safe_load_config(cfg_info.path, loader) def _load_raw_configs(self, request: TestRequest, modes: List[str], conditions: Dict, overrides: List[str]) \ @@ -667,8 +697,6 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], # Apply modes. try: test_cfg = self.apply_modes(test_cfg, modes, request.suite) - test_cfg = self.apply_host(test_cfg, self._host, request.suite) - test_cfg = self.apply_os(test_cfg, self._os, request.suite) except TestConfigError as err: err.request = request self.errors.append(err) @@ -700,8 +728,7 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], test_cfg['result_evaluate'][key] = '"{}"'.format(const) - import pdb; pdb.set_trace() - test_cfg = self._validate(test_name, test_cfg) + test_cf = self._validate(test_name, test_cfg) # Now that we've applied all general transforms to the config, make it into a ProtoTest. try: @@ -759,7 +786,7 @@ def _validate(self, test_name: str, test_cfg: Dict) -> Dict: return test_cfg - def _load_base_config(self, op_sys, host) -> Dict: + def _load_base_config(self, op_sys: str, host: str) -> TestConfig: """Load the base configuration for the given host. This is done once and saved.""" # Get the base, empty config, then apply the host config on top of it. @@ -773,15 +800,21 @@ def _load_suite_tests(self, suite_name: str): if suite_name in self._suites: return self._suites[suite_name] - suite_cfg_path = self.find_config(suite_name, "suite", suite_name) - raw_suite_cfg = self._load_raw_config(suite_cfg_path, "suite") + cfg_info = self.find_config("suite", suite_name, suite_name) + + if cfg_info.from_suite: + loader = self._suite_loader + else: + loader = self._loader + + raw_suite_cfg = self._load_raw_config(cfg_info, loader) # Make sure each test has a dict as contents. for test_name, raw_test in raw_suite_cfg.items(): if raw_test is None: raw_suite_cfg[test_name] = {} - suite_tests = self.resolve_inheritance(raw_suite_cfg, suite_cfg_path) + suite_tests = self.resolve_inheritance(raw_suite_cfg, cfg_info.path) # Perform essential transformations to each test config. for test_cfg_name, test_cfg in list(suite_tests.items()): @@ -789,10 +822,10 @@ def _load_suite_tests(self, suite_name: str): # Basic information that all test configs should have. test_cfg['name'] = test_cfg_name # test_cfg['cfg_label'] = cfg_label - working_dir = self.pav_cfg['configs'][cfg_label]['working_dir'] + working_dir = self.pav_cfg['configs'][cfg_info.label]['working_dir'] test_cfg['working_dir'] = working_dir.as_posix() test_cfg['suite'] = suite_name - test_cfg['suite_path'] = suite_cfg_path.as_posix() + test_cfg['suite_path'] = cfg_info.path.as_posix() test_cfg['host'] = self._host test_cfg['os'] = self._os @@ -867,21 +900,21 @@ def check_version_compatibility(self, test_cfg): "Incompatible with pavilion version '{}', compatible versions " "'{}'.".format(PavVars()['version'], comp_versions)) - def _get_loader(suite_name: Optional[str]) -> yc.TestConfigLoader: - """Given a suite_name, return the appropriate loader.""" - - if suite_name is None: - return self._loader - - return self._suite_loader - - def apply_host(self, test_cfg, host, suite_name: str = None): + def apply_host(self, test_cfg: TestConfig, hostname: str, suite_name: str = None) -> TestConfig: """Apply the host configuration to the given config.""" - host_cfg_path = self._get_cfg_paths(host, "host", suite_name) - loader = self._get_loader(suite_name) + if suite_name is not None: + from_suite = True + label, host_cfg_path = self._config_path_from_suite(suite_name, "host") + loader = self._suite_loader + else: + from_suite = False + label, host_cfg_path = self._get_test_config_path(hostname, "host") + loader = self._loader + + cfg_info = ConfigInfo(hostname, "host", host_cfg_path, label, from_suite) - raw_host_cfg = self._load_raw_config(host_cfg_path, loader) + raw_host_cfg = self._load_raw_config(cfg_info, loader, optional=True) if raw_host_cfg is None: return test_cfg @@ -900,13 +933,21 @@ def apply_host(self, test_cfg, host, suite_name: str = None): raise TestConfigError( "Error merging host configuration for host '{}'".format(host)) - def apply_os(self, test_cfg, op_sys, suite_name: str = None): + def apply_os(self, test_cfg: TestConfig, op_sys: str, suite_name: str = None) -> TestConfig: """Apply the OS configuration to the given config.""" - os_cfg_path = self._get_cfg_paths(op_sys, "OS", suite_name) - loader = self._get_loader(suite_name) + if suite_name is not None: + from_suite = True + label, os_cfg_path = self._config_path_from_suite(suite_name, "OS") + loader = self._suite_loader + else: + from_suite = False + label, os_cfg_path = self._get_test_config_path(op_sys, "OS") + loader = self._loader + + cfg_info = ConfigInfo(op_sys, "OS", os_cfg_path, from_suite) - raw_os_cfg = self._load_raw_config(os_cfg_path, loader) + raw_os_cfg = self._load_raw_config(cfg_info, loader, optional=True) if raw_os_cfg is None: return test_cfg @@ -925,18 +966,18 @@ def apply_os(self, test_cfg, op_sys, suite_name: str = None): raise TestConfigError( "Error merging configuration for OS '{}'".format(os)) - def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): + def apply_modes(self, test_cfg, modes: List[str], loader: yc.YamlConfigLoader, suite_name: str = None): """Apply each of the mode files to the given test config. :param test_cfg: The raw test configuration. :param modes: A list of mode names. """ - loader = self._get_loader(suite_name) - for mode in modes: mode_cfg_path = self.find_config(mode, "mode", suite_name) - raw_mode_cfg = self._load_raw_config(mode_cfg_path, loader) + + cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, True if suite_name is not None else False) + raw_mode_cfg = self._load_raw_config(cfg_info, loader) try: mode_cfg = loader.normalize( From 9fa823bb1371277c0ffca00da81c44b8694f399f Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 26 Aug 2024 12:24:25 -0600 Subject: [PATCH 27/71] Rename tests appropriately --- test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml | 2 +- test/data/pav_config_dir/suites/modes_suite_test/suite.yaml | 2 +- test/data/pav_config_dir/suites/os_suite_test/suite.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml b/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml index 24205bb82..b7749cd6b 100644 --- a/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml +++ b/test/data/pav_config_dir/suites/hosts_suite_test/suite.yaml @@ -1,4 +1,4 @@ -basic_suite_test: +hosts_suite_test: scheduler: raw run: cmds: 'echo Hello!' diff --git a/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml b/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml index 1e33457b7..abeda51e8 100644 --- a/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml +++ b/test/data/pav_config_dir/suites/modes_suite_test/suite.yaml @@ -1,4 +1,4 @@ -mode_suite_test: +modes_suite_test: scheduler: raw run: cmds: 'echo Hello!' diff --git a/test/data/pav_config_dir/suites/os_suite_test/suite.yaml b/test/data/pav_config_dir/suites/os_suite_test/suite.yaml index 24205bb82..39fc5a13a 100644 --- a/test/data/pav_config_dir/suites/os_suite_test/suite.yaml +++ b/test/data/pav_config_dir/suites/os_suite_test/suite.yaml @@ -1,4 +1,4 @@ -basic_suite_test: +os_suite_test: scheduler: raw run: cmds: 'echo Hello!' From 98ae10893480d57a950cb0558dc8613a0ddb0591 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 26 Aug 2024 12:41:35 -0600 Subject: [PATCH 28/71] Flesh out unit tests --- test/tests/suites_tests.py | 48 +++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index d6b060371..e79f3520c 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -61,19 +61,61 @@ def test_suites_host_config(self): last_test = run_cmd.last_tests[0] - self.assertTrue(False) + self.assertTrue(last_test.config["host"] == "host1") + + variables = last_test.config["variables"] + + self.assertEqual(variables.get("host1"), True) def test_suites_mode_config(self): """Test that Pavilion loads mode configs from the suites directory""" - self.assertTrue(False) + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + '-m', 'mode1', + 'modes_suite_test' + ]) + + run_cmd = commands.get_command(args.command_name) + # How do I resolve the configs without building the test? + ret = run_cmd.run(self.pav_cfg, args) + + self.assertEqual(ret, 0) + + last_test = run_cmd.last_tests[0] + + self.assertTrue("mode1" in last_test.config["modes"]) + + variables = last_test.config["variables"] + + self.assertEqual(variables.get("mode1"), True) def test_suites_os_config(self): """Test that Pavilion loads OS configs from the suites directory""" - self.assertTrue(False) + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + '-o', 'os1', + 'os_suite_test' + ]) + + run_cmd = commands.get_command(args.command_name) + # How do I resolve the configs without building the test? + ret = run_cmd.run(self.pav_cfg, args) + + self.assertEqual(ret, 0) + + last_test = run_cmd.last_tests[0] + + self.assertTrue(last_test.config["os"] == "os1") + + variables = last_test.config["variables"] + + self.assertEqual(variables.get("os1"), True) def test_suites_build_hash(self): """Test that Pavilion ignores config files in the From 02ee8587340a8b97bb31bc49fc36fe924954b84a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 27 Aug 2024 08:29:23 -0600 Subject: [PATCH 29/71] Implement hosts from suites dir --- lib/pavilion/resolver/resolver.py | 36 ++++++++++++++++++++++--------- test/tests/suites_tests.py | 6 +++--- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 1e9e9dad8..7383dd86a 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -163,12 +163,18 @@ def _config_path_from_suite(self, suite_name: str, conf_type: str) -> Tuple[str, paths = [] labels = list(self.config_labels) + + if conf_type in ("host", "mode"): + cfg_fname = conf_type + 's' + else: + cfg_fname = conf_type if conf_type == "suite": paths.extend(listmap(append_path(f"{suite_name}.yaml"), self.suites_dirs)) labels *= 2 - paths.extend(listmap(append_path(f"{suite_name}/{conf_type}.yaml"), self.suites_dirs)) + paths.extend(listmap(append_path(f"{suite_name}/{cfg_fname}.yaml"), self.suites_dirs)) + pairs = zip(labels, paths) res = first(lambda x: x[1].exists(), pairs) @@ -639,7 +645,17 @@ def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, op "Run `pav show {2}` to get a list of available {0} files." .format(cfg_info.type, cfg_info.name, cfg_info.type)) - return self._safe_load_config(cfg_info.path, loader) + raw_cfg = self._safe_load_config(cfg_info.path, loader) + + if cfg_info.from_suite and cfg_info.type is not "suite": + raw_cfg = raw_cfg.get(cfg_info.name) + + if raw_cfg is None: + raise TestConfigError( + f"Could not find {cfg_info.type} config with name {cfg_info.type}" + "in file {cfg_info.path}.") + + return raw_cfg def _load_raw_configs(self, request: TestRequest, modes: List[str], conditions: Dict, overrides: List[str]) \ @@ -697,6 +713,8 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], # Apply modes. try: test_cfg = self.apply_modes(test_cfg, modes, request.suite) + test_cfg = self.apply_host(test_cfg, self._host, request.suite) + test_cfg = self.apply_os(test_cfg, self._os, request.suite) except TestConfigError as err: err.request = request self.errors.append(err) @@ -920,18 +938,18 @@ def apply_host(self, test_cfg: TestConfig, hostname: str, suite_name: str = None return test_cfg try: - host_cfg = loader.normalize( + host_cfg = self._loader.normalize( raw_host_cfg, root_name=f"the top level of the host file.") except (KeyError, ValueError) as err: raise TestConfigError( - f"Error loading host config '{host}' from file '{host_cfg_path}'.") + f"Error loading host config '{hostname}' from file '{host_cfg_path}'.") try: - return loader.merge(test_cfg, host_cfg) + return self._loader.merge(test_cfg, host_cfg) except (KeyError, ValueError) as err: raise TestConfigError( - "Error merging host configuration for host '{}'".format(host)) + "Error merging host configuration for host '{}'".format(hostname)) def apply_os(self, test_cfg: TestConfig, op_sys: str, suite_name: str = None) -> TestConfig: """Apply the OS configuration to the given config.""" @@ -974,10 +992,8 @@ def apply_modes(self, test_cfg, modes: List[str], loader: yc.YamlConfigLoader, s """ for mode in modes: - mode_cfg_path = self.find_config(mode, "mode", suite_name) - - cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, True if suite_name is not None else False) - raw_mode_cfg = self._load_raw_config(cfg_info, loader) + cfg_info = self.find_config("mode", mode, suite_name) + raw_mode_cfg = self._load_raw_config(cfg_info, loader) try: mode_cfg = loader.normalize( diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index e79f3520c..21c3fc544 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -65,7 +65,7 @@ def test_suites_host_config(self): variables = last_test.config["variables"] - self.assertEqual(variables.get("host1"), True) + self.assertEqual(variables.get("host1")[0].get(None), "True") def test_suites_mode_config(self): """Test that Pavilion loads mode configs from the @@ -90,7 +90,7 @@ def test_suites_mode_config(self): variables = last_test.config["variables"] - self.assertEqual(variables.get("mode1"), True) + self.assertEqual(variables.get("mode1")[0].get(None), "True") def test_suites_os_config(self): """Test that Pavilion loads OS configs from the @@ -115,7 +115,7 @@ def test_suites_os_config(self): variables = last_test.config["variables"] - self.assertEqual(variables.get("os1"), True) + self.assertEqual(variables.get("os1")[0].get(None), "True") def test_suites_build_hash(self): """Test that Pavilion ignores config files in the From 215072630906ab8f231a6d0696e16c35da1bd1c4 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 27 Aug 2024 09:30:58 -0600 Subject: [PATCH 30/71] Implement os config from suites dir --- lib/pavilion/func_utils.py | 5 +++- lib/pavilion/resolver/resolver.py | 46 +++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py index d7d6e9d0e..a56bcaa0e 100644 --- a/lib/pavilion/func_utils.py +++ b/lib/pavilion/func_utils.py @@ -72,7 +72,10 @@ def exists(path: Path) -> bool: return path.exists() -def append_path(suffix: Path) -> Callable[[Path], Path]: + +Pathlike = Union[Path, str] + +def append_path(suffix: Pathlike) -> Callable[[Path], Path]: """Constructs a function that appends the given suffix to path. Intended for use with map.""" diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 7383dd86a..0aa7001f1 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -115,6 +115,7 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, def _get_config_dirname(self, cfg_type: str, use_suites_dir: bool = False) -> str: """Returns the canonical config directory name for a given config type.""" + dirname = cfg_type.lower() if cfg_type == "suite" and not use_suites_dir: @@ -125,6 +126,17 @@ def _get_config_dirname(self, cfg_type: str, use_suites_dir: bool = False) -> st return dirname + def _get_config_fname(self, cfg_type: str) -> str: + """Given a config type, returns the name of the file in the + suites directory corresponding to that type.""" + + fname = cfg_type.lower() + + if fname in ("host", "mode"): + fname += 's' + + return f"{fname}.yaml" + @property def config_paths(self) -> Iterator[Path]: """Return an iterator over all config paths.""" @@ -164,16 +176,13 @@ def _config_path_from_suite(self, suite_name: str, conf_type: str) -> Tuple[str, paths = [] labels = list(self.config_labels) - if conf_type in ("host", "mode"): - cfg_fname = conf_type + 's' - else: - cfg_fname = conf_type + cfg_fname = self._get_config_fname(conf_type) if conf_type == "suite": paths.extend(listmap(append_path(f"{suite_name}.yaml"), self.suites_dirs)) labels *= 2 - paths.extend(listmap(append_path(f"{suite_name}/{cfg_fname}.yaml"), self.suites_dirs)) + paths.extend(listmap(append_path(f"{suite_name}/{cfg_fname}"), self.suites_dirs)) pairs = zip(labels, paths) @@ -810,6 +819,7 @@ def _load_base_config(self, op_sys: str, host: str) -> TestConfig: # Get the base, empty config, then apply the host config on top of it. base_config = self._loader.load_empty() base_config = self.apply_os(base_config, op_sys) + return self.apply_host(base_config, host) def _load_suite_tests(self, suite_name: str): @@ -963,7 +973,7 @@ def apply_os(self, test_cfg: TestConfig, op_sys: str, suite_name: str = None) -> label, os_cfg_path = self._get_test_config_path(op_sys, "OS") loader = self._loader - cfg_info = ConfigInfo(op_sys, "OS", os_cfg_path, from_suite) + cfg_info = ConfigInfo(op_sys, "OS", os_cfg_path, label, from_suite) raw_os_cfg = self._load_raw_config(cfg_info, loader, optional=True) @@ -971,7 +981,7 @@ def apply_os(self, test_cfg: TestConfig, op_sys: str, suite_name: str = None) -> return test_cfg try: - os_cfg = loader.normalize( + os_cfg = self._loader.normalize( raw_os_cfg, root_name=f"the top level of the OS file.") except (KeyError, ValueError) as err: @@ -979,7 +989,7 @@ def apply_os(self, test_cfg: TestConfig, op_sys: str, suite_name: str = None) -> f"Error loading host config '{op_sys}' from file '{os_cfg_path}'") try: - return loader.merge(test_cfg, os_cfg) + return self._loader.merge(test_cfg, os_cfg) except (KeyError, ValueError) as err: raise TestConfigError( "Error merging configuration for OS '{}'".format(os)) @@ -991,12 +1001,26 @@ def apply_modes(self, test_cfg, modes: List[str], loader: yc.YamlConfigLoader, s :param modes: A list of mode names. """ + if suite_name is not None: + from_suite = True + loader = self._suite_loader + else: + from_suite = False + loader = self._loader + for mode in modes: - cfg_info = self.find_config("mode", mode, suite_name) + + if from_suite: + label, mode_cfg_path = self._config_path_from_suite(suite_name, "mode") + else: + label, mode_cfg_path = self._get_test_config_path(mode, "mode") + + cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, from_suite) + raw_mode_cfg = self._load_raw_config(cfg_info, loader) try: - mode_cfg = loader.normalize( + mode_cfg = self._loader.normalize( raw_mode_cfg, root_name=f"the top level of the OS file.") except (KeyError, ValueError) as err: @@ -1004,7 +1028,7 @@ def apply_modes(self, test_cfg, modes: List[str], loader: yc.YamlConfigLoader, s f"Error loading host config '{mode}' from file '{mode_cfg_path}'.") try: - test_cfg = loader.merge(test_cfg, mode_cfg) + test_cfg = self._loader.merge(test_cfg, mode_cfg) except (KeyError, ValueError) as err: raise TestConfigError( "Error merging mode configuration for mode '{}'".format(mode)) From f1d152c818f54b0999f3f27cb27d3ca4324eeb29 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 29 Aug 2024 14:57:52 -0600 Subject: [PATCH 31/71] Implement loading mode from suites dir --- lib/pavilion/resolver/resolver.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 0aa7001f1..a1f04a398 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -54,6 +54,7 @@ TestConfig = Dict + class ConfigInfo: def __init__(self, name: str, type: str, path: Path, label: str = None, from_suite: bool = False): self.name = name @@ -62,6 +63,7 @@ def __init__(self, name: str, type: str, path: Path, label: str = None, from_sui self.path = path self.from_suite = from_suite + class TestConfigResolver: """Converts raw test configurations into their final, fully resolved form.""" @@ -994,7 +996,7 @@ def apply_os(self, test_cfg: TestConfig, op_sys: str, suite_name: str = None) -> raise TestConfigError( "Error merging configuration for OS '{}'".format(os)) - def apply_modes(self, test_cfg, modes: List[str], loader: yc.YamlConfigLoader, suite_name: str = None): + def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): """Apply each of the mode files to the given test config. :param test_cfg: The raw test configuration. @@ -1014,8 +1016,8 @@ def apply_modes(self, test_cfg, modes: List[str], loader: yc.YamlConfigLoader, s label, mode_cfg_path = self._config_path_from_suite(suite_name, "mode") else: label, mode_cfg_path = self._get_test_config_path(mode, "mode") - - cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, from_suite) + + cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, label, from_suite) raw_mode_cfg = self._load_raw_config(cfg_info, loader) From bcc66f65417a897e2a40a0210177e0b7227f3344 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 30 Aug 2024 15:18:23 -0600 Subject: [PATCH 32/71] Add timeouts to unit tests to prevent hanging --- lib/pavilion/common.py | 15 ------ lib/pavilion/micro.py | 81 +++++++++++++++++++++++++++++++++ lib/pavilion/path_utils.py | 27 +++++++++++ lib/pavilion/series/test_set.py | 12 ++++- test/tests/group_tests.py | 8 ++-- test/tests/run_cmd_tests.py | 2 +- test/tests/series_cmd_tests.py | 4 +- test/tests/testset_tests.py | 18 ++++---- 8 files changed, 135 insertions(+), 32 deletions(-) delete mode 100644 lib/pavilion/common.py create mode 100644 lib/pavilion/micro.py create mode 100644 lib/pavilion/path_utils.py diff --git a/lib/pavilion/common.py b/lib/pavilion/common.py deleted file mode 100644 index 3146894e0..000000000 --- a/lib/pavilion/common.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import TypeVar, Optional - -T = TypeVar("T") - -def set_default(val: Optional[T], default: T) -> T: - """Set the input value to its default, if it is None.""" - - if val is None: - return default - - return val - -def get_nested(keys: Iterable[Hashable], dict: Dict) -> Dict: - """Safely get the hierarchical sequence of keys off the - dictionary. Guaranteed to return a dictionary.""" diff --git a/lib/pavilion/micro.py b/lib/pavilion/micro.py new file mode 100644 index 000000000..6ca06432a --- /dev/null +++ b/lib/pavilion/micro.py @@ -0,0 +1,81 @@ +"""A collection of 'microfunctions' primarily designed to abstract common +tasks and patterns, for the purpose of conciseness and readability.""" + +from pathlib import Path +from itertools import filterfalse, chain, tee +from typing import (List, Union, TypeVar, Iterator, Iterable, Callable, Optional, + Hashable, Dict, Tuple) + +T = TypeVar('T') +U = TypeVar('U') + + +def partition(pred: Callable[[T], bool], lst: Iterable[T]) -> Tuple[Iterator[T], Iterator[T]]: + """Partition the sequence into two sequences: one consisting of the elements + for which the given predicate is true and one consisting of those for + which it is false.""" + + f_true, f_false = tee(lst) + + return filter(pred, f_true), filterfalse(pred, f_false) + +def flatten(lst: Iterable[Iterable[T]]) -> Iterator[T]: + """Convert a singly nested iterable into an unnested iterable.""" + return chain.from_iterable(lst) + +def remove_all(lst: Iterable[T], item: T) -> Iterator[T]: + """Remove all instances of the given item from the iterable.""" + return filter(lambda x: x != item, lst) + +def unique(lst: Iterable[T]) -> List[T]: + """Return a list of the unique items in the original list.""" + return list(set(lst)) + +def replace(lst: Iterable[T], old: T, new: T) -> Iterator[T]: + """Replace all instances of old with new.""" + return map(lambda x: new if x == old else x, lst) + +def remove_none(lst: Iterable[T]) -> Iterator[T]: + """Remove all instances of None from the iterable.""" + return filter(lambda x: x is not None, lst) + +def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: + """Return the first item of the list that satisfies the given + predicate, or None if no item does.""" + + for item in filter(pred, lst): + return item + +def apply_to_first(func: Callable[[T], U], pred: Callable[[T], bool], + lst: Iterable[T]) -> Optional[U]: + """Apply the function to the first element of the list that satisfies + the given predicate. If no element satisfies the predicate, return None.""" + + fst = first(pred, lst) + + if fst is not None: + return func(fst) + +def get_nested(keys: Iterable[Hashable], nested_dict: Dict) -> Dict: + """Gets the values associated with the given sequence of keys + out of a nested dictionary. If any key in the sequence does + not exist during the process, returns an empty dictionary.""" + + for key in keys: + nested_dict = nested_dict.get(key, {}) + + return nested_dict + +def listmap(func: Callable[[T], U], lst: Iterable[T]) -> List[U]: + """Map a function over an iterable, but return a list instead + of a map object.""" + return list(map(func, lst)) + +def set_default(val: Optional[T], default: T) -> T: + """Set the input value to default, if the original value is None. + Otherwise, return the value unchanged.""" + + if val is None: + return default + + return val diff --git a/lib/pavilion/path_utils.py b/lib/pavilion/path_utils.py new file mode 100644 index 000000000..8a399d4d7 --- /dev/null +++ b/lib/pavilion/path_utils.py @@ -0,0 +1,27 @@ +"""Simple utilities for dealing with paths.""" + +from pathlib import Path +from typing import Union, Callable + +Pathlike = Union[Path, str] + +def exists(path: Path) -> bool: + """Wraps Path.exists, which obviates the need for + a lambda function when mapping it.""" + + return path.exists() + +def append_path(suffix: Pathlike) -> Callable[[Path], Path]: + """Constructs a function that appends the given suffix + to a path. Intended for use with map.""" + + def f(path: Path) -> Path: + return path / suffix + + return f + +def shortpath(path: Path, parents: int = 1) -> Path: + """Return an abbreviated version of a path, where only + the specified number of parents are included.""" + + return Path(*path.parts[-(1 + parents):]) diff --git a/lib/pavilion/series/test_set.py b/lib/pavilion/series/test_set.py index d9bf971e4..2c8670a56 100644 --- a/lib/pavilion/series/test_set.py +++ b/lib/pavilion/series/test_set.py @@ -5,6 +5,7 @@ import os import threading import time +import math from collections import defaultdict from io import StringIO from typing import List, Dict, TextIO, Union, Set, Iterator, Tuple @@ -19,6 +20,7 @@ from pavilion.utils import str_bool from pavilion.enums import Verbose from pavilion.jobs import Job +from pavilion.micro import set_default S_STATES = SERIES_STATES @@ -713,7 +715,7 @@ def mark_completed(self) -> int: TEST_WAIT_PERIOD = 0.5 - def wait(self, wait_for_all=False, wait_period: int = TEST_WAIT_PERIOD) -> int: + def wait(self, wait_for_all=False, wait_period: int = TEST_WAIT_PERIOD, timeout: int = None) -> int: """Wait for tests to complete. Returns the number of jobs that completed for this call to wait. @@ -722,14 +724,22 @@ def wait(self, wait_for_all=False, wait_period: int = TEST_WAIT_PERIOD) -> int: :return: The number of completed tests """ + timeout = set_default(timeout, math.inf) + # No tests to wait for if not self.started_tests: return 0 completed_tests = self.mark_completed() + start = time.monotonic() + while ((wait_for_all and self.started_tests) or (not wait_for_all and completed_tests == 0)): + + if time.monotonic() - start > timeout: + raise TimeoutError("Timed out waiting for test set to complete.") + time.sleep(wait_period) completed_tests += self.mark_completed() diff --git a/test/tests/group_tests.py b/test/tests/group_tests.py index 401a7abac..08399c824 100644 --- a/test/tests/group_tests.py +++ b/test/tests/group_tests.py @@ -220,8 +220,8 @@ def test_group_commands(self): run_cmd.run(self.pav_cfg, run_args) series_cmd.run(self.pav_cfg, series_args) - run_cmd.last_series.wait() - series_cmd.last_series.wait() + run_cmd.last_series.wait(timeout=10) + series_cmd.last_series.wait(timeout=10) group = groups.TestGroup(self.pav_cfg, group_name) self.assertTrue(group.exists()) @@ -230,13 +230,13 @@ def test_group_commands(self): # Prep some separate tests to add run_args2 = parser.parse_args(['run', 'hello_world']) run_cmd.run(self.pav_cfg, run_args2) - run_cmd.last_series.wait() + run_cmd.last_series.wait(timeout=10) # Create a new group with tests to add sub_group_name = self._make_group_name() run_args3 = parser.parse_args(['run', '-g', sub_group_name, 'hello_world']) run_cmd.run(self.pav_cfg, run_args3) - run_cmd.last_series.wait() + run_cmd.last_series.wait(timeout=10) add_items = [sub_group_name] + [test.full_id for test in run_cmd.last_tests] rm_tests = add_items[1:3] diff --git a/test/tests/run_cmd_tests.py b/test/tests/run_cmd_tests.py index b06e6caa8..461895792 100644 --- a/test/tests/run_cmd_tests.py +++ b/test/tests/run_cmd_tests.py @@ -232,7 +232,7 @@ def test_concurrent(self): out, err = run_cmd.clear_output() self.assertEqual(run_cmd.run(self.pav_cfg, args), 0, msg=out+err) - run_cmd.last_series.wait() + run_cmd.last_series.wait(timeout=10) # The test fails if it ever catches more tests running than its concurrency limit for test in run_cmd.last_tests: diff --git a/test/tests/series_cmd_tests.py b/test/tests/series_cmd_tests.py index ddd11bd7d..cbde2ee10 100644 --- a/test/tests/series_cmd_tests.py +++ b/test/tests/series_cmd_tests.py @@ -36,7 +36,7 @@ def test_run_series(self): run_result = run_cmd.run(self.pav_cfg, args) self.assertEqual(run_result, 0) - run_cmd.last_run_series.wait() + run_cmd.last_run_series.wait(timeout=10) self.assertEqual(run_cmd.last_run_series.complete, True) self.assertEqual(run_cmd.last_run_series.info().passed, 1) @@ -128,7 +128,7 @@ def test_series_sets(self): args = arg_parser.parse_args(['series', 'run', 'multi']) self.assertEqual(series_cmd.run(self.pav_cfg, args), 0) - series_cmd.last_run_series.wait() + series_cmd.last_run_series.wait(timeout=10) sid = series_cmd.last_run_series.sid arg_lists = [ diff --git a/test/tests/testset_tests.py b/test/tests/testset_tests.py index 6ed774d8d..cdc129326 100644 --- a/test/tests/testset_tests.py +++ b/test/tests/testset_tests.py @@ -160,7 +160,7 @@ def test_kickoff(self): ts1.make() ts1.build() self.assertEqual(len(ts1.kickoff()[0]), 10) - ts1.wait(wait_for_all=True) + ts1.wait(wait_for_all=True, timeout=10) # It shouldn't hurt to kickoff when there aren't any tests. ts1.kickoff() @@ -183,7 +183,7 @@ def test_kickoff(self): start_names.sort() exp_names.sort() self.assertEqual(start_names, exp_names) - ts2.wait(wait_for_all=True) + ts2.wait(wait_for_all=True, timeout=10) self.assertEqual(expected, []) @@ -192,7 +192,7 @@ def test_kickoff(self): ts3.make() ts3.build() self.assertEqual(len(ts3.kickoff()[0]), 0) - ts3.wait(wait_for_all=True) + ts3.wait(wait_for_all=True, timeout=10) def test_wait(self): """Checking that we can wait for partial results.""" @@ -201,7 +201,7 @@ def test_wait(self): ts1.make() ts1.build() ts1.kickoff() - self.assertNotEqual(ts1.wait(), 3) + self.assertNotEqual(ts1.wait(timeout=10), 3) def test_all_passed(self): """Make sure we properly verify pass/fail status.""" @@ -210,14 +210,14 @@ def test_all_passed(self): ts1.make() ts1.build() ts1.kickoff() - ts1.wait(wait_for_all=True) + ts1.wait(wait_for_all=True, timeout=10) self.assertFalse(ts1.all_passed) ts2 = TestSet(self.pav_cfg, "test_all_passed2", ["pass_fail.pass"] * 2) ts2.make() ts2.build() ts2.kickoff() - ts2.wait(wait_for_all=True) + ts2.wait(wait_for_all=True, timeout=10) self.assertTrue(ts2.all_passed, msg=[test.result for test in ts2.tests]) def test_cancel(self): @@ -227,7 +227,7 @@ def test_cancel(self): ts1.build() ts1.kickoff() ts1.cancel("Testing cancelation.") - ts1.wait(wait_for_all=True) + ts1.wait(wait_for_all=True, timeout=10) for test in ts1.tests: self.assertEqual(test.status.current().state, test.status.states.CANCELLED, @@ -265,7 +265,7 @@ def test_should_run(self): ts1.make() ts1.build() ts1.kickoff() - ts1.wait(wait_for_all=True) + ts1.wait(wait_for_all=True, timeout=10) self.assertTrue(ts2.should_run) self.assertFalse(ts2_pmp.should_run) self.assertFalse(ts3.should_run) @@ -290,7 +290,7 @@ def test_should_run(self): ts1.make() ts1.build() ts1.kickoff() - ts1.wait(wait_for_all=True) + ts1.wait(wait_for_all=True, timeout=10) self.assertTrue(ts2.should_run) self.assertTrue(ts2_pmp.should_run) self.assertTrue(ts3.should_run) From 5ed0bebdab127db1493c7122fcb926681c5ff4a8 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 30 Aug 2024 15:19:25 -0600 Subject: [PATCH 33/71] Fix regression in series_config --- lib/pavilion/series_config/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/series_config/__init__.py b/lib/pavilion/series_config/__init__.py index 79a230d43..32231f15e 100644 --- a/lib/pavilion/series_config/__init__.py +++ b/lib/pavilion/series_config/__init__.py @@ -82,7 +82,9 @@ def load_series_config(pav_cfg, series_name: str) -> dict: series_config_loader = SeriesConfigLoader() resolver = TestConfigResolver(pav_cfg) - _, series_file_path = resolver.find_config('series', series_name) + cfg_info = resolver.find_config('series', series_name) + + series_file_path = cfg_info.path if not series_file_path: raise SeriesConfigError('Cannot find series config: {}'. From 4352557b7b3ce94236794912bc4f22cc055371ff Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 30 Aug 2024 15:46:18 -0600 Subject: [PATCH 34/71] Add missed timeout --- test/tests/series_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/series_tests.py b/test/tests/series_tests.py index 212881099..cc7401a4b 100644 --- a/test/tests/series_tests.py +++ b/test/tests/series_tests.py @@ -283,7 +283,7 @@ def test_ignore_errors(self): series_obj = series.TestSeries(self.pav_cfg, series_cfg=cfg) series_obj.run() - series_obj.wait() + series_obj.wait(timeout=10) for test in series_obj.tests.values(): if test.name in ['test_set_errors.good', 'hello_world.hello']: From 86615b656edf8d0a7e387744705b17fd524d9188 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 30 Aug 2024 16:04:15 -0600 Subject: [PATCH 35/71] Fix error in safe_load_config --- lib/pavilion/resolver/resolver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index a1f04a398..8a48cfa48 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -605,10 +605,13 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: return multiplied_tests - def _safe_load_config(self, path: Path, loader: yc.YamlConfigLoader) -> TestConfig: + def _safe_load_config(self, cfg: ConfigInfo, loader: yc.YamlConfigLoader) -> TestConfig: """Given a path to a config, load the config, and raise an appropriate error if it can't be loaded""" + path = cfg.path + cfg_type = cfg.type + try: with path.open() as cfg_file: raw_cfg = loader.load_raw(cfg_file) @@ -656,7 +659,7 @@ def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, op "Run `pav show {2}` to get a list of available {0} files." .format(cfg_info.type, cfg_info.name, cfg_info.type)) - raw_cfg = self._safe_load_config(cfg_info.path, loader) + raw_cfg = self._safe_load_config(cfg_info, loader) if cfg_info.from_suite and cfg_info.type is not "suite": raw_cfg = raw_cfg.get(cfg_info.name) From 215a178b808ada2c1626ce43094a9ee5e5df643c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 30 Aug 2024 16:37:07 -0600 Subject: [PATCH 36/71] Fix date math (again) --- lib/pavilion/filters/parse_time.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/filters/parse_time.py b/lib/pavilion/filters/parse_time.py index ac76e9b06..37900c6c9 100644 --- a/lib/pavilion/filters/parse_time.py +++ b/lib/pavilion/filters/parse_time.py @@ -1,4 +1,5 @@ from datetime import date, time, datetime, timedelta +from calendar import monthrange from typing import Tuple, Union @@ -61,15 +62,27 @@ def parse_duration(rval: str, now: datetime) -> datetime: if unit not in UNITS: raise ValueError(f"Invalid unit {unit} for duration") + if unit not in ("years", "months"): + return now - timedelta(**{unit: mag}) + + new_day = now.day + if unit == 'years': - return now.replace(year=now.year - mag) + new_year = now.year - mag + new_month = now.month if unit == 'months': dyear, dmonth = divmod(mag, MONTHS_PER_YEAR) + new_year = now.year - dyear + new_month = (now.month - dmonth) % MONTHS_PER_YEAR + + max_day = monthrange(new_year, new_month)[1] + + if new_day > max_day: + new_day = max_day - return now.replace(year=now.year - dyear, month=now.month - dmonth) + return now.replace(year=new_year, month=new_month, day=new_day) - return now - timedelta(**{unit: mag}) def parse_iso_date(rval: str) -> date: From e25e33f3ce880e64d1b77ae98c6c173ce45259bb Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 3 Sep 2024 09:41:40 -0600 Subject: [PATCH 37/71] Rename func_utils to micro --- lib/pavilion/config.py | 2 +- lib/pavilion/func_utils.py | 86 ------------------------------- lib/pavilion/resolver/resolver.py | 7 +-- lib/pavilion/test_run/test_run.py | 2 +- 4 files changed, 6 insertions(+), 91 deletions(-) delete mode 100644 lib/pavilion/func_utils.py diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index de1f3ecb0..a5246bfe2 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -17,7 +17,7 @@ import yaml_config as yc from pavilion import output from pavilion import errors -from pavilion.func_utils import first, flatten +from pavilion.micro import first, flatten # Figure out what directories we'll search for the base configuration. PAV_CONFIG_SEARCH_DIRS = [Path('./').resolve()] diff --git a/lib/pavilion/func_utils.py b/lib/pavilion/func_utils.py deleted file mode 100644 index a56bcaa0e..000000000 --- a/lib/pavilion/func_utils.py +++ /dev/null @@ -1,86 +0,0 @@ -"""A collection of utilities defined using functional methods""" - -from pathlib import Path -from typing import (List, Union, TypeVar, Iterator, Iterable, Callable, Optional, - Hashable, Dict, Tuple) - - -T = TypeVar('T') -U = TypeVar('U') - - -def partition(pred: Callable[[T], bool], lst: Iterable[T]) -> Tuple[Iterator[T], Iterator[T]]: - """Partition the sequence into two sequences: one consisting of the elements - for which the given predicate is true and one consisting of those for - which it is false.""" - - f_true, f_false = tee(lst) - - return filter(pred, f_true), filterfalse(pred, f_false) - -def flatten(lst: Iterable[Iterable[T]]) -> Iterator[T]: - """Convert a singly nested iterable into an unnested iterable.""" - return chain.from_iterable(lst) - -def remove_all(lst: Iterable[T], item: T) -> Iterator[T]: - """Remove all instances of the given item from the iterable.""" - return filter(lambda x: x != item, lst) - -def unique(lst: Iterable[T]) -> List[T]: - """Return a list of the unique items in the original list.""" - return list(set(lst)) - -def replace(lst: Iterable[T], old: T, new: T) -> Iterator[T]: - """Replace all instances of old with new.""" - return map(lambda x: new if x == old else x, lst) - -def remove_none(lst: Iterable[T]) -> Iterator[T]: - """Remove all instances of None from the iterable.""" - return filter(lambda x: x is not None, lst) - -def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: - """Return the first item of the list that satisfies the given - predicate, or None if no item does.""" - - for item in filter(pred, lst): - return item - -def apply_to_first(func: Callable[[T], U], pred: Callable[[T], bool], - lst: Iterable[T]) -> Optional[U]: - """Apply the function to the first element of the list that satisfies - the given predicate. If no element satisfies the predicate, return None.""" - - fst = first(pred, lst) - - if fst is not None: - return func(fst) - -def get_nested(keys: Iterable[Hashable], nested_dict: Dict) -> Dict: - for key in keys: - nested_dict = nested_dict.get(key, {}) - - return nested_dict - -def listmap(func: Callable[[T], U], lst: Iterable[T]) -> List[U]: - """Map a function over an iterable, but return a list instead - of an iterator.""" - return list(map(func, lst)) - -def exists(path: Path) -> bool: - """Wraps Path.exists, which obviates the need for - a lambda function when mapping it.""" - - return path.exists() - - -Pathlike = Union[Path, str] - -def append_path(suffix: Pathlike) -> Callable[[Path], Path]: - """Constructs a function that appends the given suffix - to path. Intended for use with map.""" - - def f(path: Path) -> Path: - return path / suffix - - return f - diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 8a48cfa48..50a21bda2 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -37,7 +37,8 @@ KEY_NAME_RE) from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary -from pavilion.func_utils import first, listmap, append_path +from pavilion.micro import first, listmap +from pavilion.path_utils import append_path from yaml_config import RequiredError, YamlConfigLoader from .proto_test import RawProtoTest, ProtoTest @@ -666,8 +667,8 @@ def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, op if raw_cfg is None: raise TestConfigError( - f"Could not find {cfg_info.type} config with name {cfg_info.type}" - "in file {cfg_info.path}.") + f"Could not find {cfg_info.type} config with name {cfg_info.name}" + f" in file {cfg_info.path}.") return raw_cfg diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 7923cd34b..b6edfaaaf 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -36,7 +36,7 @@ from pavilion.test_config.file_format import NO_WORKING_DIR from pavilion.test_config.utils import parse_timeout from pavilion.types import ID_Pair -from pavilion.func_utils import get_nested +from pavilion.micro import get_nested from .test_attrs import TestAttributes From 021d177967326ba0dbf8855f344bf03f628542ce Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 5 Sep 2024 09:34:54 -0600 Subject: [PATCH 38/71] Add return type --- lib/pavilion/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 87943d95a..91b415f17 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -308,7 +308,7 @@ def deprecate(self): dep_path.touch() - def _update_src(self): + def _update_src(self) -> Optional[Path]: """Retrieve and/or check the existence of the files needed for the build. This can include pulling from URL's. :returns: src_path, extra_files From 9e4a5f80c555a5de4cb8be061456977edefa8388 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 5 Sep 2024 13:40:48 -0600 Subject: [PATCH 39/71] Flesh out hash test --- .../suites/hash_suite_test_a/hosts.yaml | 3 ++ .../suites/hash_suite_test_a/modes.yaml | 3 ++ .../suites/hash_suite_test_a/os.yaml | 3 ++ .../suites/hash_suite_test_a/suite.yaml | 4 +++ .../suites/hash_suite_test_b/suite.yaml | 4 +++ test/tests/suites_tests.py | 30 ++++++++++++++++++- 6 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_a/hosts.yaml create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_a/modes.yaml create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_a/os.yaml create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_a/suite.yaml create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_b/suite.yaml diff --git a/test/data/pav_config_dir/suites/hash_suite_test_a/hosts.yaml b/test/data/pav_config_dir/suites/hash_suite_test_a/hosts.yaml new file mode 100644 index 000000000..b65a28130 --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_a/hosts.yaml @@ -0,0 +1,3 @@ +hash_host: + variables: + foo: True diff --git a/test/data/pav_config_dir/suites/hash_suite_test_a/modes.yaml b/test/data/pav_config_dir/suites/hash_suite_test_a/modes.yaml new file mode 100644 index 000000000..5074751e2 --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_a/modes.yaml @@ -0,0 +1,3 @@ +hash_mode: + variables: + foo: True diff --git a/test/data/pav_config_dir/suites/hash_suite_test_a/os.yaml b/test/data/pav_config_dir/suites/hash_suite_test_a/os.yaml new file mode 100644 index 000000000..692d50284 --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_a/os.yaml @@ -0,0 +1,3 @@ +hash_os: + variables: + foo: True diff --git a/test/data/pav_config_dir/suites/hash_suite_test_a/suite.yaml b/test/data/pav_config_dir/suites/hash_suite_test_a/suite.yaml new file mode 100644 index 000000000..97ea92e4b --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_a/suite.yaml @@ -0,0 +1,4 @@ +hash_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/data/pav_config_dir/suites/hash_suite_test_b/suite.yaml b/test/data/pav_config_dir/suites/hash_suite_test_b/suite.yaml new file mode 100644 index 000000000..97ea92e4b --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_b/suite.yaml @@ -0,0 +1,4 @@ +hash_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index 21c3fc544..734b1b5d0 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -121,4 +121,32 @@ def test_suites_build_hash(self): """Test that Pavilion ignores config files in the suites directory when computing the build hash.""" - self.assertTrue(False) + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + 'hash_suite_test_a' + ]) + + run_cmd = commands.get_command(args.command_name) + ret = run_cmd.run(self.pav_cfg, args) + + self.assertEqual(ret, 0) + + last_test = run_cmd.last_tests[0] + hash_a = last_test.builder().build_hash + + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + 'hash_suite_test_b' + ]) + + run_cmd = commands.get_command(args.command_name) + ret = run_cmd.run(self.pav_cfg, args) + + self.assertEqual(ret, 0) + + last_test = run_cmd.last_tests[0] + hash_b = last_test.builder().build_hash + + self.assertEqual(hash_a, hash_b) From 35a5e8c3f8c83a6517f460648a9f905b0a54059c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 6 Sep 2024 14:09:32 -0600 Subject: [PATCH 40/71] Fix error due to commented line --- lib/pavilion/resolver/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 50a21bda2..f7ccb6f47 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -855,7 +855,7 @@ def _load_suite_tests(self, suite_name: str): # Basic information that all test configs should have. test_cfg['name'] = test_cfg_name - # test_cfg['cfg_label'] = cfg_label + test_cfg['cfg_label'] = cfg_info.label working_dir = self.pav_cfg['configs'][cfg_info.label]['working_dir'] test_cfg['working_dir'] = working_dir.as_posix() test_cfg['suite'] = suite_name From 3147be14d07dc4b7b2730165308c08a1c81e1977 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 6 Sep 2024 14:37:07 -0600 Subject: [PATCH 41/71] Resolve a bunch of failing tests (WOOHOO!!) --- lib/pavilion/commands/_run.py | 2 +- lib/pavilion/resolver/resolver.py | 20 +++++++++++--------- lib/pavilion/series/series.py | 6 ++++-- test/tests/general_tests.py | 2 -- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/pavilion/commands/_run.py b/lib/pavilion/commands/_run.py index b06bf4079..e535d8ad2 100644 --- a/lib/pavilion/commands/_run.py +++ b/lib/pavilion/commands/_run.py @@ -45,7 +45,7 @@ def run(self, pav_cfg, args): try: tests.append(TestRun.load_from_raw_id(pav_cfg, test_id)) except PavilionError as err: - fprint(self.outfile, "Error loading test '{}'".format(args.test_id)) + fprint(self.outfile, "Error loading test '{}'".format(test_id)) fprint(self.outfile, err.pformat()) # Filter out cancelled tests diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index f7ccb6f47..20484c053 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -416,6 +416,7 @@ def load_iter(self, tests: List[str], modes: List[str] = None, overrides: List[s requests = [TestRequest(req) for req in tests] raw_tests = [] + for request in requests: # Convert each request into a list of RawProtoTest objects. try: @@ -464,7 +465,7 @@ def load_iter(self, tests: List[str], modes: List[str] = None, overrides: List[s raw_tests.append(raw_test) ready_to_resolve.extend(permutations) - + # Now resolve all the string syntax and variables those tests at once. new_resolved_tests = [] for ptest in self._resolve_escapes(ready_to_resolve): @@ -1007,19 +1008,20 @@ def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): :param modes: A list of mode names. """ - if suite_name is not None: - from_suite = True - loader = self._suite_loader - else: - from_suite = False - loader = self._loader + # import pdb; pdb.set_trace() for mode in modes: + mode_cfg_path = None - if from_suite: + if suite_name is not None: label, mode_cfg_path = self._config_path_from_suite(suite_name, "mode") - else: + if mode_cfg_path is None: + from_suite = False label, mode_cfg_path = self._get_test_config_path(mode, "mode") + loader = self._loader + else: + from_suite = True + loader = self._suite_loader cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, label, from_suite) diff --git a/lib/pavilion/series/series.py b/lib/pavilion/series/series.py index f0087607e..b3b44d02f 100644 --- a/lib/pavilion/series/series.py +++ b/lib/pavilion/series/series.py @@ -526,17 +526,19 @@ def _run_set(self, test_set: TestSet, build_only: bool, rebuild: bool, local_bui WAIT_INTERVAL = 0.5 - def wait(self, timeout=None): + def wait(self, timeout: float = None) -> None: """Wait for the series to be complete or the timeout to expire. """ if timeout is None: end = math.inf else: end = time.time() + timeout + while time.time() < end: if self.complete: return - time.sleep(2) + + time.sleep(self.WAIT_INTERVAL) raise TimeoutError("Series {} did not complete before timeout." .format(self._id)) diff --git a/test/tests/general_tests.py b/test/tests/general_tests.py index 01ff6050b..09e5aae37 100644 --- a/test/tests/general_tests.py +++ b/test/tests/general_tests.py @@ -31,11 +31,9 @@ def __init__(self, *args, **kwargs): if not candidates: self.orig_group = None self.alt_group = None - self.alt_group2 = None else: self.orig_group = grp.getgrgid(def_gid).gr_name self.alt_group = candidates[0] # type: grp.struct_group - self.alt_group2 = candidates[1] # type: grp.struct_group self.umask = 0o007 From bc8f7e60d85626095ab0e747b8e90150798c93f2 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 6 Sep 2024 14:38:07 -0600 Subject: [PATCH 42/71] Remove legacy data --- test/data/legacy/795/RUN_COMPLETE | 1 - test/data/legacy/795/attributes | 1 - test/data/legacy/795/build.log | 1 - test/data/legacy/795/build.sh | 8 --- test/data/legacy/795/build/.built_by | 1 - test/data/legacy/795/build/pav_build_log | 1 - test/data/legacy/795/build_dir/pav_build_log | 1 - test/data/legacy/795/build_name | 1 - test/data/legacy/795/build_origin | 1 - test/data/legacy/795/config | 1 - test/data/legacy/795/job_id | 1 - test/data/legacy/795/kickoff.log | 0 test/data/legacy/795/kickoff.sh | 8 --- test/data/legacy/795/results.json | 1 - test/data/legacy/795/results.log | 75 -------------------- test/data/legacy/795/run.log | 0 test/data/legacy/795/run.sh | 9 --- test/data/legacy/795/run.tmpl | 9 --- test/data/legacy/795/status | 13 ---- test/data/legacy/795/variables | 1 - 20 files changed, 134 deletions(-) delete mode 100644 test/data/legacy/795/RUN_COMPLETE delete mode 100644 test/data/legacy/795/attributes delete mode 120000 test/data/legacy/795/build.log delete mode 100755 test/data/legacy/795/build.sh delete mode 120000 test/data/legacy/795/build/.built_by delete mode 120000 test/data/legacy/795/build/pav_build_log delete mode 100644 test/data/legacy/795/build_dir/pav_build_log delete mode 100644 test/data/legacy/795/build_name delete mode 120000 test/data/legacy/795/build_origin delete mode 100644 test/data/legacy/795/config delete mode 100644 test/data/legacy/795/job_id delete mode 100644 test/data/legacy/795/kickoff.log delete mode 100755 test/data/legacy/795/kickoff.sh delete mode 100644 test/data/legacy/795/results.json delete mode 100644 test/data/legacy/795/results.log delete mode 100644 test/data/legacy/795/run.log delete mode 100755 test/data/legacy/795/run.sh delete mode 100755 test/data/legacy/795/run.tmpl delete mode 100644 test/data/legacy/795/status delete mode 100644 test/data/legacy/795/variables diff --git a/test/data/legacy/795/RUN_COMPLETE b/test/data/legacy/795/RUN_COMPLETE deleted file mode 100644 index 3d4f06b64..000000000 --- a/test/data/legacy/795/RUN_COMPLETE +++ /dev/null @@ -1 +0,0 @@ -{"complete": "2020-07-28T16:50:52.574548"} \ No newline at end of file diff --git a/test/data/legacy/795/attributes b/test/data/legacy/795/attributes deleted file mode 100644 index 0d350a77f..000000000 --- a/test/data/legacy/795/attributes +++ /dev/null @@ -1 +0,0 @@ -{"build_only": false, "rebuild": false, "started": "2020-07-28 16:50:52.017103", "finished": "2020-07-28 16:50:52.035368"} \ No newline at end of file diff --git a/test/data/legacy/795/build.log b/test/data/legacy/795/build.log deleted file mode 120000 index aa0ee20ba..000000000 --- a/test/data/legacy/795/build.log +++ /dev/null @@ -1 +0,0 @@ -build/pav_build_log \ No newline at end of file diff --git a/test/data/legacy/795/build.sh b/test/data/legacy/795/build.sh deleted file mode 100755 index a2fa80f86..000000000 --- a/test/data/legacy/795/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# The first (and only) argument of the build script is the test id. -export TEST_ID=${1:-0} -export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml -source /yellow/usr/projects/hpctest/pav2/src/bin/pav-lib.bash - -# No commands given for this script. diff --git a/test/data/legacy/795/build/.built_by b/test/data/legacy/795/build/.built_by deleted file mode 120000 index 026ba8cc3..000000000 --- a/test/data/legacy/795/build/.built_by +++ /dev/null @@ -1 +0,0 @@ -/yellow/usr/projects/hpctest/pav2/working_dir/builds/2de3953430e63026/.built_by \ No newline at end of file diff --git a/test/data/legacy/795/build/pav_build_log b/test/data/legacy/795/build/pav_build_log deleted file mode 120000 index d9d40dc72..000000000 --- a/test/data/legacy/795/build/pav_build_log +++ /dev/null @@ -1 +0,0 @@ -../../../builds/2de3953430e63026/pav_build_log \ No newline at end of file diff --git a/test/data/legacy/795/build_dir/pav_build_log b/test/data/legacy/795/build_dir/pav_build_log deleted file mode 100644 index d7b969606..000000000 --- a/test/data/legacy/795/build_dir/pav_build_log +++ /dev/null @@ -1 +0,0 @@ -The original was empty, so I thought I'd add something. diff --git a/test/data/legacy/795/build_name b/test/data/legacy/795/build_name deleted file mode 100644 index 13160c762..000000000 --- a/test/data/legacy/795/build_name +++ /dev/null @@ -1 +0,0 @@ -2de3953430e63026 \ No newline at end of file diff --git a/test/data/legacy/795/build_origin b/test/data/legacy/795/build_origin deleted file mode 120000 index 1ed6e97da..000000000 --- a/test/data/legacy/795/build_origin +++ /dev/null @@ -1 +0,0 @@ -../../builds/2de3953430e63026 \ No newline at end of file diff --git a/test/data/legacy/795/config b/test/data/legacy/795/config deleted file mode 100644 index be4b5cc81..000000000 --- a/test/data/legacy/795/config +++ /dev/null @@ -1 +0,0 @@ -{"name": "fe", "suite": "mounts", "suite_path": "/yellow/usr/projects/hpctest/pav2/config/tests/mounts.yaml", "host": "fog", "modes": ["postDST-fog"], "inherits_from": "__base__", "subtitle": "hpctest", "group": null, "umask": null, "maintainer": {"name": "Some Body", "email": "somebody@host.org"}, "summary": "Check for the appropriate mounts on the front-ends.", "doc": "\nThis test checks for the existence of several mounted filesystems expected\nfor the system to be usable. This test checks the front-ends as well as the\ncomputes (independent subtests). The test output will include the name of\nthe node that was unable to find one of the mounts. The admins should be\nable to mount these filesystems on any nodes that are missing them, though\nif it is systematic, they may opt for rebuilding or rebroadcasting the images\nto the appropriate nodes and trigger a reboot.\nIf the missing mounts on a toss system are just the conveninece mounts\n('/net' for scratch spaces), the yeti-scripts may need to be run. Check\nthat test under rpmquery.\n", "permute_on": ["scratch_all"], "variables": {"local_rpms": [{"distro": "hpcsoft", "name": "give", "version": "3.1-6", "command": "give --help", "where": "everywhere"}, {"distro": "hpcsoft", "name": "fstools", "version": "2.4-*", "command": "chkhome", "where": "everywhere"}, {"distro": "hpcsoft", "name": "kmod-vtune", "version": "2019.4", "command": "eval lsmod |grep sep5", "where": "computes"}, {"distro": "syssw", "name": "yeti-scripts", "version": "4.*", "command": "ls -ald /run/colorize/iam/$(if [[ \"$(hostname -s)\" == *fey* ]] || [[ \"$SLURM_SUBMIT_HOST\" == *fey* ]] ; then echo \"YELLOW\"; else echo \"TURQUOISE\"; fi)", "where": "everywhere"}], "distro_rpms": [{"distro": "el7", "name": "meld", "version": "", "command": "which meld", "where": "everywhere"}, {"distro": "el7", "name": "pdsh", "version": "", "command": "pdsh -L", "where": "computes"}, {"distro": "el7", "name": "perl-Switch", "version": "", "command": "eval perl -E 'use Switch'", "where": "everywhere"}, {"distro": "chaos", "name": "Lmod", "version": "7.8.16", "command": "eval module avail |& grep hpcsoft", "where": "everywhere"}], "cpuspeedcmd": ["`egrep \"cpu MHz\" /proc/cpuinfo | uniq | awk '{print $4}' | awk -F. '{ s = $1 / 1000 } END {print s}'`"], "hpl_single": [{"header": "LANL CTS1 Single Node Linpack", "summary": "Los Alamos National Laboratory PRETeam Benchmark", "outputfilenamevalue": "HPL.out", "deviceoutvalue": "6", "nsnosvalue": "1", "nsvalue": "102144", "nbsnosvalue": "1", "nbsvalue": "192", "pmapvalue": "0", "processgridnosvalue": "1", "psvalue": "4", "qsvalue": "9", "thresholdvalue": "16.0", "pfactnosvalue": "1", "pfactsvalue": "0", "recstopnosvalue": "1", "nbminsvalue": "4", "nopanelsrecvalue": "1", "ndivsvalue": "2", "norecpanfactvalue": "1", "rfactsvalue": "1", "bcastnosvalue": "1", "bcastsvalues": "1", "lookaheaddepthnosvalues": "1", "depthsvalue": "0", "swapvalue": "0", "swapthreshvalue": "256", "l1formvalue": "0", "uformvalue": "0", "equilibriumvalue": "1", "memoryalignmentvalue": "8"}], "scratch": [{"name": "scratch3-convenience", "path": "/net/scratch3/somebody"}, {"name": "scratch4-convenience", "path": "/net/scratch4/somebody"}], "scratch_all": [{"name": "scratch3-convenience", "path": "/net/scratch3/somebody"}, {"name": "scratch4-convenience", "path": "/net/scratch4/somebody"}, {"name": "scratch3", "path": "/lustre/scratch3/yellow/somebody"}, {"name": "scratch4", "path": "/lustre/scratch4/yellow/somebody"}, {"name": "homespace", "path": "/users/somebody"}, {"name": "hpcsoft", "path": "/usr/projects/hpcsoft"}, {"name": "hpctest", "path": "/usr/projects/hpctest"}], "compilers": ["gcc", "intel", "pgi"], "mpis": ["openmpi", "intel-mpi", "mvapich2"]}, "scheduler": "raw", "only_if": {}, "not_if": {}, "compatible_pav_versions": "", "test_version": "0.0", "build": {"cmds": [], "copy_files": [], "create_files": {}, "env": {}, "extra_files": [], "modules": [], "on_nodes": "False", "preamble": [], "source_path": null, "source_url": null, "source_download": "missing", "specificity": "fog", "timeout": "30", "verbose": "False"}, "run": {"cmds": ["[ -e /usr/projects/hpctest ] || hostname"], "create_files": {}, "env": {}, "modules": [], "preamble": [], "timeout": "300", "verbose": "False"}, "result_evaluate": {}, "result_parse": {"regex": {"result": {"regex": ".*", "action": "store_false", "files": [], "per_file": null, "match_type": null}, "nodes_missing_mounts": {"regex": ".*", "match_type": "all", "action": null, "files": [], "per_file": null}}, "table": {}, "constant": {}, "command": {}, "filecheck": {}, "ior_table": {}}, "slurm_mpi": {"num_nodes": "1", "tasks_per_node": "1", "mem_per_node": null, "partition": "standard", "immediate": "false", "qos": null, "account": null, "reservation": null, "time_limit": null, "include_nodes": null, "exclude_nodes": null, "avail_states": ["IDLE", "MAINT"], "up_states": ["ALLOCATED", "COMPLETING", "IDLE", "MAINT"], "rank_by": null, "bind_to": null, "mca": []}, "slurm": {"num_nodes": "1-all", "tasks_per_node": "1", "mem_per_node": null, "partition": "standard", "immediate": "false", "qos": null, "account": "hpcdev", "reservation": "PreventMaint", "time_limit": null, "include_nodes": null, "exclude_nodes": null, "avail_states": ["IDLE", "MAINT"], "up_states": ["ALLOCATED", "COMPLETING", "IDLE", "MAINT"]}, "raw": {"concurrent": "False"}} diff --git a/test/data/legacy/795/job_id b/test/data/legacy/795/job_id deleted file mode 100644 index 1ab342fad..000000000 --- a/test/data/legacy/795/job_id +++ /dev/null @@ -1 +0,0 @@ -myhost.place.org_21487 diff --git a/test/data/legacy/795/kickoff.log b/test/data/legacy/795/kickoff.log deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/data/legacy/795/kickoff.sh b/test/data/legacy/795/kickoff.sh deleted file mode 100755 index a78b333f7..000000000 --- a/test/data/legacy/795/kickoff.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Redirect all output to kickoff.log -exec >/usr/projects/hpctest/pav2/working_dir/test_runs/0000795/kickoff.log 2>&1 -export PATH=/yellow/usr/projects/hpctest/pav2/src/bin:${PATH} -export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml -export PAV_CONFIG_DIR=/usr/projects/hpctest/pav2//config -pav _run 795 diff --git a/test/data/legacy/795/results.json b/test/data/legacy/795/results.json deleted file mode 100644 index b8970e775..000000000 --- a/test/data/legacy/795/results.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "mounts.fe.hpctest", "id": 795, "test_version": "0.0", "pav_version": "2.2", "created": "2020-07-28 16:50:52.034570", "started": "2020-07-28 16:50:52.017103", "finished": "2020-07-28 16:50:52.035368", "duration": 0.018265, "user": "somebody", "job_id": "myhost.place.org_21487", "sched": {"total_mem": "125821", "free_mem": "120482", "min_cpus": "36", "cpus": "36", "avail_mem": "120946", "test_cmd": "", "min_mem": "131933.148"}, "sys_name": "myhost", "pav_result_errors": [], "n": {}, "fn": {}, "return_value": 0, "result": "PASS", "nodes_missing_mounts": []} diff --git a/test/data/legacy/795/results.log b/test/data/legacy/795/results.log deleted file mode 100644 index 1f4c7179b..000000000 --- a/test/data/legacy/795/results.log +++ /dev/null @@ -1,75 +0,0 @@ -Gathering base results. - Base results: - {'created': '2020-07-28 16:50:52.034570', - 'duration': 0.018265, - 'finished': '2020-07-28 16:50:52.035368', - 'fn': {}, - 'id': 795, - 'job_id': 'myhost.place.org_21487', - 'n': {}, - 'name': 'mounts.fe.hpctest', - 'pav_result_errors': [], - 'pav_version': '2.2', - 'return_value': 0, - 'sched': {'avail_mem': '120946', - 'cpus': '36', - 'free_mem': '120482', - 'min_cpus': '36', - 'min_mem': '131933.148', - 'test_cmd': '', - 'total_mem': '125821'}, - 'started': '2020-07-28 16:50:52.017103', - 'sys_name': 'myhost', - 'test_version': '0.0', - 'user': 'somebody'} - Starting result parsing. - Got result parser configs: - {'command': {}, - 'constant': {}, - 'filecheck': {}, - 'ior_table': {}, - 'regex': {'nodes_missing_mounts': {'action': None, - 'files': [], - 'match_type': 'all', - 'per_file': None, - 'regex': '.*'}, - 'result': {'action': 'store_false', - 'files': [], - 'match_type': None, - 'per_file': None, - 'regex': '.*'}}, - 'table': {}} - --------------- - Parsing results for parser regex - Parsing value for key 'result' - Looking for files that match file globs: ['../run.log'] - Found 1 matching files. - Results will be stored with action 'store_false' - Parsing for file '/usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log': - Raw parse result: 'None' - Stored value 'True' for file 'run.log' - Results for each found files: - - /usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log: True - Handling results for key 'result' on a per-file basis with per_file setting 'first' - first: Picked non-empty value 'True' - Parsing value for key 'nodes_missing_mounts' - Looking for files that match file globs: ['../run.log'] - Found 1 matching files. - Results will be stored with action 'store' - Parsing for file '/usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log': - Raw parse result: '[]' - Stored value '[]' for file 'run.log' - Results for each found files: - - /usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log: [] - Handling results for key 'nodes_missing_mounts' on a per-file basis with per_file setting 'first' - first: Picked non-empty value '[]' - Parsing results for parser table - Parsing results for parser constant - Parsing results for parser command - Parsing results for parser filecheck - Parsing results for parser ior_table -Evaluating result evaluations. -Resolving evaluations. -Finished resolving expressions -Set final result key to: 'PASS' -See results.json for the final result json. diff --git a/test/data/legacy/795/run.log b/test/data/legacy/795/run.log deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/data/legacy/795/run.sh b/test/data/legacy/795/run.sh deleted file mode 100755 index 4df45f2ab..000000000 --- a/test/data/legacy/795/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# The first (and only) argument of the build script is the test id. -export TEST_ID=${1:-0} -export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml -source /yellow/usr/projects/hpctest/pav2/src/bin/pav-lib.bash - -# Perform the sequence of test commands. -[ -e /usr/projects/hpctest ] || hostname diff --git a/test/data/legacy/795/run.tmpl b/test/data/legacy/795/run.tmpl deleted file mode 100755 index 4df45f2ab..000000000 --- a/test/data/legacy/795/run.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# The first (and only) argument of the build script is the test id. -export TEST_ID=${1:-0} -export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml -source /yellow/usr/projects/hpctest/pav2/src/bin/pav-lib.bash - -# Perform the sequence of test commands. -[ -e /usr/projects/hpctest ] || hostname diff --git a/test/data/legacy/795/status b/test/data/legacy/795/status deleted file mode 100644 index e78eca9fd..000000000 --- a/test/data/legacy/795/status +++ /dev/null @@ -1,13 +0,0 @@ -2020-07-28T16:50:38.920490 CREATED Created status file. -2020-07-28T16:50:38.921466 CREATED Test directory and status file created. -2020-07-28T16:50:38.927275 BUILD_CREATED Builder created. -2020-07-28T16:50:38.931711 CREATED Test directory setup complete. -2020-07-28T16:50:47.701546 BUILD_REUSED Test 2de3953430e63026 run 795 reusing build. -2020-07-28T16:50:50.564503 SCHEDULED Test raw has job ID myhost.place.org_21487. -2020-07-28T16:50:52.014514 PREPPING_RUN Converting run template into run script. -2020-07-28T16:50:52.016367 RUNNING Starting the run script. -2020-07-28T16:50:52.019109 RUNNING Currently running. -2020-07-28T16:50:52.037495 RUN_DONE Test run has completed. -2020-07-28T16:50:52.042470 RESULTS Parsing 6 result types. -2020-07-28T16:50:52.044960 RESULTS Performing 0 result evaluations. -2020-07-28T16:50:52.572860 COMPLETE The test completed with result: PASS diff --git a/test/data/legacy/795/variables b/test/data/legacy/795/variables deleted file mode 100644 index 88a2e5c8f..000000000 --- a/test/data/legacy/795/variables +++ /dev/null @@ -1 +0,0 @@ -{"sys": {"sys_name": ["fog"], "host_name": ["fog"], "host_os": [{"name": "toss", "version": "3"}], "sys_os": [{"name": "toss", "version": "3"}], "host_arch": ["x86_64"], "sys_host": ["fg-fey1"], "sys_arch": ["x86_64"], "sys_net": ["yellow"]}, "host_name": {}, "host_os": {}, "host_arch": {}, "pav": {"user": ["somebody"], "year": ["2020"], "time": ["16:50:24.957844"], "weekday": ["Tuesday"], "month": ["7"], "timestamp": ["1595976624.9580042"], "day": ["28"], "version": ["2.2"]}, "var": {"local_rpms": [{"distro": "hpcsoft", "name": "give", "version": "3.1-6", "command": "give --help", "where": "everywhere"}, {"distro": "hpcsoft", "name": "fstools", "version": "2.4-*", "command": "chkhome", "where": "everywhere"}, {"distro": "hpcsoft", "name": "kmod-vtune", "version": "2019.4", "command": "eval lsmod |grep sep5", "where": "computes"}, {"distro": "syssw", "name": "yeti-scripts", "version": "4.*", "command": "ls -ald /run/colorize/iam/$(if [[ \"$(hostname -s)\" == *fey* ]] || [[ \"$SLURM_SUBMIT_HOST\" == *fey* ]] ; then echo \"YELLOW\"; else echo \"TURQUOISE\"; fi)", "where": "everywhere"}], "distro_rpms": [{"distro": "el7", "name": "meld", "version": "", "command": "which meld", "where": "everywhere"}, {"distro": "el7", "name": "pdsh", "version": "", "command": "pdsh -L", "where": "computes"}, {"distro": "el7", "name": "perl-Switch", "version": "", "command": "eval perl -E 'use Switch'", "where": "everywhere"}, {"distro": "chaos", "name": "Lmod", "version": "7.8.16", "command": "eval module avail |& grep hpcsoft", "where": "everywhere"}], "cpuspeedcmd": ["`egrep \"cpu MHz\" /proc/cpuinfo | uniq | awk '{print $4}' | awk -F. '{ s = $1 / 1000 } END {print s}'`"], "hpl_single": [{"header": "LANL CTS1 Single Node Linpack", "summary": "Los Alamos National Laboratory PRETeam Benchmark", "outputfilenamevalue": "HPL.out", "deviceoutvalue": "6", "nsnosvalue": "1", "nsvalue": "102144", "nbsnosvalue": "1", "nbsvalue": "192", "pmapvalue": "0", "processgridnosvalue": "1", "psvalue": "4", "qsvalue": "9", "thresholdvalue": "16.0", "pfactnosvalue": "1", "pfactsvalue": "0", "recstopnosvalue": "1", "nbminsvalue": "4", "nopanelsrecvalue": "1", "ndivsvalue": "2", "norecpanfactvalue": "1", "rfactsvalue": "1", "bcastnosvalue": "1", "bcastsvalues": "1", "lookaheaddepthnosvalues": "1", "depthsvalue": "0", "swapvalue": "0", "swapthreshvalue": "256", "l1formvalue": "0", "uformvalue": "0", "equilibriumvalue": "1", "memoryalignmentvalue": "8"}], "scratch": [{"name": "scratch3-convenience", "path": "/net/scratch3/somebody"}, {"name": "scratch4-convenience", "path": "/net/scratch4/somebody"}], "scratch_all": [{"name": "hpctest", "path": "/usr/projects/hpctest"}], "compilers": ["gcc", "intel", "pgi"], "mpis": ["openmpi", "intel-mpi", "mvapich2"]}, "sched": {"total_mem": ["125821"], "free_mem": ["120482"], "min_cpus": ["36"], "cpus": ["36"], "avail_mem": ["120946"], "test_cmd": [""], "min_mem": ["131933.148"]}, "__deferred": []} From 2d23e8462b2c628b45e13722c68fd8e289405731 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 6 Sep 2024 15:31:03 -0600 Subject: [PATCH 43/71] Revert "Remove legacy data" This reverts commit 2c7f6fab294451a5b8786501fa6ea6b3387bfa04. --- test/data/legacy/795/RUN_COMPLETE | 1 + test/data/legacy/795/attributes | 1 + test/data/legacy/795/build.log | 1 + test/data/legacy/795/build.sh | 8 +++ test/data/legacy/795/build/.built_by | 1 + test/data/legacy/795/build/pav_build_log | 1 + test/data/legacy/795/build_dir/pav_build_log | 1 + test/data/legacy/795/build_name | 1 + test/data/legacy/795/build_origin | 1 + test/data/legacy/795/config | 1 + test/data/legacy/795/job_id | 1 + test/data/legacy/795/kickoff.log | 0 test/data/legacy/795/kickoff.sh | 8 +++ test/data/legacy/795/results.json | 1 + test/data/legacy/795/results.log | 75 ++++++++++++++++++++ test/data/legacy/795/run.log | 0 test/data/legacy/795/run.sh | 9 +++ test/data/legacy/795/run.tmpl | 9 +++ test/data/legacy/795/status | 13 ++++ test/data/legacy/795/variables | 1 + 20 files changed, 134 insertions(+) create mode 100644 test/data/legacy/795/RUN_COMPLETE create mode 100644 test/data/legacy/795/attributes create mode 120000 test/data/legacy/795/build.log create mode 100755 test/data/legacy/795/build.sh create mode 120000 test/data/legacy/795/build/.built_by create mode 120000 test/data/legacy/795/build/pav_build_log create mode 100644 test/data/legacy/795/build_dir/pav_build_log create mode 100644 test/data/legacy/795/build_name create mode 120000 test/data/legacy/795/build_origin create mode 100644 test/data/legacy/795/config create mode 100644 test/data/legacy/795/job_id create mode 100644 test/data/legacy/795/kickoff.log create mode 100755 test/data/legacy/795/kickoff.sh create mode 100644 test/data/legacy/795/results.json create mode 100644 test/data/legacy/795/results.log create mode 100644 test/data/legacy/795/run.log create mode 100755 test/data/legacy/795/run.sh create mode 100755 test/data/legacy/795/run.tmpl create mode 100644 test/data/legacy/795/status create mode 100644 test/data/legacy/795/variables diff --git a/test/data/legacy/795/RUN_COMPLETE b/test/data/legacy/795/RUN_COMPLETE new file mode 100644 index 000000000..3d4f06b64 --- /dev/null +++ b/test/data/legacy/795/RUN_COMPLETE @@ -0,0 +1 @@ +{"complete": "2020-07-28T16:50:52.574548"} \ No newline at end of file diff --git a/test/data/legacy/795/attributes b/test/data/legacy/795/attributes new file mode 100644 index 000000000..0d350a77f --- /dev/null +++ b/test/data/legacy/795/attributes @@ -0,0 +1 @@ +{"build_only": false, "rebuild": false, "started": "2020-07-28 16:50:52.017103", "finished": "2020-07-28 16:50:52.035368"} \ No newline at end of file diff --git a/test/data/legacy/795/build.log b/test/data/legacy/795/build.log new file mode 120000 index 000000000..aa0ee20ba --- /dev/null +++ b/test/data/legacy/795/build.log @@ -0,0 +1 @@ +build/pav_build_log \ No newline at end of file diff --git a/test/data/legacy/795/build.sh b/test/data/legacy/795/build.sh new file mode 100755 index 000000000..a2fa80f86 --- /dev/null +++ b/test/data/legacy/795/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# The first (and only) argument of the build script is the test id. +export TEST_ID=${1:-0} +export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml +source /yellow/usr/projects/hpctest/pav2/src/bin/pav-lib.bash + +# No commands given for this script. diff --git a/test/data/legacy/795/build/.built_by b/test/data/legacy/795/build/.built_by new file mode 120000 index 000000000..026ba8cc3 --- /dev/null +++ b/test/data/legacy/795/build/.built_by @@ -0,0 +1 @@ +/yellow/usr/projects/hpctest/pav2/working_dir/builds/2de3953430e63026/.built_by \ No newline at end of file diff --git a/test/data/legacy/795/build/pav_build_log b/test/data/legacy/795/build/pav_build_log new file mode 120000 index 000000000..d9d40dc72 --- /dev/null +++ b/test/data/legacy/795/build/pav_build_log @@ -0,0 +1 @@ +../../../builds/2de3953430e63026/pav_build_log \ No newline at end of file diff --git a/test/data/legacy/795/build_dir/pav_build_log b/test/data/legacy/795/build_dir/pav_build_log new file mode 100644 index 000000000..d7b969606 --- /dev/null +++ b/test/data/legacy/795/build_dir/pav_build_log @@ -0,0 +1 @@ +The original was empty, so I thought I'd add something. diff --git a/test/data/legacy/795/build_name b/test/data/legacy/795/build_name new file mode 100644 index 000000000..13160c762 --- /dev/null +++ b/test/data/legacy/795/build_name @@ -0,0 +1 @@ +2de3953430e63026 \ No newline at end of file diff --git a/test/data/legacy/795/build_origin b/test/data/legacy/795/build_origin new file mode 120000 index 000000000..1ed6e97da --- /dev/null +++ b/test/data/legacy/795/build_origin @@ -0,0 +1 @@ +../../builds/2de3953430e63026 \ No newline at end of file diff --git a/test/data/legacy/795/config b/test/data/legacy/795/config new file mode 100644 index 000000000..be4b5cc81 --- /dev/null +++ b/test/data/legacy/795/config @@ -0,0 +1 @@ +{"name": "fe", "suite": "mounts", "suite_path": "/yellow/usr/projects/hpctest/pav2/config/tests/mounts.yaml", "host": "fog", "modes": ["postDST-fog"], "inherits_from": "__base__", "subtitle": "hpctest", "group": null, "umask": null, "maintainer": {"name": "Some Body", "email": "somebody@host.org"}, "summary": "Check for the appropriate mounts on the front-ends.", "doc": "\nThis test checks for the existence of several mounted filesystems expected\nfor the system to be usable. This test checks the front-ends as well as the\ncomputes (independent subtests). The test output will include the name of\nthe node that was unable to find one of the mounts. The admins should be\nable to mount these filesystems on any nodes that are missing them, though\nif it is systematic, they may opt for rebuilding or rebroadcasting the images\nto the appropriate nodes and trigger a reboot.\nIf the missing mounts on a toss system are just the conveninece mounts\n('/net' for scratch spaces), the yeti-scripts may need to be run. Check\nthat test under rpmquery.\n", "permute_on": ["scratch_all"], "variables": {"local_rpms": [{"distro": "hpcsoft", "name": "give", "version": "3.1-6", "command": "give --help", "where": "everywhere"}, {"distro": "hpcsoft", "name": "fstools", "version": "2.4-*", "command": "chkhome", "where": "everywhere"}, {"distro": "hpcsoft", "name": "kmod-vtune", "version": "2019.4", "command": "eval lsmod |grep sep5", "where": "computes"}, {"distro": "syssw", "name": "yeti-scripts", "version": "4.*", "command": "ls -ald /run/colorize/iam/$(if [[ \"$(hostname -s)\" == *fey* ]] || [[ \"$SLURM_SUBMIT_HOST\" == *fey* ]] ; then echo \"YELLOW\"; else echo \"TURQUOISE\"; fi)", "where": "everywhere"}], "distro_rpms": [{"distro": "el7", "name": "meld", "version": "", "command": "which meld", "where": "everywhere"}, {"distro": "el7", "name": "pdsh", "version": "", "command": "pdsh -L", "where": "computes"}, {"distro": "el7", "name": "perl-Switch", "version": "", "command": "eval perl -E 'use Switch'", "where": "everywhere"}, {"distro": "chaos", "name": "Lmod", "version": "7.8.16", "command": "eval module avail |& grep hpcsoft", "where": "everywhere"}], "cpuspeedcmd": ["`egrep \"cpu MHz\" /proc/cpuinfo | uniq | awk '{print $4}' | awk -F. '{ s = $1 / 1000 } END {print s}'`"], "hpl_single": [{"header": "LANL CTS1 Single Node Linpack", "summary": "Los Alamos National Laboratory PRETeam Benchmark", "outputfilenamevalue": "HPL.out", "deviceoutvalue": "6", "nsnosvalue": "1", "nsvalue": "102144", "nbsnosvalue": "1", "nbsvalue": "192", "pmapvalue": "0", "processgridnosvalue": "1", "psvalue": "4", "qsvalue": "9", "thresholdvalue": "16.0", "pfactnosvalue": "1", "pfactsvalue": "0", "recstopnosvalue": "1", "nbminsvalue": "4", "nopanelsrecvalue": "1", "ndivsvalue": "2", "norecpanfactvalue": "1", "rfactsvalue": "1", "bcastnosvalue": "1", "bcastsvalues": "1", "lookaheaddepthnosvalues": "1", "depthsvalue": "0", "swapvalue": "0", "swapthreshvalue": "256", "l1formvalue": "0", "uformvalue": "0", "equilibriumvalue": "1", "memoryalignmentvalue": "8"}], "scratch": [{"name": "scratch3-convenience", "path": "/net/scratch3/somebody"}, {"name": "scratch4-convenience", "path": "/net/scratch4/somebody"}], "scratch_all": [{"name": "scratch3-convenience", "path": "/net/scratch3/somebody"}, {"name": "scratch4-convenience", "path": "/net/scratch4/somebody"}, {"name": "scratch3", "path": "/lustre/scratch3/yellow/somebody"}, {"name": "scratch4", "path": "/lustre/scratch4/yellow/somebody"}, {"name": "homespace", "path": "/users/somebody"}, {"name": "hpcsoft", "path": "/usr/projects/hpcsoft"}, {"name": "hpctest", "path": "/usr/projects/hpctest"}], "compilers": ["gcc", "intel", "pgi"], "mpis": ["openmpi", "intel-mpi", "mvapich2"]}, "scheduler": "raw", "only_if": {}, "not_if": {}, "compatible_pav_versions": "", "test_version": "0.0", "build": {"cmds": [], "copy_files": [], "create_files": {}, "env": {}, "extra_files": [], "modules": [], "on_nodes": "False", "preamble": [], "source_path": null, "source_url": null, "source_download": "missing", "specificity": "fog", "timeout": "30", "verbose": "False"}, "run": {"cmds": ["[ -e /usr/projects/hpctest ] || hostname"], "create_files": {}, "env": {}, "modules": [], "preamble": [], "timeout": "300", "verbose": "False"}, "result_evaluate": {}, "result_parse": {"regex": {"result": {"regex": ".*", "action": "store_false", "files": [], "per_file": null, "match_type": null}, "nodes_missing_mounts": {"regex": ".*", "match_type": "all", "action": null, "files": [], "per_file": null}}, "table": {}, "constant": {}, "command": {}, "filecheck": {}, "ior_table": {}}, "slurm_mpi": {"num_nodes": "1", "tasks_per_node": "1", "mem_per_node": null, "partition": "standard", "immediate": "false", "qos": null, "account": null, "reservation": null, "time_limit": null, "include_nodes": null, "exclude_nodes": null, "avail_states": ["IDLE", "MAINT"], "up_states": ["ALLOCATED", "COMPLETING", "IDLE", "MAINT"], "rank_by": null, "bind_to": null, "mca": []}, "slurm": {"num_nodes": "1-all", "tasks_per_node": "1", "mem_per_node": null, "partition": "standard", "immediate": "false", "qos": null, "account": "hpcdev", "reservation": "PreventMaint", "time_limit": null, "include_nodes": null, "exclude_nodes": null, "avail_states": ["IDLE", "MAINT"], "up_states": ["ALLOCATED", "COMPLETING", "IDLE", "MAINT"]}, "raw": {"concurrent": "False"}} diff --git a/test/data/legacy/795/job_id b/test/data/legacy/795/job_id new file mode 100644 index 000000000..1ab342fad --- /dev/null +++ b/test/data/legacy/795/job_id @@ -0,0 +1 @@ +myhost.place.org_21487 diff --git a/test/data/legacy/795/kickoff.log b/test/data/legacy/795/kickoff.log new file mode 100644 index 000000000..e69de29bb diff --git a/test/data/legacy/795/kickoff.sh b/test/data/legacy/795/kickoff.sh new file mode 100755 index 000000000..a78b333f7 --- /dev/null +++ b/test/data/legacy/795/kickoff.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Redirect all output to kickoff.log +exec >/usr/projects/hpctest/pav2/working_dir/test_runs/0000795/kickoff.log 2>&1 +export PATH=/yellow/usr/projects/hpctest/pav2/src/bin:${PATH} +export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml +export PAV_CONFIG_DIR=/usr/projects/hpctest/pav2//config +pav _run 795 diff --git a/test/data/legacy/795/results.json b/test/data/legacy/795/results.json new file mode 100644 index 000000000..b8970e775 --- /dev/null +++ b/test/data/legacy/795/results.json @@ -0,0 +1 @@ +{"name": "mounts.fe.hpctest", "id": 795, "test_version": "0.0", "pav_version": "2.2", "created": "2020-07-28 16:50:52.034570", "started": "2020-07-28 16:50:52.017103", "finished": "2020-07-28 16:50:52.035368", "duration": 0.018265, "user": "somebody", "job_id": "myhost.place.org_21487", "sched": {"total_mem": "125821", "free_mem": "120482", "min_cpus": "36", "cpus": "36", "avail_mem": "120946", "test_cmd": "", "min_mem": "131933.148"}, "sys_name": "myhost", "pav_result_errors": [], "n": {}, "fn": {}, "return_value": 0, "result": "PASS", "nodes_missing_mounts": []} diff --git a/test/data/legacy/795/results.log b/test/data/legacy/795/results.log new file mode 100644 index 000000000..1f4c7179b --- /dev/null +++ b/test/data/legacy/795/results.log @@ -0,0 +1,75 @@ +Gathering base results. + Base results: + {'created': '2020-07-28 16:50:52.034570', + 'duration': 0.018265, + 'finished': '2020-07-28 16:50:52.035368', + 'fn': {}, + 'id': 795, + 'job_id': 'myhost.place.org_21487', + 'n': {}, + 'name': 'mounts.fe.hpctest', + 'pav_result_errors': [], + 'pav_version': '2.2', + 'return_value': 0, + 'sched': {'avail_mem': '120946', + 'cpus': '36', + 'free_mem': '120482', + 'min_cpus': '36', + 'min_mem': '131933.148', + 'test_cmd': '', + 'total_mem': '125821'}, + 'started': '2020-07-28 16:50:52.017103', + 'sys_name': 'myhost', + 'test_version': '0.0', + 'user': 'somebody'} + Starting result parsing. + Got result parser configs: + {'command': {}, + 'constant': {}, + 'filecheck': {}, + 'ior_table': {}, + 'regex': {'nodes_missing_mounts': {'action': None, + 'files': [], + 'match_type': 'all', + 'per_file': None, + 'regex': '.*'}, + 'result': {'action': 'store_false', + 'files': [], + 'match_type': None, + 'per_file': None, + 'regex': '.*'}}, + 'table': {}} + --------------- + Parsing results for parser regex + Parsing value for key 'result' + Looking for files that match file globs: ['../run.log'] + Found 1 matching files. + Results will be stored with action 'store_false' + Parsing for file '/usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log': + Raw parse result: 'None' + Stored value 'True' for file 'run.log' + Results for each found files: + - /usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log: True + Handling results for key 'result' on a per-file basis with per_file setting 'first' + first: Picked non-empty value 'True' + Parsing value for key 'nodes_missing_mounts' + Looking for files that match file globs: ['../run.log'] + Found 1 matching files. + Results will be stored with action 'store' + Parsing for file '/usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log': + Raw parse result: '[]' + Stored value '[]' for file 'run.log' + Results for each found files: + - /usr/projects/hpctest/pav2/working_dir/test_runs/0000795/build/../run.log: [] + Handling results for key 'nodes_missing_mounts' on a per-file basis with per_file setting 'first' + first: Picked non-empty value '[]' + Parsing results for parser table + Parsing results for parser constant + Parsing results for parser command + Parsing results for parser filecheck + Parsing results for parser ior_table +Evaluating result evaluations. +Resolving evaluations. +Finished resolving expressions +Set final result key to: 'PASS' +See results.json for the final result json. diff --git a/test/data/legacy/795/run.log b/test/data/legacy/795/run.log new file mode 100644 index 000000000..e69de29bb diff --git a/test/data/legacy/795/run.sh b/test/data/legacy/795/run.sh new file mode 100755 index 000000000..4df45f2ab --- /dev/null +++ b/test/data/legacy/795/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# The first (and only) argument of the build script is the test id. +export TEST_ID=${1:-0} +export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml +source /yellow/usr/projects/hpctest/pav2/src/bin/pav-lib.bash + +# Perform the sequence of test commands. +[ -e /usr/projects/hpctest ] || hostname diff --git a/test/data/legacy/795/run.tmpl b/test/data/legacy/795/run.tmpl new file mode 100755 index 000000000..4df45f2ab --- /dev/null +++ b/test/data/legacy/795/run.tmpl @@ -0,0 +1,9 @@ +#!/bin/bash + +# The first (and only) argument of the build script is the test id. +export TEST_ID=${1:-0} +export PAV_CONFIG_FILE=/usr/projects/hpctest/pav2/config/pavilion.yaml +source /yellow/usr/projects/hpctest/pav2/src/bin/pav-lib.bash + +# Perform the sequence of test commands. +[ -e /usr/projects/hpctest ] || hostname diff --git a/test/data/legacy/795/status b/test/data/legacy/795/status new file mode 100644 index 000000000..e78eca9fd --- /dev/null +++ b/test/data/legacy/795/status @@ -0,0 +1,13 @@ +2020-07-28T16:50:38.920490 CREATED Created status file. +2020-07-28T16:50:38.921466 CREATED Test directory and status file created. +2020-07-28T16:50:38.927275 BUILD_CREATED Builder created. +2020-07-28T16:50:38.931711 CREATED Test directory setup complete. +2020-07-28T16:50:47.701546 BUILD_REUSED Test 2de3953430e63026 run 795 reusing build. +2020-07-28T16:50:50.564503 SCHEDULED Test raw has job ID myhost.place.org_21487. +2020-07-28T16:50:52.014514 PREPPING_RUN Converting run template into run script. +2020-07-28T16:50:52.016367 RUNNING Starting the run script. +2020-07-28T16:50:52.019109 RUNNING Currently running. +2020-07-28T16:50:52.037495 RUN_DONE Test run has completed. +2020-07-28T16:50:52.042470 RESULTS Parsing 6 result types. +2020-07-28T16:50:52.044960 RESULTS Performing 0 result evaluations. +2020-07-28T16:50:52.572860 COMPLETE The test completed with result: PASS diff --git a/test/data/legacy/795/variables b/test/data/legacy/795/variables new file mode 100644 index 000000000..88a2e5c8f --- /dev/null +++ b/test/data/legacy/795/variables @@ -0,0 +1 @@ +{"sys": {"sys_name": ["fog"], "host_name": ["fog"], "host_os": [{"name": "toss", "version": "3"}], "sys_os": [{"name": "toss", "version": "3"}], "host_arch": ["x86_64"], "sys_host": ["fg-fey1"], "sys_arch": ["x86_64"], "sys_net": ["yellow"]}, "host_name": {}, "host_os": {}, "host_arch": {}, "pav": {"user": ["somebody"], "year": ["2020"], "time": ["16:50:24.957844"], "weekday": ["Tuesday"], "month": ["7"], "timestamp": ["1595976624.9580042"], "day": ["28"], "version": ["2.2"]}, "var": {"local_rpms": [{"distro": "hpcsoft", "name": "give", "version": "3.1-6", "command": "give --help", "where": "everywhere"}, {"distro": "hpcsoft", "name": "fstools", "version": "2.4-*", "command": "chkhome", "where": "everywhere"}, {"distro": "hpcsoft", "name": "kmod-vtune", "version": "2019.4", "command": "eval lsmod |grep sep5", "where": "computes"}, {"distro": "syssw", "name": "yeti-scripts", "version": "4.*", "command": "ls -ald /run/colorize/iam/$(if [[ \"$(hostname -s)\" == *fey* ]] || [[ \"$SLURM_SUBMIT_HOST\" == *fey* ]] ; then echo \"YELLOW\"; else echo \"TURQUOISE\"; fi)", "where": "everywhere"}], "distro_rpms": [{"distro": "el7", "name": "meld", "version": "", "command": "which meld", "where": "everywhere"}, {"distro": "el7", "name": "pdsh", "version": "", "command": "pdsh -L", "where": "computes"}, {"distro": "el7", "name": "perl-Switch", "version": "", "command": "eval perl -E 'use Switch'", "where": "everywhere"}, {"distro": "chaos", "name": "Lmod", "version": "7.8.16", "command": "eval module avail |& grep hpcsoft", "where": "everywhere"}], "cpuspeedcmd": ["`egrep \"cpu MHz\" /proc/cpuinfo | uniq | awk '{print $4}' | awk -F. '{ s = $1 / 1000 } END {print s}'`"], "hpl_single": [{"header": "LANL CTS1 Single Node Linpack", "summary": "Los Alamos National Laboratory PRETeam Benchmark", "outputfilenamevalue": "HPL.out", "deviceoutvalue": "6", "nsnosvalue": "1", "nsvalue": "102144", "nbsnosvalue": "1", "nbsvalue": "192", "pmapvalue": "0", "processgridnosvalue": "1", "psvalue": "4", "qsvalue": "9", "thresholdvalue": "16.0", "pfactnosvalue": "1", "pfactsvalue": "0", "recstopnosvalue": "1", "nbminsvalue": "4", "nopanelsrecvalue": "1", "ndivsvalue": "2", "norecpanfactvalue": "1", "rfactsvalue": "1", "bcastnosvalue": "1", "bcastsvalues": "1", "lookaheaddepthnosvalues": "1", "depthsvalue": "0", "swapvalue": "0", "swapthreshvalue": "256", "l1formvalue": "0", "uformvalue": "0", "equilibriumvalue": "1", "memoryalignmentvalue": "8"}], "scratch": [{"name": "scratch3-convenience", "path": "/net/scratch3/somebody"}, {"name": "scratch4-convenience", "path": "/net/scratch4/somebody"}], "scratch_all": [{"name": "hpctest", "path": "/usr/projects/hpctest"}], "compilers": ["gcc", "intel", "pgi"], "mpis": ["openmpi", "intel-mpi", "mvapich2"]}, "sched": {"total_mem": ["125821"], "free_mem": ["120482"], "min_cpus": ["36"], "cpus": ["36"], "avail_mem": ["120946"], "test_cmd": [""], "min_mem": ["131933.148"]}, "__deferred": []} From 1f6e2762c6a08288d98cca0e9e7b2ac2ba7303bc Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 6 Sep 2024 15:34:08 -0600 Subject: [PATCH 44/71] Fix error due to introduction of ConfigInfo --- lib/pavilion/commands/show.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/commands/show.py b/lib/pavilion/commands/show.py index ec9526bc3..d228ec46e 100644 --- a/lib/pavilion/commands/show.py +++ b/lib/pavilion/commands/show.py @@ -492,7 +492,9 @@ def show_vars(self, pav_cfg, cfg, conf_type): """Show the variables of a config, each variable is displayed as a table.""" - _, file = resolver.TestConfigResolver(pav_cfg).find_config(conf_type, cfg) + cfg_info = resolver.TestConfigResolver(pav_cfg).find_config(conf_type, cfg) + file = cfg_info.path + if file is None: output.fprint( self.errfile, @@ -604,7 +606,9 @@ def show_configs_table(self, pav_cfg, conf_type, errors=False, def show_full_config(self, pav_cfg, cfg_name, conf_type): """Show the full config of a given os/host/mode.""" - _, file = resolver.TestConfigResolver(pav_cfg).find_config(conf_type, cfg_name) + cfg_info = resolver.TestConfigResolver(pav_cfg).find_config(conf_type, cfg_name) + file = cfg_info.path + config_data = None if file is not None: with file.open() as config_file: From 0c26d17c361512b80ac4b7e848ec7d239ed1688a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 6 Sep 2024 16:26:54 -0600 Subject: [PATCH 45/71] Implement true directory hashing --- lib/pavilion/builder.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 91b415f17..d3edce413 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -15,7 +15,7 @@ import time import urllib.parse from pathlib import Path -from typing import Union, Dict, Optional +from typing import Union, Dict, Optional, List, TextIO from contextlib import ExitStack import pavilion.config @@ -27,6 +27,7 @@ from pavilion.status_file import TestStatusFile, STATES from pavilion.test_config import parse_timeout from pavilion.test_config.spack import SpackEnvConfig +from pavilion.micro import set_default class TestBuilder: @@ -961,7 +962,7 @@ def _hash_file(self, path, save=True): return file_hash @classmethod - def _hash_io(cls, contents): + def _hash_io(cls, contents: TextIO) -> bytes: """Hash the given file in IOString format. :param IOString contents: file name (as relative path to build directory) and file contents to hash.""" @@ -974,17 +975,27 @@ def _hash_io(cls, contents): return hash_obj.digest() - @staticmethod - def _hash_dir(path): - """Instead of hashing the files within a directory, we just create a - 'hash' based on it's name and mtime, assuming we've run _date_dir - on it before hand. This produces an arbitrary string, not a hash. - :param Path path: The path to the directory. - :returns: The 'hash' + @classmethod + def _hash_dir(cls, path: Path, exclude: List[str] = None) -> str: + """Recursively hash the files in the given directory, excluding + files with the given suffixes. + + We might need to modify this to handle circular symlinks. """ - dir_stat = path.stat() - return '{} {:0.5f}'.format(path, dir_stat.st_mtime).encode() + exclude = set_default(exclude, []) + + # Order is indeterminate, so sort them + files = sorted(path.rglob('*')) + files = filter(lambda x: x.suffix not in exclude, files) + + hash_obj = hashlib.sha256() + + for file in files: + with open(file, 'w') as fin: + hash_obj.update(cls._hash_io(fin)) + + return hash_obj.hexdigest() @staticmethod def _isurl(url): From 6d999c00023439dfefad7005cd31b66135da7258 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 9 Sep 2024 11:43:18 -0600 Subject: [PATCH 46/71] Improve directory hashing algorithm --- lib/pavilion/builder.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index d3edce413..c00251574 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -962,7 +962,7 @@ def _hash_file(self, path, save=True): return file_hash @classmethod - def _hash_io(cls, contents: TextIO) -> bytes: + def _hash_io(cls, contents: IO) -> bytes: """Hash the given file in IOString format. :param IOString contents: file name (as relative path to build directory) and file contents to hash.""" @@ -992,10 +992,16 @@ def _hash_dir(cls, path: Path, exclude: List[str] = None) -> str: hash_obj = hashlib.sha256() for file in files: - with open(file, 'w') as fin: + if file.is_dir(): + # This has the effect of flattening directories, + # thus ignoring the structure of the directory. + # This might not be what we want. + continue + + with open(file, 'rb') as fin: hash_obj.update(cls._hash_io(fin)) - return hash_obj.hexdigest() + return hash_obj.hexdigest().encode("utf-8") @staticmethod def _isurl(url): From a2047b8223ea80d2cf84da8960b471a8eaa32c05 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 9 Sep 2024 11:45:53 -0600 Subject: [PATCH 47/71] Add missing import --- lib/pavilion/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index c00251574..815cd43ed 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -15,7 +15,7 @@ import time import urllib.parse from pathlib import Path -from typing import Union, Dict, Optional, List, TextIO +from typing import Union, Dict, Optional, List, IO from contextlib import ExitStack import pavilion.config From 4d2bc83cf16163a5ce56f44e38d2af5264b00e72 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 9 Sep 2024 11:58:37 -0600 Subject: [PATCH 48/71] Ignore yaml config files when hashing directories --- lib/pavilion/builder.py | 7 ++++--- test/data/pav_config_dir/hosts/hash_host.yaml | 2 ++ test/data/pav_config_dir/modes/hash_mode.yaml | 2 ++ test/data/pav_config_dir/os/hash_os.yaml | 2 ++ test/data/pav_config_dir/tests/hash_suite_test.yaml | 4 ++++ 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 test/data/pav_config_dir/hosts/hash_host.yaml create mode 100644 test/data/pav_config_dir/modes/hash_mode.yaml create mode 100644 test/data/pav_config_dir/os/hash_os.yaml create mode 100644 test/data/pav_config_dir/tests/hash_suite_test.yaml diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 815cd43ed..329fb12fb 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -29,6 +29,7 @@ from pavilion.test_config.spack import SpackEnvConfig from pavilion.micro import set_default +CONFIG_FNAMES = ("suite.yaml", "hosts.yaml", "modes.yaml", "os.yaml") class TestBuilder: """Manages a test build and their organization. @@ -220,7 +221,7 @@ def _create_build_hash(self) -> str: if src_path.is_file(): hash_obj.update(self._hash_file(src_path)) elif src_path.is_dir(): - hash_obj.update(self._hash_dir(src_path)) + hash_obj.update(self._hash_dir(src_path, exclude=CONFIG_FNAMES)) else: raise TestBuilderError( "Invalid src location {}." @@ -243,7 +244,7 @@ def _create_build_hash(self) -> str: hash_obj.update(self._hash_file(full_path)) elif full_path.is_dir(): self._date_dir(full_path) - hash_obj.update(self._hash_dir(full_path)) + hash_obj.update(self._hash_dir(full_path, exclude=CONFIG_NAMES)) else: raise TestBuilderError( "Extra file '{}' must be a regular file or directory." @@ -987,7 +988,7 @@ def _hash_dir(cls, path: Path, exclude: List[str] = None) -> str: # Order is indeterminate, so sort them files = sorted(path.rglob('*')) - files = filter(lambda x: x.suffix not in exclude, files) + files = filter(lambda x: x.name not in exclude, files) hash_obj = hashlib.sha256() diff --git a/test/data/pav_config_dir/hosts/hash_host.yaml b/test/data/pav_config_dir/hosts/hash_host.yaml new file mode 100644 index 000000000..880468c86 --- /dev/null +++ b/test/data/pav_config_dir/hosts/hash_host.yaml @@ -0,0 +1,2 @@ +variables: + foo: True diff --git a/test/data/pav_config_dir/modes/hash_mode.yaml b/test/data/pav_config_dir/modes/hash_mode.yaml new file mode 100644 index 000000000..880468c86 --- /dev/null +++ b/test/data/pav_config_dir/modes/hash_mode.yaml @@ -0,0 +1,2 @@ +variables: + foo: True diff --git a/test/data/pav_config_dir/os/hash_os.yaml b/test/data/pav_config_dir/os/hash_os.yaml new file mode 100644 index 000000000..880468c86 --- /dev/null +++ b/test/data/pav_config_dir/os/hash_os.yaml @@ -0,0 +1,2 @@ +variables: + foo: True diff --git a/test/data/pav_config_dir/tests/hash_suite_test.yaml b/test/data/pav_config_dir/tests/hash_suite_test.yaml new file mode 100644 index 000000000..97ea92e4b --- /dev/null +++ b/test/data/pav_config_dir/tests/hash_suite_test.yaml @@ -0,0 +1,4 @@ +hash_suite_test: + scheduler: raw + run: + cmds: 'echo Hello!' From 21444975b1a3a066f33ee4820ebcbbcf0f6a3d58 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 9 Sep 2024 12:46:11 -0600 Subject: [PATCH 49/71] Fix style issues --- lib/pavilion/path_utils.py | 4 ++-- lib/pavilion/resolver/resolver.py | 33 +++++++++++++++++++------------ lib/pavilion/series/test_set.py | 3 ++- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/pavilion/path_utils.py b/lib/pavilion/path_utils.py index 8a399d4d7..813bf53cc 100644 --- a/lib/pavilion/path_utils.py +++ b/lib/pavilion/path_utils.py @@ -15,10 +15,10 @@ def append_path(suffix: Pathlike) -> Callable[[Path], Path]: """Constructs a function that appends the given suffix to a path. Intended for use with map.""" - def f(path: Path) -> Path: + def func(path: Path) -> Path: return path / suffix - return f + return func def shortpath(path: Path, parents: int = 1) -> Path: """Return an abbreviated version of a path, where only diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 20484c053..2120a3419 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -57,7 +57,9 @@ class ConfigInfo: - def __init__(self, name: str, type: str, path: Path, label: str = None, from_suite: bool = False): + def __init__(self, name: str, type: str, path: Path, label: str = None, + from_suite: bool = False): + self.name = name self.type = type self.label = label @@ -116,7 +118,8 @@ def __init__(self, pav_cfg, op_sys: str = None, host: str = None, # Raw loaded test suites self._suites: Dict[Dict] = {} - def _get_config_dirname(self, cfg_type: str, use_suites_dir: bool = False) -> str: + @staticmethod + def _get_config_dirname(cfg_type: str, use_suites_dir: bool = False) -> str: """Returns the canonical config directory name for a given config type.""" dirname = cfg_type.lower() @@ -129,7 +132,8 @@ def _get_config_dirname(self, cfg_type: str, use_suites_dir: bool = False) -> st return dirname - def _get_config_fname(self, cfg_type: str) -> str: + @staticmethod + def _get_config_fname(cfg_type: str) -> str: """Given a config type, returns the name of the file in the suites directory corresponding to that type.""" @@ -171,7 +175,8 @@ def _get_test_config_path(self, cfg_name: str, cfg_type: str) -> Tuple[str, Opti return res - def _config_path_from_suite(self, suite_name: str, conf_type: str) -> Tuple[str, Optional[Path]]: + def _config_path_from_suite(self, suite_name: str, + conf_type: str) -> Tuple[str, Optional[Path]]: """Given a suite name, return the path to the config file of the specified type, if one exists. If the file does not exist in any known suites directory, returns None.""" @@ -180,7 +185,7 @@ def _config_path_from_suite(self, suite_name: str, conf_type: str) -> Tuple[str, labels = list(self.config_labels) cfg_fname = self._get_config_fname(conf_type) - + if conf_type == "suite": paths.extend(listmap(append_path(f"{suite_name}.yaml"), self.suites_dirs)) labels *= 2 @@ -205,7 +210,7 @@ def find_config(self, cfg_type: str, cfg_name: str, suite_name: str = None) -> C :return: A tuple of the path to that config, if it exists, and a boolean indicating whether it was found in the suites directory (True) or not (False). """ - + cfg_path = None if suite_name is not None: @@ -465,7 +470,7 @@ def load_iter(self, tests: List[str], modes: List[str] = None, overrides: List[s raw_tests.append(raw_test) ready_to_resolve.extend(permutations) - + # Now resolve all the string syntax and variables those tests at once. new_resolved_tests = [] for ptest in self._resolve_escapes(ready_to_resolve): @@ -606,8 +611,9 @@ def _resolve_escapes(self, ptests: ProtoTest) -> List[ProtoTest]: resolved_tests = remaining return multiplied_tests - - def _safe_load_config(self, cfg: ConfigInfo, loader: yc.YamlConfigLoader) -> TestConfig: + + @staticmethod + def _safe_load_config(cfg: ConfigInfo, loader: yc.YamlConfigLoader) -> TestConfig: """Given a path to a config, load the config, and raise an appropriate error if it can't be loaded""" @@ -639,7 +645,8 @@ def _safe_load_config(self, cfg: ConfigInfo, loader: yc.YamlConfigLoader) -> Tes return raw_cfg - def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, optional: bool = False) -> Optional[TestConfig]: + def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, + optional: bool = False) -> Optional[TestConfig]: """Given a path to a config file and a loader, attempt to load the config, handle errors appropriately.""" @@ -663,7 +670,7 @@ def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, op raw_cfg = self._safe_load_config(cfg_info, loader) - if cfg_info.from_suite and cfg_info.type is not "suite": + if cfg_info.from_suite and cfg_info.type != "suite": raw_cfg = raw_cfg.get(cfg_info.name) if raw_cfg is None: @@ -1012,7 +1019,7 @@ def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): for mode in modes: mode_cfg_path = None - + if suite_name is not None: label, mode_cfg_path = self._config_path_from_suite(suite_name, "mode") if mode_cfg_path is None: @@ -1024,7 +1031,7 @@ def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): loader = self._suite_loader cfg_info = ConfigInfo(mode, "mode", mode_cfg_path, label, from_suite) - + raw_mode_cfg = self._load_raw_config(cfg_info, loader) try: diff --git a/lib/pavilion/series/test_set.py b/lib/pavilion/series/test_set.py index 2c8670a56..17b4a174a 100644 --- a/lib/pavilion/series/test_set.py +++ b/lib/pavilion/series/test_set.py @@ -715,7 +715,8 @@ def mark_completed(self) -> int: TEST_WAIT_PERIOD = 0.5 - def wait(self, wait_for_all=False, wait_period: int = TEST_WAIT_PERIOD, timeout: int = None) -> int: + def wait(self, wait_for_all=False, wait_period: int = TEST_WAIT_PERIOD, + timeout: int = None) -> int: """Wait for tests to complete. Returns the number of jobs that completed for this call to wait. From 4670b7d8f02d1e92d03b049cffb28f0a24d96bc1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 9 Sep 2024 12:58:05 -0600 Subject: [PATCH 50/71] Remove stray pdb statement --- lib/pavilion/resolver/resolver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 2120a3419..aa61466aa 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -1015,8 +1015,6 @@ def apply_modes(self, test_cfg, modes: List[str], suite_name: str = None): :param modes: A list of mode names. """ - # import pdb; pdb.set_trace() - for mode in modes: mode_cfg_path = None From 41adf46c47d93012a2f7cc61d4de0950cb70cac7 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 9 Sep 2024 13:25:52 -0600 Subject: [PATCH 51/71] Pull parsetime bugfix over from other branch --- lib/pavilion/filters/__init__.py | 2 +- lib/pavilion/filters/parse_time.py | 41 ++++++++++++++++++++---------- test/tests/filter_tests.py | 8 +++--- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/lib/pavilion/filters/__init__.py b/lib/pavilion/filters/__init__.py index 04c412ce2..a6e14399a 100644 --- a/lib/pavilion/filters/__init__.py +++ b/lib/pavilion/filters/__init__.py @@ -5,4 +5,4 @@ validate_str_list, validate_datetime) from .errors import FilterParseError from .common import identity, const -from .parse_time import parse_duration +from .parse_time import parse_duration, safe_update diff --git a/lib/pavilion/filters/parse_time.py b/lib/pavilion/filters/parse_time.py index 37900c6c9..4669ddafd 100644 --- a/lib/pavilion/filters/parse_time.py +++ b/lib/pavilion/filters/parse_time.py @@ -62,27 +62,19 @@ def parse_duration(rval: str, now: datetime) -> datetime: if unit not in UNITS: raise ValueError(f"Invalid unit {unit} for duration") - if unit not in ("years", "months"): - return now - timedelta(**{unit: mag}) - - new_day = now.day - if unit == 'years': - new_year = now.year - mag - new_month = now.month + return safe_update(now, year=now.year - mag) if unit == 'months': dyear, dmonth = divmod(mag, MONTHS_PER_YEAR) - new_year = now.year - dyear - new_month = (now.month - dmonth) % MONTHS_PER_YEAR - max_day = monthrange(new_year, new_month)[1] - - if new_day > max_day: - new_day = max_day + new_day = now.day + new_month = now.month - dmonth + new_year = now.year - dyear - return now.replace(year=new_year, month=new_month, day=new_day) + return safe_update(now, year=new_year, month=new_month, day=new_day) + return now - timedelta(**{unit: mag}) def parse_iso_date(rval: str) -> date: @@ -150,3 +142,24 @@ def normalize(unit: str) -> str: return unit + "s" return unit + + +def safe_update(date: datetime, + year: int = None, + month: int = None, day: int = None) -> datetime: + if year is None: + year = date.year + if month is None: + month = date.month + if day is None: + day = date.day + + max_day = monthrange(year, month)[1] + + # If the day is too large, adjust it to be the last day + # of the new month. This might not be the best solution, + # but it's a reasonable way to handle it. + if day > max_day: + day = max_day + + return date.replace(year=year, month=month, day=day) diff --git a/test/tests/filter_tests.py b/test/tests/filter_tests.py index fc4f23d1b..845e2e44e 100644 --- a/test/tests/filter_tests.py +++ b/test/tests/filter_tests.py @@ -17,7 +17,7 @@ from pavilion.status_file import TestStatusFile, SeriesStatusFile from pavilion.filters import (FilterParseError, validate_int, validate_glob, validate_glob_list, validate_str_list, validate_datetime, - parse_query, parse_duration) + parse_query, parse_duration, safe_update) class FiltersTest(PavTestCase): @@ -500,15 +500,15 @@ def test_parse_duration(self): self.assertTrue(parse_duration('-0days', now) == now) parsed = parse_duration('13 months', now) - expected = date(year=now.year - 1, month=now.month - 1, day=now.day) + expected = safe_update(now, year=now.year - 1, month=now.month - 1).date() self.assertTrue(parsed.date() == expected) parsed = parse_duration('12 months', now) - expected = date(year=now.year - 1, month=now.month, day=now.day) + expected = safe_update(now, year=now.year - 1).date() self.assertTrue(parsed.date() == expected) parsed = parse_duration('1 months', now) - expected = date(year=now.year, month=now.month - 1, day=now.day) + expected = safe_update(now, month=now.month - 1).date() self.assertTrue(parsed.date() == expected) def test_filter_keywords_implemented(self): From ec879521bdd3b3070154afe22f05372c1845a2df Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 10 Sep 2024 13:16:06 -0600 Subject: [PATCH 52/71] Pass hashing unit test --- lib/pavilion/resolver/resolver.py | 2 +- test/data/pav_config_dir/suites/hash_suite_test_a/fox.txt | 1 + test/data/pav_config_dir/suites/hash_suite_test_b/fox.txt | 1 + test/tests/suites_tests.py | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_a/fox.txt create mode 100644 test/data/pav_config_dir/suites/hash_suite_test_b/fox.txt diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index aa61466aa..e2b22a01e 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -673,7 +673,7 @@ def _load_raw_config(self, cfg_info: ConfigInfo, loader: yc.YamlConfigLoader, if cfg_info.from_suite and cfg_info.type != "suite": raw_cfg = raw_cfg.get(cfg_info.name) - if raw_cfg is None: + if raw_cfg is None and not optional: raise TestConfigError( f"Could not find {cfg_info.type} config with name {cfg_info.name}" f" in file {cfg_info.path}.") diff --git a/test/data/pav_config_dir/suites/hash_suite_test_a/fox.txt b/test/data/pav_config_dir/suites/hash_suite_test_a/fox.txt new file mode 100644 index 000000000..458d9cb93 --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_a/fox.txt @@ -0,0 +1 @@ +The quick brown fox jumped over the lazy dog diff --git a/test/data/pav_config_dir/suites/hash_suite_test_b/fox.txt b/test/data/pav_config_dir/suites/hash_suite_test_b/fox.txt new file mode 100644 index 000000000..458d9cb93 --- /dev/null +++ b/test/data/pav_config_dir/suites/hash_suite_test_b/fox.txt @@ -0,0 +1 @@ +The quick brown fox jumped over the lazy dog diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index 734b1b5d0..293febdf6 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -133,7 +133,7 @@ def test_suites_build_hash(self): self.assertEqual(ret, 0) last_test = run_cmd.last_tests[0] - hash_a = last_test.builder().build_hash + hash_a = last_test.builder.build_hash arg_parser = arguments.get_parser() args = arg_parser.parse_args([ @@ -147,6 +147,6 @@ def test_suites_build_hash(self): self.assertEqual(ret, 0) last_test = run_cmd.last_tests[0] - hash_b = last_test.builder().build_hash + hash_b = last_test.builder.build_hash self.assertEqual(hash_a, hash_b) From 5b279642ed7abcb547e32ee866ab7794bdb940a1 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 10 Sep 2024 13:18:24 -0600 Subject: [PATCH 53/71] Remove unused test data --- test/data/pav_config_dir/hosts/hash_host.yaml | 2 -- test/data/pav_config_dir/modes/hash_mode.yaml | 2 -- test/data/pav_config_dir/os/hash_os.yaml | 2 -- 3 files changed, 6 deletions(-) delete mode 100644 test/data/pav_config_dir/hosts/hash_host.yaml delete mode 100644 test/data/pav_config_dir/modes/hash_mode.yaml delete mode 100644 test/data/pav_config_dir/os/hash_os.yaml diff --git a/test/data/pav_config_dir/hosts/hash_host.yaml b/test/data/pav_config_dir/hosts/hash_host.yaml deleted file mode 100644 index 880468c86..000000000 --- a/test/data/pav_config_dir/hosts/hash_host.yaml +++ /dev/null @@ -1,2 +0,0 @@ -variables: - foo: True diff --git a/test/data/pav_config_dir/modes/hash_mode.yaml b/test/data/pav_config_dir/modes/hash_mode.yaml deleted file mode 100644 index 880468c86..000000000 --- a/test/data/pav_config_dir/modes/hash_mode.yaml +++ /dev/null @@ -1,2 +0,0 @@ -variables: - foo: True diff --git a/test/data/pav_config_dir/os/hash_os.yaml b/test/data/pav_config_dir/os/hash_os.yaml deleted file mode 100644 index 880468c86..000000000 --- a/test/data/pav_config_dir/os/hash_os.yaml +++ /dev/null @@ -1,2 +0,0 @@ -variables: - foo: True From 1962e7c74463b017dbeb8c11873860aec1925e12 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 10 Sep 2024 15:44:33 -0600 Subject: [PATCH 54/71] Refactor find_file --- lib/pavilion/config.py | 26 ++++++++------------------ lib/pavilion/micro.py | 2 +- lib/pavilion/path_utils.py | 25 +++++++++++++++++++++---- lib/pavilion/resolver/resolver.py | 10 +++++----- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index a5246bfe2..3c23b8b0b 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -17,7 +17,8 @@ import yaml_config as yc from pavilion import output from pavilion import errors -from pavilion.micro import first, flatten +from pavilion.micro import first, flatten, remove_none +from pavilion.path_utils import Pathlike, append_to_path, append_suffix, exists, path_product # Figure out what directories we'll search for the base configuration. PAV_CONFIG_SEARCH_DIRS = [Path('./').resolve()] @@ -184,8 +185,6 @@ def as_dict(self) -> dict: return adict -Pathlike = Union[str, Path] - class PavConfig(PavConfigDict): """Define types and attributes for Pavilion config options.""" @@ -221,14 +220,14 @@ def __init__(self, set_attrs=None): super().__init__(set_attrs) @property - def cfg_paths(self) -> Iterator[Path]: + def config_paths(self) -> Iterator[Path]: """Return an iterator of paths to all config directories""" return (Path(cfg['path']) for cfg in self.configs.values()) @property def suites_dirs(self) -> Iterator[Path]: """Return an iterator of paths to all suites directories""" - return (path / 'suites' for path in self.cfg_paths) + return (path / 'suites' for path in self.config_paths) @property def suite_paths(self) -> Iterator[Path]: @@ -265,21 +264,12 @@ def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = else: return None - sub_dirs = list(sub_dirs) - - path_comps = product(self.cfg_paths, sub_dirs, [file]) - - def make_path(path: Path, subdir: Pathlike, file: Path) -> Path: - if subdir is None: - return path / file - - return path / subdir / file - - paths = starmap(make_path, path_comps) + sub_dirs = remove_none(list(sub_dirs)) + paths = path_product(self.config_dirs, sub_dirs) + files = map(append_to_path(file), paths) # Return the first path to the file that exists (or None) - return first(lambda x: x.exists(), paths) - + return first(exists, files) class ExPathElem(yc.PathElem): """Expand environment variables in the path.""" diff --git a/lib/pavilion/micro.py b/lib/pavilion/micro.py index 6ca06432a..a4b22d744 100644 --- a/lib/pavilion/micro.py +++ b/lib/pavilion/micro.py @@ -37,7 +37,7 @@ def replace(lst: Iterable[T], old: T, new: T) -> Iterator[T]: def remove_none(lst: Iterable[T]) -> Iterator[T]: """Remove all instances of None from the iterable.""" - return filter(lambda x: x is not None, lst) + return remove_all(lst, None) def first(pred: Callable[[T], bool], lst: Iterable[T]) -> Optional[T]: """Return the first item of the list that satisfies the given diff --git a/lib/pavilion/path_utils.py b/lib/pavilion/path_utils.py index 813bf53cc..0ec949bc0 100644 --- a/lib/pavilion/path_utils.py +++ b/lib/pavilion/path_utils.py @@ -1,7 +1,9 @@ """Simple utilities for dealing with paths.""" from pathlib import Path -from typing import Union, Callable +from operator import truediv +from itertools import starmap, product +from typing import Union, Callable, Iterable, Iterator Pathlike = Union[Path, str] @@ -11,9 +13,18 @@ def exists(path: Path) -> bool: return path.exists() -def append_path(suffix: Pathlike) -> Callable[[Path], Path]: - """Constructs a function that appends the given suffix - to a path. Intended for use with map.""" +def append_suffix(path: Path) -> Callable[[Pathlike], Path]: + """Constructs a function that takes a suffix and appends + it to a constant path. Intended for use with map.""" + + def func(suffix: Pathlike) -> Path: + return path / suffix + + return func + +def append_to_path(suffix: Pathlike) -> Callable[[Path], Path]: + """Constructs a function that takes a path and appends a + constant suffix to it. Intended for use with map.""" def func(path: Path) -> Path: return path / suffix @@ -25,3 +36,9 @@ def shortpath(path: Path, parents: int = 1) -> Path: the specified number of parents are included.""" return Path(*path.parts[-(1 + parents):]) + +def path_product(roots: Iterable[Path], stems: Iterable[Pathlike]) -> Iterator[Path]: + """Given a list of root paths and a list of stem paths, returns an iterator + over all paths formed by the Cartesian products of those lists.""" + + return starmap(truediv, product(roots, stems)) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index e2b22a01e..a8ce2ef88 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -38,7 +38,7 @@ from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary from pavilion.micro import first, listmap -from pavilion.path_utils import append_path +from pavilion.path_utils import append_to_path from yaml_config import RequiredError, YamlConfigLoader from .proto_test import RawProtoTest, ProtoTest @@ -152,7 +152,7 @@ def config_paths(self) -> Iterator[Path]: @property def suites_dirs(self) -> Iterator[Path]: """Return an iterator over all suites directories.""" - return map(append_path("suites"), self.config_paths) + return map(append_to_path("suites"), self.config_paths) @property def config_labels(self) -> Iterator[str]: @@ -165,7 +165,7 @@ def _get_test_config_path(self, cfg_name: str, cfg_type: str) -> Tuple[str, Opti return None.""" cfg_dir = self._get_config_dirname(cfg_type) - paths = map(append_path(f"{cfg_dir}/{cfg_name}.yaml"), self.config_paths) + paths = map(append_to_path(f"{cfg_dir}/{cfg_name}.yaml"), self.config_paths) pairs = zip(self.config_labels, paths) res = first(lambda x: x[1].exists(), pairs) @@ -187,10 +187,10 @@ def _config_path_from_suite(self, suite_name: str, cfg_fname = self._get_config_fname(conf_type) if conf_type == "suite": - paths.extend(listmap(append_path(f"{suite_name}.yaml"), self.suites_dirs)) + paths.extend(listmap(append_to_path(f"{suite_name}.yaml"), self.suites_dirs)) labels *= 2 - paths.extend(listmap(append_path(f"{suite_name}/{cfg_fname}"), self.suites_dirs)) + paths.extend(listmap(append_to_path(f"{suite_name}/{cfg_fname}"), self.suites_dirs)) pairs = zip(labels, paths) From a83ac9ebc8716da402bfa7c548532753ab577179 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 10 Sep 2024 15:44:53 -0600 Subject: [PATCH 55/71] Increase timeout for series cmd tests --- test/tests/series_cmd_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/series_cmd_tests.py b/test/tests/series_cmd_tests.py index cbde2ee10..ef2df6f04 100644 --- a/test/tests/series_cmd_tests.py +++ b/test/tests/series_cmd_tests.py @@ -59,7 +59,7 @@ def test_run_series_modes(self): self.assertEqual(run_result, 0) series_obj = run_cmd.last_run_series - series_obj.wait(5) + series_obj.wait(timeout=10) self.assertEqual(series_obj.complete, True) self.assertEqual(series_obj.info().passed, 1) @@ -86,7 +86,7 @@ def test_run_series_overrides(self): self.assertEqual(run_result, 0) series_obj = run_cmd.last_run_series - series_obj.wait(5) + series_obj.wait(timeout=10) self.assertEqual(series_obj.complete, True) self.assertEqual(series_obj.info().passed, 1) From f8e2e1234991c3575983192f23c057cfb22d51a3 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Tue, 10 Sep 2024 16:39:48 -0600 Subject: [PATCH 56/71] Fix bug in find_file --- lib/pavilion/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 3c23b8b0b..a6edb9651 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -265,7 +265,7 @@ def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = return None sub_dirs = remove_none(list(sub_dirs)) - paths = path_product(self.config_dirs, sub_dirs) + paths = path_product(self.config_paths, sub_dirs) files = map(append_to_path(file), paths) # Return the first path to the file that exists (or None) From 90693a96d4bd9860533e785f0ca06cabae22d420 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 11 Sep 2024 10:39:43 -0600 Subject: [PATCH 57/71] Fix suite path (hopefully) --- lib/pavilion/builder.py | 21 +++++++++++---------- lib/pavilion/resolver/resolver.py | 6 +++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 329fb12fb..63219f081 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -27,7 +27,7 @@ from pavilion.status_file import TestStatusFile, STATES from pavilion.test_config import parse_timeout from pavilion.test_config.spack import SpackEnvConfig -from pavilion.micro import set_default +from pavilion.micro import set_default, remove_none CONFIG_FNAMES = ("suite.yaml", "hosts.yaml", "modes.yaml", "os.yaml") @@ -137,13 +137,14 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config .format(dest), err) @property - def suite_path(self) -> Path: + def suite_path(self) -> Optional[Path]: spath = self._config.get('suite_path') + + if spath == "": + return None - if spath in (None, ''): - return Path('..').resolve() - - return Path(spath) + if spath is not None: + return Path(spath) def exists(self): """Return True if the given build exists.""" @@ -234,7 +235,7 @@ def _create_build_hash(self) -> str: # Hash extra files. for extra_file in self._config.get('extra_files', []): extra_file = Path(extra_file) - full_path = self._pav_cfg.find_file(extra_file, [self.suite_path, Path('test_src')]) + full_path = self._pav_cfg.find_file(extra_file, remove_none([self.suite_path, Path('test_src')])) if full_path is None: raise TestBuilderError( @@ -328,7 +329,7 @@ def _update_src(self) -> Optional[Path]: "The source path must be a valid unix path, either relative " "or absolute, got '{}'".format(src_path), err) - found_src_path = self._pav_cfg.find_file(src_path, [self.suite_path, Path('test_src')]) + found_src_path = self._pav_cfg.find_file(src_path, remove_none([self.suite_path, Path('test_src')])) src_url = self._config.get('source_url') src_download = self._config.get('source_download') @@ -681,7 +682,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): if raw_src_path is None: src_path = None else: - src_path = self._pav_cfg.find_file(raw_src_path, [self.suite_path, Path('test_src')]) + src_path = self._pav_cfg.find_file(raw_src_path, remove_none([self.suite_path, Path('test_src')])) if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) @@ -788,7 +789,7 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): # Now we just need to copy over all the extra files. for extra in self._config.get('extra_files', []): extra = Path(extra) - path = self._pav_cfg.find_file(extra, [self.suite_path, Path('test_src')]) + path = self._pav_cfg.find_file(extra, remove_none([self.suite_path, Path('test_src')])) final_dest = dest / path.name try: if path.is_dir(): diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index a8ce2ef88..873986e15 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -867,10 +867,14 @@ def _load_suite_tests(self, suite_name: str): working_dir = self.pav_cfg['configs'][cfg_info.label]['working_dir'] test_cfg['working_dir'] = working_dir.as_posix() test_cfg['suite'] = suite_name - test_cfg['suite_path'] = cfg_info.path.as_posix() test_cfg['host'] = self._host test_cfg['os'] = self._os + if cfg_info.from_suite: + test_cfg['suite_path'] = cfg_info.path.as_posix().parent + else: + test_cfg['suite_path'] = cfg_info.path.as_posix() + self._suites[suite_name] = suite_tests return suite_tests From c25ed77aaf6ede975e83f2e4c2869028d162079e Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 11 Sep 2024 11:37:15 -0600 Subject: [PATCH 58/71] Fix source path issue (finger crossed) --- lib/pavilion/builder.py | 23 ++++++++++++----------- lib/pavilion/resolver/resolver.py | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 63219f081..8aeb0af1f 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -137,14 +137,11 @@ def __init__(self, pav_cfg: pavilion.config.PavConfig, working_dir: Path, config .format(dest), err) @property - def suite_path(self) -> Optional[Path]: - spath = self._config.get('suite_path') - - if spath == "": - return None + def suite_subdir(self) -> Optional[Path]: + sname = self._config.get('suite_name') - if spath is not None: - return Path(spath) + if sname is not None: + return Path(f"suites/{sname}/") def exists(self): """Return True if the given build exists.""" @@ -235,7 +232,8 @@ def _create_build_hash(self) -> str: # Hash extra files. for extra_file in self._config.get('extra_files', []): extra_file = Path(extra_file) - full_path = self._pav_cfg.find_file(extra_file, remove_none([self.suite_path, Path('test_src')])) + sub_dirs = [self.suite_subdir, Path('test_src')] + full_path = self._pav_cfg.find_file(extra_file, sub_dirs) if full_path is None: raise TestBuilderError( @@ -329,7 +327,8 @@ def _update_src(self) -> Optional[Path]: "The source path must be a valid unix path, either relative " "or absolute, got '{}'".format(src_path), err) - found_src_path = self._pav_cfg.find_file(src_path, remove_none([self.suite_path, Path('test_src')])) + sub_dirs = [self.suite_subdir, Path('test_src')] + found_src_path = self._pav_cfg.find_file(src_path, sub_dirs) src_url = self._config.get('source_url') src_download = self._config.get('source_download') @@ -682,7 +681,8 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): if raw_src_path is None: src_path = None else: - src_path = self._pav_cfg.find_file(raw_src_path, remove_none([self.suite_path, Path('test_src')])) + sub_dirs = [self.suite_subdir, Path('test_src')] + src_path = self._pav_cfg.find_file(raw_src_path, sub_dirs) if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) @@ -789,7 +789,8 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): # Now we just need to copy over all the extra files. for extra in self._config.get('extra_files', []): extra = Path(extra) - path = self._pav_cfg.find_file(extra, remove_none([self.suite_path, Path('test_src')])) + sub_dirs = [self.suite_subdir, Path('test_src')] + path = self._pav_cfg.find_file(extra, sub_dirs) final_dest = dest / path.name try: if path.is_dir(): diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 873986e15..f982bc283 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -871,7 +871,7 @@ def _load_suite_tests(self, suite_name: str): test_cfg['os'] = self._os if cfg_info.from_suite: - test_cfg['suite_path'] = cfg_info.path.as_posix().parent + test_cfg['suite_path'] = cfg_info.path.parent.as_posix() else: test_cfg['suite_path'] = cfg_info.path.as_posix() From 570ec4eb3301e16e4b13a4857f0e5506d0e92573 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 11 Sep 2024 16:19:25 -0600 Subject: [PATCH 59/71] Fix download destination --- lib/pavilion/test_run/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index b6edfaaaf..86de1219a 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -343,7 +343,7 @@ def _make_builder(self) -> builder.TestBuilder: spack_config = (self.config.get('spack_config', {}) if self.spack_enabled() else None) if self.suite_path != Path('..') and self.suite_path is not None: - download_dest = self.suite_path.parents[1] / 'suites' + download_dest = self.suite_path # Check the deprecated directory if not download_dest.exists(): From 8edc49013567918defd617ca0eb77aa1e765a92e Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 23 Sep 2024 08:24:37 -0600 Subject: [PATCH 60/71] Add docstring for safe_update --- lib/pavilion/filters/parse_time.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/filters/parse_time.py b/lib/pavilion/filters/parse_time.py index 4669ddafd..d8a775d1d 100644 --- a/lib/pavilion/filters/parse_time.py +++ b/lib/pavilion/filters/parse_time.py @@ -144,9 +144,13 @@ def normalize(unit: str) -> str: return unit -def safe_update(date: datetime, - year: int = None, - month: int = None, day: int = None) -> datetime: +def safe_update(date: datetime, year: int = None, month: int = None, day: int = None) -> datetime: + """Update the datetime object with the given year, month, and day, ensuring that the final + day is valid for the month and year, returning the modified datetime object. For instance, we + want to guard against February 30th. This function is necessary because datetime.timedelta + can't handle variable-size units of time (i.e. months and years). + """ + if year is None: year = date.year if month is None: From ada1aeea9527a09738b1905440e351b953cf3940 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 25 Sep 2024 08:52:14 -0600 Subject: [PATCH 61/71] Miscellaneous changes --- lib/pavilion/builder.py | 17 ++- lib/pavilion/commands/config.py | 3 +- lib/pavilion/config.py | 35 +++++- lib/pavilion/resolver/resolver.py | 118 +++++++++--------- .../suites/circular_symlinks/bar | 1 + .../suites/circular_symlinks/foo | 1 + .../suites/circular_symlinks/suite.yaml | 7 ++ .../suites/suite_with_source/hello.c | 7 ++ .../suites/suite_with_source/suite.yaml | 6 + test/tests/builder_tests.py | 25 +++- test/tests/suites_tests.py | 14 +++ 11 files changed, 163 insertions(+), 71 deletions(-) create mode 120000 test/data/pav_config_dir/suites/circular_symlinks/bar create mode 120000 test/data/pav_config_dir/suites/circular_symlinks/foo create mode 100644 test/data/pav_config_dir/suites/circular_symlinks/suite.yaml create mode 100644 test/data/pav_config_dir/suites/suite_with_source/hello.c create mode 100644 test/data/pav_config_dir/suites/suite_with_source/suite.yaml diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 8aeb0af1f..b5d00f72b 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -204,6 +204,8 @@ def _create_build_hash(self) -> str: # - All of the build's 'extra_files' # - All files needed to be created at build time 'create_files' + self.status.set(STATES.INFO, "Creating build hash.") + hash_obj = hashlib.sha256() # Update the hash with the contents of the build script. @@ -316,6 +318,7 @@ def _update_src(self) -> Optional[Path]: """ src_path = self._config.get('source_path') + if src_path is None: # There is no source to do anything with. return None @@ -980,16 +983,20 @@ def _hash_io(cls, contents: IO) -> bytes: @classmethod def _hash_dir(cls, path: Path, exclude: List[str] = None) -> str: - """Recursively hash the files in the given directory, excluding - files with the given suffixes. + """Recursively hash the files in the given directory, optionally excluding files with + the given names. - We might need to modify this to handle circular symlinks. + Returns the hexadecimal hash digest of all files in the directory, as a UTF-8 string. """ exclude = set_default(exclude, []) - # Order is indeterminate, so sort them - files = sorted(path.rglob('*')) + try: + # Order is indeterminate, so sort the files + files = sorted(path.rglob('*')) + except OSError: + raise TestBuilderError(f"Unable to hash directory: {path}. Possible circular symlink.") + files = filter(lambda x: x.name not in exclude, files) hash_obj = hashlib.sha256() diff --git a/lib/pavilion/commands/config.py b/lib/pavilion/commands/config.py index 3d0866b83..1b40b2577 100644 --- a/lib/pavilion/commands/config.py +++ b/lib/pavilion/commands/config.py @@ -244,8 +244,7 @@ def create_config_dir(self, pav_cfg: config.PavConfig, path: Path, raise ConfigCmdError("Error writing config file at '{}'" .format(config_file_path), err) - for subdir in ('hosts', 'modes', 'tests', 'os', 'test_src', - 'plugins', 'collections', 'suites'): + for subdir in ('hosts', 'modes', 'os', 'plugins', 'collections', 'suites'): subdir = path/subdir try: subdir.mkdir() diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index a6edb9651..89ce93a2f 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -12,7 +12,7 @@ from collections import OrderedDict from pathlib import Path from itertools import product, starmap -from typing import List, Union, Dict, NewType, Iterator +from typing import List, Union, Dict, NewType, Iterator, Tuple import yaml_config as yc from pavilion import output @@ -224,6 +224,11 @@ def config_paths(self) -> Iterator[Path]: """Return an iterator of paths to all config directories""" return (Path(cfg['path']) for cfg in self.configs.values()) + @property + def tests_dirs(self) -> Iterator[Path]: + """Return an iterator of paths to all test directories.""" + return (path / 'tests' for path in self.config_paths) + @property def suites_dirs(self) -> Iterator[Path]: """Return an iterator of paths to all suites directories""" @@ -245,6 +250,34 @@ def is_suite(file: Path) -> bool: return map(lambda x: x.stem, self.suite_paths) + @property + def suite_info(self) -> List[Tuple[str, str, Path]]: + """Get the label, name, and path for every suite the config + knows about.""" + + suites = [] + + for label, cfg in self.configs.values(): + tests_dir = Path(cfg['path']) / 'tests' + suites_dir = Path(cfg['path']) / 'suites' + + if tests_dir.exists(): + tests = [file for file in test_dir.iterdir() if (file.suffix.lower() == ".yaml")] + names = [test.name for test in tests] + labels = [label] * len(tests) + + suites.extend(zip(labels, names, tests)) + + if suites_dir.exists(): + suites = [sdir / "suite.yaml" in suites_dir.iterdir()] + suites = list(filter(exists, suites)) + names = [suite.parent.name for suite in suites] + labels = [label] * len(suites) + + suites.extend(zip(labels, names, suites)) + + return suites + def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ -> Union[Path, None]: """Look for the given file and return a full path to it. Relative paths diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index f982bc283..1f7476f99 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -18,7 +18,7 @@ import sys from collections import defaultdict from pathlib import Path -from typing import List, IO, Dict, Tuple, NewType, Union, Any, Iterator, TextIO, Optional +from typing import List, IO, Dict, Tuple, NewType, Union, Any, Iterator, TextIO, Optional, Iterable import similarity import yc_yaml @@ -38,7 +38,7 @@ from pavilion.test_config.file_format import TestConfigLoader, TestSuiteLoader from pavilion.utils import union_dictionary from pavilion.micro import first, listmap -from pavilion.path_utils import append_to_path +from pavilion.path_utils import append_to_path, exists from yaml_config import RequiredError, YamlConfigLoader from .proto_test import RawProtoTest, ProtoTest @@ -147,12 +147,12 @@ def _get_config_fname(cfg_type: str) -> str: @property def config_paths(self) -> Iterator[Path]: """Return an iterator over all config paths.""" - return map(lambda x: x["path"], self.pav_cfg.configs.values()) + return self.pav_cfg.config_paths @property def suites_dirs(self) -> Iterator[Path]: """Return an iterator over all suites directories.""" - return map(append_to_path("suites"), self.config_paths) + return self.pav_cfg.suites_dirs @property def config_labels(self) -> Iterator[str]: @@ -244,6 +244,16 @@ def find_similar_configs(self, conf_type: str, conf_name: str) -> List[str]: return similarity.find_matches(conf_name, names) + + @staticmethod + def _make_suites_dict(self, suite_dirs: Iterable[Path]) -> Dict[str, Path]: + """Construct a dictionary mapping suite names to their respective suite.yaml + files.""" + + pairs = ((sdir.stem, sdir / "suite.yaml") for sdir in suite_dirs) + + return dict(pairs) + def find_all_tests(self): """Find all the tests within known config directories. @@ -267,67 +277,55 @@ def find_all_tests(self): suites = {} - for label, config in self.pav_cfg.configs.items(): - path = config['path']/'tests' - - if not (path.exists() and path.is_dir()): - continue - - for file in os.listdir(path.as_posix()): + # TODO: Figure out how to do this and get the labels + for label, name, path in self.pav_cfg.suite_info: + if name not in suites: + suites[suite_name] = { + 'path': path, + 'label': label, + 'err': '', + 'tests': {}, + 'supersedes': [], + } + else: + suites[name]['supersedes'].append(path) - file = path/file - if file.suffix != '.yaml' or not file.is_file(): + # It's ok if the tests aren't completely validated. They + # may have been written to require a real host/mode file. + with path.open('r') as suite_file: + try: + suite_cfg = self._suite_loader.load(suite_file, partial=True) + except (TypeError, + KeyError, + ValueError, + yc_yaml.YAMLError) as err: + suites[name]['err'] = err continue - suite_name = file.stem - - if suite_name not in suites: - suites[suite_name] = { - 'path': file, - 'label': label, - 'err': '', - 'tests': {}, - 'supersedes': [], - } - else: - suites[suite_name]['supersedes'].append(file) - - # It's ok if the tests aren't completely validated. They - # may have been written to require a real host/mode file. - with file.open('r') as suite_file: - try: - suite_cfg = self._suite_loader.load(suite_file, partial=True) - except (TypeError, - KeyError, - ValueError, - yc_yaml.YAMLError) as err: - suites[suite_name]['err'] = err - continue + base = self._loader.load_empty() - base = self._loader.load_empty() - - try: - suite_cfgs = self.resolve_inheritance( - suite_cfg=suite_cfg, - suite_path=file) - except Exception as err: # pylint: disable=W0703 - suites[suite_name]['err'] = err - continue + try: + suite_cfgs = self.resolve_inheritance( + suite_cfg=suite_cfg, + suite_path=path) + except Exception as err: # pylint: disable=W0703 + suites[name]['err'] = err + continue - def default(val, dval): - """Return the dval if val is None.""" + def default(val, dval): + """Return the dval if val is None.""" - return dval if val is None else val + return dval if val is None else val - for test_name, conf in suite_cfgs.items(): - suites[suite_name]['tests'][test_name] = { - 'conf': conf, - 'maintainer': default( - conf['maintainer']['name'], ''), - 'email': default(conf['maintainer']['email'], ''), - 'summary': default(conf.get('summary', ''), ''), - 'doc': default(conf.get('doc', ''), ''), - } + for test_name, conf in suite_cfgs.items(): + suites[name]['tests'][test_name] = { + 'conf': conf, + 'maintainer': default( + conf['maintainer']['name'], ''), + 'email': default(conf['maintainer']['email'], ''), + 'summary': default(conf.get('summary', ''), ''), + 'doc': default(conf.get('doc', ''), ''), + } return suites def find_all_configs(self, conf_type): @@ -735,9 +733,9 @@ def _load_raw_configs(self, request: TestRequest, modes: List[str], # Apply modes. try: - test_cfg = self.apply_modes(test_cfg, modes, request.suite) - test_cfg = self.apply_host(test_cfg, self._host, request.suite) test_cfg = self.apply_os(test_cfg, self._os, request.suite) + test_cfg = self.apply_host(test_cfg, self._host, request.suite) + test_cfg = self.apply_modes(test_cfg, modes, request.suite) except TestConfigError as err: err.request = request self.errors.append(err) diff --git a/test/data/pav_config_dir/suites/circular_symlinks/bar b/test/data/pav_config_dir/suites/circular_symlinks/bar new file mode 120000 index 000000000..191028156 --- /dev/null +++ b/test/data/pav_config_dir/suites/circular_symlinks/bar @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/test/data/pav_config_dir/suites/circular_symlinks/foo b/test/data/pav_config_dir/suites/circular_symlinks/foo new file mode 120000 index 000000000..ba0e162e1 --- /dev/null +++ b/test/data/pav_config_dir/suites/circular_symlinks/foo @@ -0,0 +1 @@ +bar \ No newline at end of file diff --git a/test/data/pav_config_dir/suites/circular_symlinks/suite.yaml b/test/data/pav_config_dir/suites/circular_symlinks/suite.yaml new file mode 100644 index 000000000..dbcc128b7 --- /dev/null +++ b/test/data/pav_config_dir/suites/circular_symlinks/suite.yaml @@ -0,0 +1,7 @@ +circular_symlink: + summary: Build for which the directory contains circular symlinks. + scheduler: raw + build: + source_path: "." + run: + cmds: "echo hello" diff --git a/test/data/pav_config_dir/suites/suite_with_source/hello.c b/test/data/pav_config_dir/suites/suite_with_source/hello.c new file mode 100644 index 000000000..87a87b0b3 --- /dev/null +++ b/test/data/pav_config_dir/suites/suite_with_source/hello.c @@ -0,0 +1,7 @@ +#include + +int main() { + printf("Hello, World!\n"); + + return 0; +} diff --git a/test/data/pav_config_dir/suites/suite_with_source/suite.yaml b/test/data/pav_config_dir/suites/suite_with_source/suite.yaml new file mode 100644 index 000000000..495610976 --- /dev/null +++ b/test/data/pav_config_dir/suites/suite_with_source/suite.yaml @@ -0,0 +1,6 @@ +suite_with_source: + scheduler: raw + build: + cmds: "gcc -o hello hello.c" + run: + cmds: "{{sched.test_cmd}} ./hello" diff --git a/test/tests/builder_tests.py b/test/tests/builder_tests.py index eb975d582..f03951049 100644 --- a/test/tests/builder_tests.py +++ b/test/tests/builder_tests.py @@ -9,15 +9,18 @@ import uuid from pathlib import Path -from pavilion import builder -from pavilion import wget +from pavilion import builder, arguments, wget, plugins, commands from pavilion.build_tracker import MultiBuildTracker, DummyTracker -from pavilion.errors import TestRunError +from pavilion.errors import TestRunError, TestBuilderError from pavilion.status_file import STATES from pavilion.unittest import PavTestCase class BuilderTests(PavTestCase): + def setUp(self): + plugins.initialize_plugins(self.pav_cfg) + build_cmd = commands.get_command('run') + build_cmd.silence() def test_build_locking(self): """Make sure multiple builds of the same hash lock each other out.""" @@ -451,3 +454,19 @@ def test_create_files(self): with self.assertRaises(TestRunError): self._quick_test(cfg) + + def test_hash_circular_symlinks(self): + """Test that the builder errors out when trying to hash a directory + containing circular symlinks.""" + + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + '-H', 'this', + 'circular_symlinks' + ]) + + build_cmd = commands.get_command(args.command_name) + + with self.assertRaises(TestBuilderError): + build_cmd.run(self.pav_cfg, args) diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index 293febdf6..ef2588c96 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -150,3 +150,17 @@ def test_suites_build_hash(self): hash_b = last_test.builder.build_hash self.assertEqual(hash_a, hash_b) + + def test_suite_with_source(self): + """Test that Pavilion correctly finds and uses source files in the suites directory.""" + + arg_parser = arguments.get_parser() + args = arg_parser.parse_args([ + 'run', + 'suite_with_source' + ]) + + run_cmd = commands.get_command(args.command_name) + ret = run_cmd.run(self.pav_cfg, args) + + self.assertEqual(ret, 0) From 5b4d6d4093fe0756a7f6fa8f309086e4eb91ea5c Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 25 Sep 2024 11:32:08 -0600 Subject: [PATCH 62/71] Pass suite_with_source test --- lib/pavilion/builder.py | 13 ++++++++++--- lib/pavilion/config.py | 2 +- lib/pavilion/resolver/resolver.py | 15 ++------------- lib/pavilion/test_run/test_run.py | 20 +++++++++++++------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index b5d00f72b..f94c2cb67 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -143,6 +143,9 @@ def suite_subdir(self) -> Optional[Path]: if sname is not None: return Path(f"suites/{sname}/") + self.status.set(STATES.WARNING, + "Unable to determine name of test suite. Suite directory is unknown.") + def exists(self): """Return True if the given build exists.""" return self.path.exists() @@ -317,11 +320,12 @@ def _update_src(self) -> Optional[Path]: :returns: src_path, extra_files """ + self.status.set(STATES.INFO, "Updating source.") + src_path = self._config.get('source_path') if src_path is None: - # There is no source to do anything with. - return None + return try: src_path = Path(src_path) @@ -681,11 +685,14 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): os.umask(umask) raw_src_path = self._config.get('source_path') + raw_src_path = set_default(raw_source_path, self.suite_subdir) + if raw_src_path is None: src_path = None else: - sub_dirs = [self.suite_subdir, Path('test_src')] + sub_dirs = [Path('test_src')] src_path = self._pav_cfg.find_file(raw_src_path, sub_dirs) + if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 89ce93a2f..517838d34 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -262,7 +262,7 @@ def suite_info(self) -> List[Tuple[str, str, Path]]: suites_dir = Path(cfg['path']) / 'suites' if tests_dir.exists(): - tests = [file for file in test_dir.iterdir() if (file.suffix.lower() == ".yaml")] + tests = [file for file in tests_dir.iterdir() if (file.suffix.lower() == ".yaml")] names = [test.name for test in tests] labels = [label] * len(tests) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 1f7476f99..01e2f04ee 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -244,17 +244,6 @@ def find_similar_configs(self, conf_type: str, conf_name: str) -> List[str]: return similarity.find_matches(conf_name, names) - - @staticmethod - def _make_suites_dict(self, suite_dirs: Iterable[Path]) -> Dict[str, Path]: - """Construct a dictionary mapping suite names to their respective suite.yaml - files.""" - - pairs = ((sdir.stem, sdir / "suite.yaml") for sdir in suite_dirs) - - return dict(pairs) - - def find_all_tests(self): """Find all the tests within known config directories. @@ -280,7 +269,7 @@ def find_all_tests(self): # TODO: Figure out how to do this and get the labels for label, name, path in self.pav_cfg.suite_info: if name not in suites: - suites[suite_name] = { + suites[name] = { 'path': path, 'label': label, 'err': '', @@ -328,7 +317,7 @@ def default(val, dval): } return suites - def find_all_configs(self, conf_type): + def find_all_configs(self, conf_type: str): """ Find all configs (host/modes) within known config directories. :return: Returns a dictionary of suite names to an info dict. diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 86de1219a..4c9b72c72 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -13,7 +13,7 @@ import time import uuid from pathlib import Path -from typing import TextIO, Union, Dict +from typing import TextIO, Union, Dict, Optional import yc_yaml as yaml from pavilion.config import PavConfig @@ -153,8 +153,8 @@ def __init__(self, pav_cfg: PavConfig, config: Dict, var_man: VariableSetManager self.rebuild = rebuild self.cfg_label = config.get('cfg_label', self.NO_LABEL) suite_path = config.get('suite_path') - if suite_path == '' or suite_path is None: - self.suite_path = Path('..') + if suite_path == '': + self.suite_path = None else: self.suite_path = Path(suite_path) self.user = utils.get_login() @@ -253,9 +253,10 @@ def is_empty(section: str) -> bool: return not all(map(is_empty, sections)) @property - def suite_name(self) -> str: + def suite_name(self) -> Optional[str]: """Return the name of the suite associated with the test.""" - return self.suite_path.stem + if self.suite_path is not None: + return self.suite_path.stem @property def id_pair(self) -> ID_Pair: @@ -342,7 +343,7 @@ def _make_builder(self) -> builder.TestBuilder: spack_config = (self.config.get('spack_config', {}) if self.spack_enabled() else None) - if self.suite_path != Path('..') and self.suite_path is not None: + if self.suite_path is not None: download_dest = self.suite_path # Check the deprecated directory @@ -353,10 +354,15 @@ def _make_builder(self) -> builder.TestBuilder: templates = self._create_build_templates() + config = self.config.get('build', {}) + + if self.suite_name is not None: + config['suite_name'] = self.suite_name + try: test_builder = builder.TestBuilder( pav_cfg=self._pav_cfg, - config=self.config.get('build', {}), + config=config, script=self.build_script_path, spack_config=spack_config, status=self.status, From 6ad35b99d095cc100d12fec49956325833772401 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Wed, 25 Sep 2024 15:01:06 -0600 Subject: [PATCH 63/71] Fix regressions --- lib/pavilion/builder.py | 26 ++++++++++++++++---------- lib/pavilion/config.py | 17 +++++++++-------- lib/pavilion/resolver/resolver.py | 1 - lib/pavilion/test_run/test_run.py | 6 ++++-- test/tests/config_cmd_tests.py | 6 +++--- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index f94c2cb67..ebd65420f 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -672,7 +672,7 @@ def _build(self, build_dir, cancel_event, test_id, tracker: BuildTracker) -> boo 'x-lzma', ) - def _setup_build_dir(self, dest, tracker: BuildTracker): + def _setup_build_dir(self, dest: Path, tracker: BuildTracker) -> None: """Setup the build directory, by extracting or copying the source and any extra files. :param dest: Path to the intended build directory. This is generally a @@ -684,32 +684,36 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): umask = os.umask(0) os.umask(umask) + src_path = None raw_src_path = self._config.get('source_path') - raw_src_path = set_default(raw_source_path, self.suite_subdir) - if raw_src_path is None: - src_path = None - else: + if raw_src_path is not None: sub_dirs = [Path('test_src')] src_path = self._pav_cfg.find_file(raw_src_path, sub_dirs) + # Only raise an error if a path that is explicitly identified is missing if src_path is None: raise TestBuilderError("Could not find source file '{}'" .format(raw_src_path)) - - # Resolve any softlinks to get the real file. - src_path = src_path.resolve() + else: + # Default to the suite directory, which may or may not exist + # If it doesn't exist, we should just continue without raising an error. + if self.suite_subdir is not None: + src_path = self._pav_cfg.find_file(self.suite_subdir) umask = int(self._pav_cfg['umask'], 8) # All of the file extraction functions return an error message on failure, None on success. extract_error = None + if src_path is not None: + # Resolve any softlinks to get the real file. + src_path = src_path.resolve() + if src_path is None: # If there is no source archive or data, just make the build # directory. dest.mkdir() - elif src_path.is_dir(): # Recursively copy the src directory to the build directory. tracker.update( @@ -723,7 +727,9 @@ def _setup_build_dir(self, dest, tracker: BuildTracker): dest.as_posix(), copy_function=shutil.copyfile, copystat=utils.make_umask_filtered_copystat(umask), - symlinks=True) + symlinks=True, + ignore=shutil.ignore_patterns("*.yaml") + ) elif src_path.is_file(): category, subtype = utils.get_mime_type(src_path) diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 517838d34..922deaedf 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -17,7 +17,7 @@ import yaml_config as yc from pavilion import output from pavilion import errors -from pavilion.micro import first, flatten, remove_none +from pavilion.micro import first, flatten, remove_none, set_default from pavilion.path_utils import Pathlike, append_to_path, append_suffix, exists, path_product # Figure out what directories we'll search for the base configuration. @@ -255,28 +255,28 @@ def suite_info(self) -> List[Tuple[str, str, Path]]: """Get the label, name, and path for every suite the config knows about.""" - suites = [] + suite_infos = [] - for label, cfg in self.configs.values(): + for label, cfg in self.configs.items(): tests_dir = Path(cfg['path']) / 'tests' suites_dir = Path(cfg['path']) / 'suites' if tests_dir.exists(): - tests = [file for file in tests_dir.iterdir() if (file.suffix.lower() == ".yaml")] + tests = [file for file in tests_dir.iterdir() if file.suffix.lower() == ".yaml"] names = [test.name for test in tests] labels = [label] * len(tests) - suites.extend(zip(labels, names, tests)) + suite_infos.extend(zip(labels, names, tests)) if suites_dir.exists(): - suites = [sdir / "suite.yaml" in suites_dir.iterdir()] + suites = [sdir / "suite.yaml" for sdir in suites_dir.iterdir()] suites = list(filter(exists, suites)) names = [suite.parent.name for suite in suites] labels = [label] * len(suites) - suites.extend(zip(labels, names, suites)) + suite_infos.extend(zip(labels, names, suites)) - return suites + return suite_infos def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = None) \ -> Union[Path, None]: @@ -297,6 +297,7 @@ def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = else: return None + sub_dirs = set_default(sub_dirs, []) sub_dirs = remove_none(list(sub_dirs)) paths = path_product(self.config_paths, sub_dirs) files = map(append_to_path(file), paths) diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 01e2f04ee..b473abc78 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -266,7 +266,6 @@ def find_all_tests(self): suites = {} - # TODO: Figure out how to do this and get the labels for label, name, path in self.pav_cfg.suite_info: if name not in suites: suites[name] = { diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 4c9b72c72..e5b6e89ec 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -153,10 +153,12 @@ def __init__(self, pav_cfg: PavConfig, config: Dict, var_man: VariableSetManager self.rebuild = rebuild self.cfg_label = config.get('cfg_label', self.NO_LABEL) suite_path = config.get('suite_path') - if suite_path == '': + + if suite_path is None or suite_path == '': self.suite_path = None else: self.suite_path = Path(suite_path) + self.user = utils.get_login() self.uuid = str(uuid.uuid4()) @@ -355,7 +357,7 @@ def _make_builder(self) -> builder.TestBuilder: templates = self._create_build_templates() config = self.config.get('build', {}) - + if self.suite_name is not None: config['suite_name'] = self.suite_name diff --git a/test/tests/config_cmd_tests.py b/test/tests/config_cmd_tests.py index 922283da1..26a9ca857 100644 --- a/test/tests/config_cmd_tests.py +++ b/test/tests/config_cmd_tests.py @@ -49,12 +49,12 @@ def test_config_cmds(self): self.assertTrue('main' in pav_cfg.configs) self.assertEqual(pav_cfg.working_dir, test_config_wd) # Make sure all created files exist. - for subfile in ['config.yaml', 'test_src', 'tests', 'hosts', 'modes', 'plugins', + for subfile in ['config.yaml', 'suites', 'hosts', 'modes', 'plugins', 'pavilion.yaml', 'os']: self.assertTrue((test_config_root / subfile).exists()) # Make sure groups are sane. if other_group is not None: - for path in (test_config_root, test_config_root/'tests', test_pav_config_path, + for path in (test_config_root, test_config_root/'suites', test_pav_config_path, pav_cfg.working_dir): self.assertEqual(path.group(), other_group, msg="Path '{}' should have group '{}', but had group '{}'" @@ -70,7 +70,7 @@ def test_config_cmds(self): foo_config_dir = test_config_root/'foo' args = arg_parser.parse_args(['config', 'create', 'foo', foo_config_dir.as_posix()]) self.assertEqual(config_cmd.run(pav_cfg, args), 0) - for subfile in ['config.yaml', 'test_src', 'tests', 'hosts', 'modes', 'plugins', + for subfile in ['config.yaml', 'suites', 'hosts', 'modes', 'plugins', 'os']: self.assertTrue((foo_config_dir/subfile).exists()) From d39693cffa164c5a54b2926c612e71e4dd77e44a Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 26 Sep 2024 14:19:03 -0600 Subject: [PATCH 64/71] Pass suite_with_source unittest --- lib/pavilion/builder.py | 7 +++++++ lib/pavilion/config.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index ebd65420f..6de5659eb 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -681,6 +681,8 @@ def _setup_build_dir(self, dest: Path, tracker: BuildTracker) -> None: :return: None """ + tracker.update(state=STATES.BUILDING, note="Setting up build directory.") + umask = os.umask(0) os.umask(umask) @@ -688,6 +690,7 @@ def _setup_build_dir(self, dest: Path, tracker: BuildTracker) -> None: raw_src_path = self._config.get('source_path') if raw_src_path is not None: + tracker.update(state=STATES.BUILDING, note=f"Looking for source path: {raw_src_path}.") sub_dirs = [Path('test_src')] src_path = self._pav_cfg.find_file(raw_src_path, sub_dirs) @@ -699,6 +702,8 @@ def _setup_build_dir(self, dest: Path, tracker: BuildTracker) -> None: # Default to the suite directory, which may or may not exist # If it doesn't exist, we should just continue without raising an error. if self.suite_subdir is not None: + tracker.update(state=STATES.BUILDING, + note=f"No source path given. Defaulting to {self.suite_subdir}.") src_path = self._pav_cfg.find_file(self.suite_subdir) umask = int(self._pav_cfg['umask'], 8) @@ -711,6 +716,8 @@ def _setup_build_dir(self, dest: Path, tracker: BuildTracker) -> None: src_path = src_path.resolve() if src_path is None: + tracker.update(state=STATES.BUILDING, + note=f"No source path found. Creating empty build directory.") # If there is no source archive or data, just make the build # directory. dest.mkdir() diff --git a/lib/pavilion/config.py b/lib/pavilion/config.py index 922deaedf..a7b2b1a20 100644 --- a/lib/pavilion/config.py +++ b/lib/pavilion/config.py @@ -19,6 +19,7 @@ from pavilion import errors from pavilion.micro import first, flatten, remove_none, set_default from pavilion.path_utils import Pathlike, append_to_path, append_suffix, exists, path_product +from pavilion.status_file import STATES # Figure out what directories we'll search for the base configuration. PAV_CONFIG_SEARCH_DIRS = [Path('./').resolve()] @@ -298,9 +299,14 @@ def find_file(self, file: Pathlike, sub_dirs: Union[List[Pathlike], Pathlike] = return None sub_dirs = set_default(sub_dirs, []) - sub_dirs = remove_none(list(sub_dirs)) - paths = path_product(self.config_paths, sub_dirs) - files = map(append_to_path(file), paths) + sub_dirs = list(remove_none(list(sub_dirs))) + + if len(sub_dirs) > 0: + paths = list(path_product(self.config_paths, sub_dirs)) + else: + paths = list(self.config_paths) + + files = list(map(append_to_path(file), paths)) # Return the first path to the file that exists (or None) return first(exists, files) From 084c151119c7c9ea5983c70d480cb7546e437302 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Thu, 26 Sep 2024 16:28:31 -0600 Subject: [PATCH 65/71] Pass circular_symlink unittest --- lib/pavilion/builder.py | 9 +++++++-- test/tests/builder_tests.py | 11 ++++++++--- test/tests/suites_tests.py | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 6de5659eb..3aebca3dc 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -222,13 +222,17 @@ def _create_build_hash(self) -> str: if src_path is not None: if src_path.is_file(): + self.status.set(STATES.INFO, f"Hashing file {src_path}.") hash_obj.update(self._hash_file(src_path)) elif src_path.is_dir(): + self.status.set(STATES.INFO, f"Hashing directory {src_path}.") hash_obj.update(self._hash_dir(src_path, exclude=CONFIG_FNAMES)) else: raise TestBuilderError( "Invalid src location {}." .format(src_path)) + else: + self.status.set(STATES.INFO, "No files to hash.") # Hash all the given template files. for tmpl_src in sorted(self._templates.keys()): @@ -1015,6 +1019,7 @@ def _hash_dir(cls, path: Path, exclude: List[str] = None) -> str: # Order is indeterminate, so sort the files files = sorted(path.rglob('*')) except OSError: + # This will typically be caught earlier by _date_dir raise TestBuilderError(f"Unable to hash directory: {path}. Possible circular symlink.") files = filter(lambda x: x.name not in exclude, files) @@ -1054,8 +1059,8 @@ def _date_dir(base_path): dir_stat = path.stat() except OSError as err: raise TestBuilderError( - "Could not stat file in test source dir '{}'" - .format(base_path), err) + (f"Could not stat file in test source dir '{base_path}'. " + "Possible circular symlink."), err) if dir_stat.st_mtime > latest: latest = dir_stat.st_mtime diff --git a/test/tests/builder_tests.py b/test/tests/builder_tests.py index f03951049..eae6b909e 100644 --- a/test/tests/builder_tests.py +++ b/test/tests/builder_tests.py @@ -466,7 +466,12 @@ def test_hash_circular_symlinks(self): 'circular_symlinks' ]) - build_cmd = commands.get_command(args.command_name) + run_cmd = commands.get_command(args.command_name) - with self.assertRaises(TestBuilderError): - build_cmd.run(self.pav_cfg, args) + run_cmd.run(self.pav_cfg, args) + + last_series = run_cmd.last_series + + last_series.wait() + + self.assertEqual(last_series.status.current().state, "ERROR") diff --git a/test/tests/suites_tests.py b/test/tests/suites_tests.py index ef2588c96..c153ede14 100644 --- a/test/tests/suites_tests.py +++ b/test/tests/suites_tests.py @@ -10,7 +10,7 @@ def setUp(self): plugins.initialize_plugins(self.pav_cfg) run_cmd = commands.get_command('run') build_cmd = commands.get_command('build') - # run_cmd.silence() + run_cmd.silence() def test_suite_run_from_suite_directory(self): """Test that Pavilion can find and run a test from From 793f9111b9c498dad0d944221c71b9e5cf7975fd Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 27 Sep 2024 08:47:21 -0600 Subject: [PATCH 66/71] Finish implementing PR feedback --- lib/pavilion/series/test_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/series/test_set.py b/lib/pavilion/series/test_set.py index 17b4a174a..7d5fe40fc 100644 --- a/lib/pavilion/series/test_set.py +++ b/lib/pavilion/series/test_set.py @@ -733,12 +733,12 @@ def wait(self, wait_for_all=False, wait_period: int = TEST_WAIT_PERIOD, completed_tests = self.mark_completed() - start = time.monotonic() + start = time.time() while ((wait_for_all and self.started_tests) or (not wait_for_all and completed_tests == 0)): - if time.monotonic() - start > timeout: + if time.time() - start > timeout: raise TimeoutError("Timed out waiting for test set to complete.") time.sleep(wait_period) From 67c8e2b4342665b8efc99e124df8ce1e279be7ca Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 30 Sep 2024 08:40:14 -0600 Subject: [PATCH 67/71] Fix failing status_history unittest (hopefully) --- test/tests/status_cmd_tests.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/tests/status_cmd_tests.py b/test/tests/status_cmd_tests.py index 928ecc073..f4e66b289 100644 --- a/test/tests/status_cmd_tests.py +++ b/test/tests/status_cmd_tests.py @@ -1,6 +1,8 @@ import argparse import io import time +import re +from typing import List from pavilion import commands from pavilion import plugins @@ -11,6 +13,9 @@ from pavilion.unittest import PavTestCase +STATUS_REGEX = re.compile(r"^\s*[A-Z]") + + class StatusCmdTests(PavTestCase): def setUp(self): @@ -273,6 +278,7 @@ def test_status_history(self): raw = schedulers.get_plugin('raw') raw.schedule_tests(self.pav_cfg, [test]) end = time.time() + 5 + while not test.complete and time.time() < end: time.sleep(.1) @@ -282,7 +288,17 @@ def test_status_history(self): out.seek(0) output = out.readlines()[4:] statuses = test.status.history() - self.assertEqual(len(output), len(statuses), msg='output: {}, statuses: {}' + + def count_statuses(out: List[str]) -> int: + num_statuses = 0 + + for line in out: + if re.search(STATUS_REGEX, line): + num_statuses += 1 + + return num_statuses + + self.assertEqual(count_statuses(output), len(statuses), msg='output: {}, statuses: {}' .format(output, statuses)) for i in range(len(output)): self.assertTrue(statuses[i].state in output[i]) From 9f59e78f2ccafc6fda2028e26d86580777bb7919 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 30 Sep 2024 08:59:00 -0600 Subject: [PATCH 68/71] Try again to fix status_history unittest in CI --- test/tests/status_cmd_tests.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test/tests/status_cmd_tests.py b/test/tests/status_cmd_tests.py index f4e66b289..756bd36a5 100644 --- a/test/tests/status_cmd_tests.py +++ b/test/tests/status_cmd_tests.py @@ -289,16 +289,11 @@ def test_status_history(self): output = out.readlines()[4:] statuses = test.status.history() - def count_statuses(out: List[str]) -> int: - num_statuses = 0 + # Some statuses are split over multiple lines, but only in CI for some reason + output = list(filter(lambda x: re.search(STATUS_REGEX, x), output)) - for line in out: - if re.search(STATUS_REGEX, line): - num_statuses += 1 - - return num_statuses - - self.assertEqual(count_statuses(output), len(statuses), msg='output: {}, statuses: {}' + self.assertEqual(len(output), len(statuses), msg='output: {}, statuses: {}' .format(output, statuses)) + for i in range(len(output)): self.assertTrue(statuses[i].state in output[i]) From b7c39ef43c563bc00fbf1fb1af0cfaec376f11dc Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 30 Sep 2024 10:07:20 -0600 Subject: [PATCH 69/71] Attempt to fix demo unittest in CI --- lib/pavilion/test_run/test_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index e5b6e89ec..d268976c2 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -345,11 +345,12 @@ def _make_builder(self) -> builder.TestBuilder: spack_config = (self.config.get('spack_config', {}) if self.spack_enabled() else None) + if self.suite_path is not None: download_dest = self.suite_path # Check the deprecated directory - if not download_dest.exists(): + if not download_dest.exists() or download_dest.is_dir(): download_dest = self.suite_path.parents[1] / 'test_src' else: download_dest = None From 7999eb9d84e749fc4262b2aaf395d31a7977c1a7 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Mon, 30 Sep 2024 10:12:57 -0600 Subject: [PATCH 70/71] Try again --- lib/pavilion/test_run/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index d268976c2..18b6a5025 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -350,7 +350,7 @@ def _make_builder(self) -> builder.TestBuilder: download_dest = self.suite_path # Check the deprecated directory - if not download_dest.exists() or download_dest.is_dir(): + if not download_dest.exists() or not download_dest.is_dir(): download_dest = self.suite_path.parents[1] / 'test_src' else: download_dest = None From 318c32af61b9ca9699fc507e3c7cbfb52c135465 Mon Sep 17 00:00:00 2001 From: hwikle-lanl <154543628+hwikle-lanl@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:09:07 -0600 Subject: [PATCH 71/71] Fix failing pylint tests in CI (#786) * Update version of checkout action to elminate warnings * Update setup-python action to fix warning --- .github/workflows/unittests.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 1caae731f..fd717ced3 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -1,4 +1,4 @@ -name: unitests +name: unittests on: push: @@ -15,12 +15,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -65,7 +64,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v2 @@ -87,12 +86,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -117,12 +115,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -148,12 +145,12 @@ jobs: - docs steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.6 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.6 @@ -209,12 +206,11 @@ jobs: - docs steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10"