Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Derived test module reloads and cleanup #14

Merged
merged 12 commits into from
Nov 25, 2024
Merged

Derived test module reloads and cleanup #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9a40b5b
Derived test module reloads and cleanup
Nov 21, 2024
09d2592
Fix log file creation
Nov 21, 2024
b78fbff
Fix CI restarts
Nov 21, 2024
1640ee7
Fix nbody test
Nov 21, 2024
1e4e730
Fix a few more edge cases
Nov 21, 2024
c35073e
Attempt to address review comments
Nov 22, 2024
7ebe636
analytic --> analysis fixes
Nov 22, 2024
e629b97
Change artifacts path
Nov 22, 2024
59906ec
Fix artemis module access
Nov 23, 2024
9e0e774
Merge remote-tracking branch 'upstream/develop' into pdmullen/derived…
Nov 23, 2024
5943b31
Merge branch 'develop' into pdmullen/derived_tests
brryan Nov 24, 2024
5e90a73
Add --cwd option
Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,18 +77,19 @@ jobs:
python3 run_tests.py regression.suite \
--save_build \
--reuse_build \
--use_oversubscribe \
--log_file=ci_cpu_log.txt
- name: Upload CPU test log
- name: Upload logs
if: always()
uses: actions/upload-artifact@v3
with:
name: ci_cpu_log.txt
path: tst/ci_cpu_log.txt
name: logs
path: tst/testing/logs
retention-days: 3
- name: Upload figures
if: always()
uses: actions/upload-artifact@v3
with:
name: figs
path: tst/figs
path: tst/testing/figs
retention-days: 3
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ doc/src/parameters.rst

# CI files
artemis_ci_*.out
tst/logs
tst/figs
**/testing/

# Visualization package files
.smhist
Expand Down
219 changes: 123 additions & 96 deletions tst/run_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
# the public, perform publicly and display publicly, and to permit others to do so.
# ========================================================================================

# This file was created in part by one of OpenAI's generative AI models

# Regression test script for Artemis.

# Usage: From this directory, call this script with python:
Expand All @@ -24,14 +22,16 @@
# - This file should not be modified when adding new scripts.
# - This file is largely borrowed from the open-source Athena++/AthenaK CI regression,
# adapted to work with LANL Darwin runners
# - This file was created in part by one of OpenAI's generative AI models

# Modules
import argparse
import os
from collections import OrderedDict
from importlib import reload
import logging
import logging.config
import shutil
from collections import OrderedDict
from importlib import reload
from pkgutil import iter_modules
from timeit import default_timer as timer

Expand All @@ -48,16 +48,10 @@
logger = logging.getLogger("artemis")


def read_cmakecache(filename):
with open(filename, "r") as f:
for line in f.readlines():
if "artemis_BINARY_DIR" in line:
return os.path.join(line.split("=")[-1].strip(), "src", "artemis")


def process_suite(filename):
tests = []
fname = os.path.join(artemis.get_source_directory(), "tst", "suites", filename)
aname = os.path.join(artemis.get_artemis_dir(), "tst")
fname = os.path.join(aname, "suites", filename)

with open(fname, "r") as f:
for line in f.readlines():
Expand All @@ -75,15 +69,7 @@ def process_suite(filename):
dir_test_names = [
name
for _, name, _ in iter_modules(
# path=["scripts/" + test_name], prefix=test_name + "."
path=[
os.path.join(
artemis.get_source_directory(),
"tst",
"scripts",
test_name,
)
],
path=[os.path.join(aname, "scripts", test_name)],
prefix=test_name + ".",
)
]
Expand All @@ -94,22 +80,16 @@ def process_suite(filename):
# Main function
def main(**kwargs):
# Make list of tests to run
scripts_path = os.path.join(artemis.get_artemis_dir(), "tst/scripts/")
tests = kwargs.pop("tests")
test_names = []
if len(tests) == 0: # run all tests
for _, directory, ispkg in iter_modules(path=["scripts"]):
if ispkg and (directory != "utils" and directory != "style"):
for _, directory, ispkg in iter_modules(path=[scripts_path]):
if ispkg and directory != "utils":
dir_test_names = [
name
for _, name, _ in iter_modules(
path=[
os.path.join(
artemis.get_source_directory(),
"tst",
"scripts",
directory,
)
],
path=[os.path.join(scripts_path, directory)],
prefix=directory + ".",
)
]
Expand All @@ -126,11 +106,7 @@ def main(**kwargs):
dir_test_names = [
name
for _, name, _ in iter_modules(
path=[
os.path.join(
artemis.get_source_directory(), "tst", "scripts", test
)
],
path=[os.path.join(scripts_path, test)],
prefix=test + ".",
)
]
Expand All @@ -141,7 +117,7 @@ def main(**kwargs):
logger.info("Running: " + ", ".join(test_names))

# Run tests
current_dir = os.getcwd()
build_dir = os.path.join(artemis.get_artemis_dir(), "tst")
test_times = []
test_results = []
test_errors = []
Expand All @@ -164,21 +140,28 @@ def main(**kwargs):
logger.warning("Unable to " 'import "{:}".'.format(missing_module))
deps_installed = False
if not deps_installed:
logger.warning("WARNING! Not all required Python modules " "are available")
logger.warning("WARNING! Not all required Python modules are available")

# Build Artemis
if not artemis.custom_exe and not kwargs.pop("reuse_build"):
if not artemis.get_supplied_exe() and not kwargs.pop("reuse_build"):
try:
os.system("rm -rf {0}/build".format(current_dir))
os.system("rm -rf {0}/build".format(build_dir))
# insert arguments for artemis.make()
artemis_cmake_args = kwargs.pop("cmake")
artemis_make_nproc = kwargs.pop("make_nproc")
module.artemis.make(artemis_cmake_args, artemis_make_nproc)
artemis.make(artemis_cmake_args, artemis_make_nproc)
except Exception:
logger.error("Exception occurred", exc_info=True)
test_errors.append("make()")
raise TestError("Unable to build Artemis")

# Build working directory and copy librebound.so into working directory
os.makedirs(artemis.get_outputs_dir(), exist_ok=True)
reb_path = os.path.join(artemis.get_exe_dir(), "librebound.so")
if not os.path.exists(reb_path):
raise TestError(f'librebound.so not found at "{reb_path}"!')
shutil.copy(reb_path, artemis.get_outputs_dir())

# Run each test
for name in test_names:
t0 = timer()
Expand Down Expand Up @@ -215,8 +198,8 @@ def main(**kwargs):
# For CI, print after every individual test has finished
logger.info("{} test: run(), analyze() finished".format(name))
finally:
if not kwargs.pop("save_build") and not artemis.custom_exe:
os.system("rm -rf {0}/build".format(current_dir))
if not kwargs.pop("save_build") and not artemis.get_supplied_exe():
os.system("rm -rf {0}/build".format(build_dir))

# Report test results
logger.info("\nResults:")
Expand Down Expand Up @@ -269,7 +252,8 @@ def log_init(args):
# setup log_file
log_fn = kwargs.pop("log_file")
if log_fn:
f_handler = logging.FileHandler(os.path.join(artemis.artemis_log_dir, log_fn))
os.makedirs(artemis.get_log_dir(), exist_ok=True)
f_handler = logging.FileHandler(os.path.join(artemis.get_log_dir(), log_fn))
f_handler.setLevel(0) # log everything
f_format = logging.Formatter(
"%(asctime)s|%(levelname)s" ":%(name)s: %(message)s"
Expand All @@ -279,61 +263,91 @@ def log_init(args):
logger.debug("Starting Artemis regression tests")


def set_paths(args):
# Reads Artemis CMakeCache.txt to find executable
def read_cmakecache(filename):
with open(filename, "r") as f:
for line in f.readlines():
if "artemis_BINARY_DIR" in line:
return os.path.join(line.split("=")[-1].strip(), "src", "artemis")


# Sets global variables
def set_globals(args):
kwargs = vars(args)

# Set MPI oversubscribe
if kwargs.pop("use_oversubscribe"):
artemis.set_mpi_oversubscribe(True)

# Check for executable path and output directory args
out_dir = kwargs.pop("output_dir")
out_dir = os.path.join(out_dir, "testing") if out_dir is not None else None
exe_path = kwargs.pop("exe")
use_cwd = kwargs.pop("cwd")

# Set the correct paths
artemis_exe_path = kwargs.pop("exe")
output_dir = kwargs.pop("output_dir")

# Check that path is valid
if output_dir is not None:
if not (os.path.exists(output_dir) and os.access(output_dir, os.W_OK)):
raise TestError(
f'Provided output directory "{output_dir}" not found or it is not writeable!'
)
# output_dir is a valid path or None

if artemis_exe_path is not None:
# Check that path is valid
if not (
os.path.exists(artemis_exe_path) and os.access(artemis_exe_path, os.X_OK)
):
raise TestError(
f'Provided executable "{artemis_exe_path}" not found or cannot be executed!'
)

# If no output_dir was passed, set it to the exe path
if output_dir is None:
output_dir = os.path.dirname(artemis_exe_path)
# Set the valid provided executable path
abs_exe_path = os.path.abspath(artemis_exe_path)
print(f"Found local executable {abs_exe_path}")
print(f"Outputting results to {output_dir}")
artemis.set_executable(abs_exe_path, output_dir)
if exe_path is not None:
adir = os.path.join(artemis.get_artemis_dir(), "tst")
out_dir = os.path.join(adir, "testing") if out_dir is None else out_dir
reb_path = os.path.join(os.path.dirname(exe_path), "librebound.so")

if use_cwd:
raise TestError("--cwd and --exe=PATH cannot be passed together!")

if not (os.path.exists(exe_path) and os.access(exe_path, os.X_OK)):
raise TestError(f'Provided exe "{exe_path}" not found or cannot be run!')

if not os.path.exists(reb_path):
raise TestError(f'librebound.so not found at "{reb_path}"!')

abs_out_dir = os.path.abspath(out_dir)
abs_exe_dir = os.path.abspath(os.path.dirname(exe_path))
artemis.set_paths(abs_exe_dir, abs_out_dir)
artemis.set_supplied_exe(True)
else:
# If no output_dir was passed, set it to the cwd
if output_dir is None:
output_dir = os.getcwd()
# If we are in a directory with an executable, default to using that
local_path = os.path.join(os.getcwd(), "artemis")
if os.path.exists(local_path) and os.access(local_path, os.X_OK):
print(f"Found local executable {local_path}")
print(f"Outputting results to {output_dir}")
artemis.set_executable(local_path, output_dir)
cwd = os.getcwd()
lpath_exe = os.path.join(cwd, "artemis")
lpath_cache = os.path.join(cwd, "CMakeCache.txt")
if use_cwd:
# If we are in a directory with an executable, default to using that
if os.path.isfile(lpath_exe) and os.access(lpath_exe, os.X_OK):
exe_path = lpath_exe
out_dir = os.path.join(cwd, "testing") if out_dir is None else out_dir
reb_path = os.path.join(os.path.dirname(exe_path), "librebound.so")

if not os.path.exists(reb_path):
raise TestError(f'librebound.so not found at "{reb_path}"!')

abs_out_dir = os.path.abspath(out_dir)
abs_exe_dir = os.path.abspath(os.path.dirname(exe_path))
artemis.set_paths(abs_exe_dir, abs_out_dir)
artemis.set_supplied_exe(True)
elif os.path.exists(lpath_cache) and os.access(lpath_cache, os.R_OK):
# Check if we are one level up from the executable
exe_path = read_cmakecache(lpath_cache)
out_dir = os.path.join(cwd, "testing") if out_dir is None else out_dir
reb_path = os.path.join(os.path.dirname(exe_path), "librebound.so")

if not (os.path.exists(exe_path) and os.access(exe_path, os.X_OK)):
raise TestError(f'No exe in "{exe_path}" or cannot be run!')

if not os.path.exists(reb_path):
raise TestError(f'librebound.so not found at "{reb_path}"!')

abs_out_dir = os.path.abspath(out_dir)
abs_exe_dir = os.path.abspath(os.path.dirname(exe_path))
artemis.set_paths(abs_exe_dir, abs_out_dir)
artemis.set_supplied_exe(True)
else:
raise TestError("Unable to find executable via --cwd")
else:
# Check if we are one level up from the executable
local_path = os.path.join(os.getcwd(), "CMakeCache.txt")
if os.path.exists(local_path) and os.access(local_path, os.R_OK):
# Pull out the executable path
exe_path = read_cmakecache(local_path)
if os.path.exists(exe_path) and os.access(exe_path, os.X_OK):
print(f"Found local executable {exe_path}")
print(f"Outputting results to {output_dir}")
artemis.set_executable(exe_path, output_dir)
else:
raise TestError(
f'Could not find executable in "{exe_path}" or cannot be executed!'
)
adir = os.path.join(artemis.get_artemis_dir(), "tst")
exe_path = os.path.join(adir, "build/src/artemis")
out_dir = os.path.join(adir, "testing") if out_dir is None else out_dir
abs_out_dir = os.path.abspath(out_dir)
abs_exe_dir = os.path.abspath(os.path.dirname(exe_path))
artemis.set_paths(abs_exe_dir, abs_out_dir)
artemis.set_supplied_exe(False)


# Execute main function
Expand All @@ -353,6 +367,12 @@ def set_paths(args):
"--make_nproc", type=int, default=8, help="set nproc N for make -jN"
)

parser.add_argument(
"--use_oversubscribe",
action="store_true",
help="use MPI oversubscribe",
)

parser.add_argument(
"--log_file", type=str, default=None, help="set filename of logfile"
)
Expand All @@ -375,15 +395,22 @@ def set_paths(args):
default=None,
help="path to pre-built executable",
)

parser.add_argument(
"--output_dir",
type=str,
default=None,
help="path to output directory",
)

parser.add_argument(
"--cwd",
action="store_true",
help="search local path for preexisting executable",
)

args = parser.parse_args()
set_paths(args)
set_globals(args)
log_init(args)

try:
Expand Down
Loading
Loading