From c04501751d6c1fe9851690f491f61d697ba57a87 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Wed, 12 Jun 2024 18:52:35 -0500 Subject: [PATCH 01/17] in progress command generation --- smartsim/_core/generation/commandgenerator.py | 115 ++++++ tests/test_command_generation.py | 198 ++++++++++ .../tagged_tests/correct/MOM_input | 363 ++++++++++++++++++ .../tagged_tests/correct/example_input.i | 118 ++++++ .../tagged_tests/correct/in.airebo | 22 ++ .../test_configs/tagged_tests/correct/in.atm | 31 ++ .../tagged_tests/correct/in.crack | 78 ++++ .../tagged_tests/correct/in.ellipse.gayberne | 66 ++++ .../tagged_tests/correct/input-file.inp | 245 ++++++++++++ .../tagged_tests/correct/input.nml | 22 ++ .../tagged_tests/correct/simple-H20.xml | 86 +++++ .../tagged_tests/marked/MOM_input | 363 ++++++++++++++++++ .../tagged_tests/marked/example_input.i | 118 ++++++ .../tagged_tests/marked/in.airebo | 22 ++ tests/test_configs/tagged_tests/marked/in.atm | 31 ++ .../test_configs/tagged_tests/marked/in.crack | 78 ++++ .../tagged_tests/marked/in.ellipse.gayberne | 66 ++++ .../tagged_tests/marked/input-file.inp | 245 ++++++++++++ .../tagged_tests/marked/input.nml | 22 ++ .../tagged_tests/marked/simple-H20.xml | 86 +++++ 20 files changed, 2375 insertions(+) create mode 100644 smartsim/_core/generation/commandgenerator.py create mode 100644 tests/test_command_generation.py create mode 100644 tests/test_configs/tagged_tests/correct/MOM_input create mode 100644 tests/test_configs/tagged_tests/correct/example_input.i create mode 100644 tests/test_configs/tagged_tests/correct/in.airebo create mode 100644 tests/test_configs/tagged_tests/correct/in.atm create mode 100644 tests/test_configs/tagged_tests/correct/in.crack create mode 100644 tests/test_configs/tagged_tests/correct/in.ellipse.gayberne create mode 100644 tests/test_configs/tagged_tests/correct/input-file.inp create mode 100644 tests/test_configs/tagged_tests/correct/input.nml create mode 100644 tests/test_configs/tagged_tests/correct/simple-H20.xml create mode 100644 tests/test_configs/tagged_tests/marked/MOM_input create mode 100644 tests/test_configs/tagged_tests/marked/example_input.i create mode 100644 tests/test_configs/tagged_tests/marked/in.airebo create mode 100644 tests/test_configs/tagged_tests/marked/in.atm create mode 100644 tests/test_configs/tagged_tests/marked/in.crack create mode 100644 tests/test_configs/tagged_tests/marked/in.ellipse.gayberne create mode 100644 tests/test_configs/tagged_tests/marked/input-file.inp create mode 100644 tests/test_configs/tagged_tests/marked/input.nml create mode 100644 tests/test_configs/tagged_tests/marked/simple-H20.xml diff --git a/smartsim/_core/generation/commandgenerator.py b/smartsim/_core/generation/commandgenerator.py new file mode 100644 index 000000000..6870f07f9 --- /dev/null +++ b/smartsim/_core/generation/commandgenerator.py @@ -0,0 +1,115 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# TODO: rename and make all the descriptions accurate and good +# including the params and stuff + +def delete_op(to_delete: str): + """Produce string of a script that deletes a file when executed. + + :param to_delete: path to the file to be deleted + :return: string to be shipped as a command that deletes the given file + """ + return f"import os;os.remove('{to_delete}')" + + +def copy_op(source_file: str, dest_file: str): + """return command that can be popened""" + return f"import shutil;shutil.copyfile('{source_file}', '{dest_file}')" + + +def symlink_op(dest_file: str, source_file: str): + """ + Make the source path a symlink pointing to the destination path. + + :param source_file: + :param dest_file: + :return: + """ + return f"import os;os.symlink(str('{source_file}'), str('{dest_file}'))" + +def move_op(source_file, dest_file): + """Execute a script that moves a file""" + return f"import shutil;shutil.move('{source_file}','{dest_file}')" + + +def configure_op(file_path, param_dict): + """configure file with params + TODO: clean this up""" + + return fr""" +import re +import collections + +edited = [] +used_params = {{}} +files_to_tags = {{}} +params = {param_dict} +regex = "(;.+;)" +tag = ";" + +def _get_prev_value(tagged_line: str) -> str: + split_tag = tagged_line.split(tag) + return split_tag[1] + +def _is_ensemble_spec( + tagged_line: str, model_params: dict + ) -> bool: + split_tag = tagged_line.split(tag) + prev_val = split_tag[1] + if prev_val in model_params.keys(): + return True + return False + +with open('{file_path}','r+', encoding='utf-8') as file_stream: + lines = file_stream.readlines() + +unused_tags = collections.defaultdict(list) + +for i, line in enumerate(lines, 1): + while search := re.search(regex, line): + tagged_line = search.group(0) + previous_value = _get_prev_value(tagged_line) + if _is_ensemble_spec(tagged_line, params): + new_val = str(params[previous_value]) + line = re.sub(regex, new_val, line, 1) + used_params[previous_value] = new_val + + # if a tag is found but is not in this model's configurations + # put in placeholder value + else: + tag = tagged_line.split(tag)[1] + unused_tags[tag].append(i) + line = re.sub(regex, previous_value, line) + break + edited.append(line) + +lines = edited + +with open('{file_path}', "w+", encoding="utf-8") as file_stream: + for line in lines: + file_stream.write(line) +""" diff --git a/tests/test_command_generation.py b/tests/test_command_generation.py new file mode 100644 index 000000000..0680d8895 --- /dev/null +++ b/tests/test_command_generation.py @@ -0,0 +1,198 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import filecmp +from glob import glob +import os +import pathlib +import subprocess +import sys +import typing as t +from os import path as osp + +from distutils import dir_util + +from smartsim._core.generation import commandgenerator + +test_path = os.path.dirname(os.path.abspath(__file__)) + +@staticmethod +def make_test_file( + file_name: str, file_dir: str, file_content: t.Optional[str] = None +) -> str: + """Create a dummy file in the test output directory. + + :param file_name: name of file to create, e.g. "file.txt" + :param file_dir: path + :return: String path to test output file + """ + file_path = os.path.join(file_dir, file_name) + os.makedirs(file_dir) + with open(file_path, "w+", encoding="utf-8") as dummy_file: + if not file_content: + dummy_file.write("dummy\n") + else: + dummy_file.write(file_content) + return file_path + + +def test_symlink(): + """ + Test symlink. Make path a symlink pointing to the given path + """ + # Prep test directory + test_output_root = os.path.join(test_path, "tests", "test_output") + target = pathlib.Path(test_output_root) / "sym_target_dir" / "target" + source = pathlib.Path(test_output_root) / "sym_source" + + # Build the command + cmd = commandgenerator.symlink_op(source, target) + + # execute the command + subprocess.run([sys.executable, "-c", cmd]) + + # Assert the two files are the same file + assert source.is_symlink() + assert os.readlink(source) == str(target) + + # Clean up the test directory + os.unlink(source) + + +def test_copy(): + """Copy the content of the source file to the destination file.""" + + test_output_root = os.path.join(test_path, "tests", "test_output") + + # make a test file with some contents + source_file = make_test_file( + "source.txt", pathlib.Path(test_output_root) / "source", "dummy" + ) + dest_file = make_test_file("dest.txt", pathlib.Path(test_output_root) / "dest", "") + + # assert that source file exists, has correct contents + assert osp.exists(source_file) + with open(source_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + cmd = commandgenerator.copy_op(source_file, dest_file) + subprocess.run([sys.executable, "-c", cmd]) + + with open(dest_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + # Clean up the test directory + os.remove(source_file) + os.remove(dest_file) + os.rmdir(pathlib.Path(test_output_root) / "source") + os.rmdir(pathlib.Path(test_output_root) / "dest") + + +def test_move(): + """Test to move command execution""" + test_output_root = os.path.join(test_path, "tests", "test_output") + source_dir = os.path.join(test_output_root, "from_here") + os.mkdir(source_dir) + dest_dir = os.path.join(test_output_root, "to_here") + os.mkdir(dest_dir) + dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") + source_file = pathlib.Path(source_dir) / "app_move.txt" + + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + assert osp.exists(source_file) + with open(source_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + + cmd = commandgenerator.move_op(source_file, dest_file) + subprocess.run([sys.executable, "-c", cmd]) + + # assert that the move was successful + assert not osp.exists(source_file) + assert osp.exists(dest_file) + with open(dest_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + # Clean up the directories + os.rmdir(source_dir) + os.remove(dest_file) + os.rmdir(dest_dir) + + +def test_delete(): + """Test python inline command to delete a file""" + test_output_root = os.path.join(test_path, "tests", "test_output") + + # Make a test file with dummy text + to_del = pathlib.Path(test_output_root) / "app_del.txt" + with open(to_del, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + assert osp.exists(to_del) + with open(to_del, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + cmd = commandgenerator.delete_op(to_del) + subprocess.run([sys.executable, "-c", cmd]) + + # Assert file has been deleted + assert not osp.exists(to_del) + + +def test_configure(test_dir, fileutils): + """ test configure param operations """ + + # the param dict for configure operations + param_dict = { + "5": 10, # MOM_input + "FIRST": "SECOND", # example_input.i + "17": 20, # in.airebo + "65": "70", # in.atm + "placeholder": "group leftupper region", # in.crack + "1200": "120", # input.nml + } + # retreive tagged files + conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) + # retrieve files to compare after test + correct_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "correct/")) + + # copy files to test directory + dir_util.copy_tree(conf_path, test_dir) + assert osp.isdir(test_dir) + + tagged_files = sorted(glob(test_dir + "/*")) + correct_files = sorted(glob(correct_path + "*")) + + # Run configure op on test files + for tagged_file in tagged_files: + cmd = commandgenerator.configure_op(tagged_file, param_dict) + subprocess.run([sys.executable, "-c", cmd]) + + # check that files and correct files are the same + for written, correct in zip(tagged_files, correct_files): + assert filecmp.cmp(written, correct) diff --git a/tests/test_configs/tagged_tests/correct/MOM_input b/tests/test_configs/tagged_tests/correct/MOM_input new file mode 100644 index 000000000..1e9002333 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/MOM_input @@ -0,0 +1,363 @@ +! This file was written by the model and records the non-default parameters used at run-time. + +! === module MOM === + +! === module MOM_unit_scaling === +! Parameters for doing unit scaling of variables. +ENABLE_THERMODYNAMICS = False ! [Boolean] default = True + ! If true, Temperature and salinity are used as state + ! variables. +ADIABATIC = True ! [Boolean] default = False + ! There are no diapycnal mass fluxes if ADIABATIC is + ! true. This assumes that KD = KDML = 0.0 and that + ! there is no buoyancy forcing, but makes the model + ! faster by eliminating subroutine calls. +THICKNESSDIFFUSE = True ! [Boolean] default = False + ! If true, interface heights are diffused with a + ! coefficient of KHTH. +DT = 1200.0 ! [s] + ! The (baroclinic) dynamics time step. The time-step that + ! is actually used will be an integer fraction of the + ! forcing time-step (DT_FORCING in ocean-only mode or the + ! coupling timestep in coupled mode.) +DTBT_RESET_PERIOD = -1.0 ! [s] default = 1200.0 + ! The period between recalculations of DTBT (if DTBT <= 0). + ! If DTBT_RESET_PERIOD is negative, DTBT is set based + ! only on information available at initialization. If 0, + ! DTBT will be set every dynamics time step. The default + ! is set by DT_THERM. This is only used if SPLIT is true. + +! === module MOM_domains === +REENTRANT_X = False ! [Boolean] default = True + ! If true, the domain is zonally reentrant. +NIGLOBAL = 80 ! + ! The total number of thickness grid points in the + ! x-direction in the physical domain. With STATIC_MEMORY_ + ! this is set in MOM_memory.h at compile time. +NJGLOBAL = 40 ! + ! The total number of thickness grid points in the + ! y-direction in the physical domain. With STATIC_MEMORY_ + ! this is set in MOM_memory.h at compile time. + +! === module MOM_hor_index === +! Sets the horizontal array index types. + +! === module MOM_verticalGrid === +! Parameters providing information about the vertical grid. +NK = 10 ! [nondim] + ! The number of model layers. + +! === module MOM_fixed_initialization === +INPUTDIR = "INPUT" ! default = "." + ! The directory in which input files are found. + +! === module MOM_grid_init === +GRID_CONFIG = "spherical" ! + ! A character string that determines the method for + ! defining the horizontal grid. Current options are: + ! mosaic - read the grid from a mosaic (supergrid) + ! file set by GRID_FILE. + ! cartesian - use a (flat) Cartesian grid. + ! spherical - use a simple spherical grid. + ! mercator - use a Mercator spherical grid. +SOUTHLAT = 30.0 ! [degrees] + ! The southern latitude of the domain. +LENLAT = 20.0 ! [degrees] + ! The latitudinal length of the domain. +LENLON = 40.0 ! [degrees] + ! The longitudinal length of the domain. +TOPO_CONFIG = "spoon" ! + ! This specifies how bathymetry is specified: + ! file - read bathymetric information from the file + ! specified by (TOPO_FILE). + ! flat - flat bottom set to MAXIMUM_DEPTH. + ! bowl - an analytically specified bowl-shaped basin + ! ranging between MAXIMUM_DEPTH and MINIMUM_DEPTH. + ! spoon - a similar shape to 'bowl', but with an vertical + ! wall at the southern face. + ! halfpipe - a zonally uniform channel with a half-sine + ! profile in the meridional direction. + ! benchmark - use the benchmark test case topography. + ! Neverland - use the Neverland test case topography. + ! DOME - use a slope and channel configuration for the + ! DOME sill-overflow test case. + ! ISOMIP - use a slope and channel configuration for the + ! ISOMIP test case. + ! DOME2D - use a shelf and slope configuration for the + ! DOME2D gravity current/overflow test case. + ! Kelvin - flat but with rotated land mask. + ! seamount - Gaussian bump for spontaneous motion test case. + ! dumbbell - Sloshing channel with reservoirs on both ends. + ! shelfwave - exponential slope for shelfwave test case. + ! Phillips - ACC-like idealized topography used in the Phillips config. + ! dense - Denmark Strait-like dense water formation and overflow. + ! USER - call a user modified routine. +MINIMUM_DEPTH = 1.0 ! [m] default = 0.0 + ! The minimum depth of the ocean. +MAXIMUM_DEPTH = 2000.0 ! [m] + ! The maximum depth of the ocean. + +! === module MOM_open_boundary === +! Controls where open boundaries are located, what kind of boundary condition to impose, and what data to apply, if any. + +! === module MOM_tracer_registry === + +! === module MOM_restart === + +! === module MOM_tracer_flow_control === +USE_REGIONAL_DYES = True ! [Boolean] default = False + ! If true, use the regional_dyes tracer package. + +! === module regional_dyes === +NUM_DYE_TRACERS = 3 ! default = 0 + ! The number of dye tracers in this run. Each tracer + ! should have a separate region. +DYE_SOURCE_MINLON = 3*10.0 ! [not defined] + ! This is the starting longitude at which we start injecting dyes. +DYE_SOURCE_MAXLON = 3*12.0 ! [not defined] + ! This is the ending longitude at which we finish injecting dyes. +DYE_SOURCE_MINLAT = 33.0, 39.0, 45.0 ! [not defined] + ! This is the starting latitude at which we start injecting dyes. +DYE_SOURCE_MAXLAT = 35.0, 41.0, 47.0 ! [not defined] + ! This is the ending latitude at which we finish injecting dyes. +DYE_SOURCE_MINDEPTH = 3*0.0 ! [m] + ! This is the minumum depth at which we inject dyes. +DYE_SOURCE_MAXDEPTH = 3*2000.0 ! [m] + ! This is the maximum depth at which we inject dyes. + +! === module MOM_coord_initialization === +COORD_CONFIG = "gprime" ! + ! This specifies how layers are to be defined: + ! ALE or none - used to avoid defining layers in ALE mode + ! file - read coordinate information from the file + ! specified by (COORD_FILE). + ! BFB - Custom coords for buoyancy-forced basin case + ! based on SST_S, T_BOT and DRHO_DT. + ! linear - linear based on interfaces not layers + ! layer_ref - linear based on layer densities + ! ts_ref - use reference temperature and salinity + ! ts_range - use range of temperature and salinity + ! (T_REF and S_REF) to determine surface density + ! and GINT calculate internal densities. + ! gprime - use reference density (RHO_0) for surface + ! density and GINT calculate internal densities. + ! ts_profile - use temperature and salinity profiles + ! (read from COORD_FILE) to set layer densities. + ! USER - call a user modified routine. +GFS = 0.98 ! [m s-2] default = 9.8 + ! The reduced gravity at the free surface. +GINT = 0.004 ! [m s-2] + ! The reduced gravity across internal interfaces. + +! === module MOM_grid === +! Parameters providing information about the lateral grid. + +! === module MOM_state_initialization === +THICKNESS_CONFIG = "uniform" ! + ! A string that determines how the initial layer + ! thicknesses are specified for a new run: + ! file - read interface heights from the file specified + ! thickness_file - read thicknesses from the file specified + ! by (THICKNESS_FILE). + ! coord - determined by ALE coordinate. + ! uniform - uniform thickness layers evenly distributed + ! between the surface and MAXIMUM_DEPTH. + ! list - read a list of positive interface depths. + ! DOME - use a slope and channel configuration for the + ! DOME sill-overflow test case. + ! ISOMIP - use a configuration for the + ! ISOMIP test case. + ! benchmark - use the benchmark test case thicknesses. + ! Neverland - use the Neverland test case thicknesses. + ! search - search a density profile for the interface + ! densities. This is not yet implemented. + ! circle_obcs - the circle_obcs test case is used. + ! DOME2D - 2D version of DOME initialization. + ! adjustment2d - 2D lock exchange thickness ICs. + ! sloshing - sloshing gravity thickness ICs. + ! seamount - no motion test with seamount ICs. + ! dumbbell - sloshing channel ICs. + ! soliton - Equatorial Rossby soliton. + ! rossby_front - a mixed layer front in thermal wind balance. + ! USER - call a user modified routine. + +! === module MOM_diag_mediator === + +! === module MOM_MEKE === + +! === module MOM_lateral_mixing_coeffs === + +! === module MOM_set_visc === +LINEAR_DRAG = True ! [Boolean] default = False + ! If LINEAR_DRAG and BOTTOMDRAGLAW are defined the drag + ! law is cdrag*DRAG_BG_VEL*u. +HBBL = 10.0 ! [m] + ! The thickness of a bottom boundary layer with a + ! viscosity of KVBBL if BOTTOMDRAGLAW is not defined, or + ! the thickness over which near-bottom velocities are + ! averaged for the drag law if BOTTOMDRAGLAW is defined + ! but LINEAR_DRAG is not. +DRAG_BG_VEL = 0.1 ! [m s-1] default = 0.0 + ! DRAG_BG_VEL is either the assumed bottom velocity (with + ! LINEAR_DRAG) or an unresolved velocity that is + ! combined with the resolved velocity to estimate the + ! velocity magnitude. DRAG_BG_VEL is only used when + ! BOTTOMDRAGLAW is defined. +BBL_THICK_MIN = 0.1 ! [m] default = 0.0 + ! The minimum bottom boundary layer thickness that can be + ! used with BOTTOMDRAGLAW. This might be + ! Kv / (cdrag * drag_bg_vel) to give Kv as the minimum + ! near-bottom viscosity. +KV = 1.0E-04 ! [m2 s-1] + ! The background kinematic viscosity in the interior. + ! The molecular value, ~1e-6 m2 s-1, may be used. + +! === module MOM_continuity === + +! === module MOM_continuity_PPM === +ETA_TOLERANCE = 1.0E-12 ! [m] default = 2.5E-10 + ! The tolerance for the differences between the + ! barotropic and baroclinic estimates of the sea surface + ! height due to the fluxes through each face. The total + ! tolerance for SSH is 4 times this value. The default + ! is 0.5*NK*ANGSTROM, and this should not be set less x + ! than about 10^-15*MAXIMUM_DEPTH. + +! === module MOM_CoriolisAdv === +BOUND_CORIOLIS = True ! [Boolean] default = False + ! If true, the Coriolis terms at u-points are bounded by + ! the four estimates of (f+rv)v from the four neighboring + ! v-points, and similarly at v-points. This option would + ! have no effect on the SADOURNY Coriolis scheme if it + ! were possible to use centered difference thickness fluxes. + +! === module MOM_PressureForce === + +! === module MOM_PressureForce_AFV === + +! === module MOM_hor_visc === +SMAGORINSKY_AH = True ! [Boolean] default = False + ! If true, use a biharmonic Smagorinsky nonlinear eddy + ! viscosity. +SMAG_BI_CONST = 0.06 ! [nondim] default = 0.0 + ! The nondimensional biharmonic Smagorinsky constant, + ! typically 0.015 - 0.06. + +! === module MOM_vert_friction === +DIRECT_STRESS = True ! [Boolean] default = False + ! If true, the wind stress is distributed over the + ! topmost HMIX_STRESS of fluid (like in HYCOM), and KVML + ! may be set to a very small value. +HARMONIC_VISC = True ! [Boolean] default = False + ! If true, use the harmonic mean thicknesses for + ! calculating the vertical viscosity. +HMIX_FIXED = 20.0 ! [m] + ! The prescribed depth over which the near-surface + ! viscosity and diffusivity are elevated when the bulk + ! mixed layer is not used. +KVML = 0.01 ! [m2 s-1] default = 1.0E-04 + ! The kinematic viscosity in the mixed layer. A typical + ! value is ~1e-2 m2 s-1. KVML is not used if + ! BULKMIXEDLAYER is true. The default is set by KV. +MAXVEL = 6.0 ! [m s-1] default = 3.0E+08 + ! The maximum velocity allowed before the velocity + ! components are truncated. + +! === module MOM_barotropic === +BOUND_BT_CORRECTION = True ! [Boolean] default = False + ! If true, the corrective pseudo mass-fluxes into the + ! barotropic solver are limited to values that require + ! less than maxCFL_BT_cont to be accommodated. +BT_PROJECT_VELOCITY = True ! [Boolean] default = False + ! If true, step the barotropic velocity first and project + ! out the velocity tendancy by 1+BEBT when calculating the + ! transport. The default (false) is to use a predictor + ! continuity step to find the pressure field, and then + ! to do a corrector continuity step using a weighted + ! average of the old and new velocities, with weights + ! of (1-BEBT) and BEBT. +BEBT = 0.2 ! [nondim] default = 0.1 + ! BEBT determines whether the barotropic time stepping + ! uses the forward-backward time-stepping scheme or a + ! backward Euler scheme. BEBT is valid in the range from + ! 0 (for a forward-backward treatment of nonrotating + ! gravity waves) to 1 (for a backward Euler treatment). + ! In practice, BEBT must be greater than about 0.05. +DTBT = -0.9 ! [s or nondim] default = -0.98 + ! The barotropic time step, in s. DTBT is only used with + ! the split explicit time stepping. To set the time step + ! automatically based the maximum stable value use 0, or + ! a negative value gives the fraction of the stable value. + ! Setting DTBT to 0 is the same as setting it to -0.98. + ! The value of DTBT that will actually be used is an + ! integer fraction of DT, rounding down. + +! === module MOM_thickness_diffuse === +KHTH = 0. ! [m2 s-1] default = 0.0 + ! The background horizontal thickness diffusivity. +KHTH_USE_FGNV_STREAMFUNCTION = False ! [Boolean] default = False + ! If true, use the streamfunction formulation of + ! Ferrari et al., 2010, which effectively emphasizes + ! graver vertical modes by smoothing in the vertical. + +! === module MOM_mixed_layer_restrat === + +! === module MOM_diag_to_Z === + +! === module MOM_diabatic_driver === +! The following parameters are used for diabatic processes. + +! === module MOM_tracer_advect === + +! === module MOM_tracer_hor_diff === + +! === module MOM_neutral_diffusion === +! This module implements neutral diffusion of tracers + +! === module MOM_sum_output === +DATE_STAMPED_STDOUT = False ! [Boolean] default = True + ! If true, use dates (not times) in messages to stdout +READ_DEPTH_LIST = True ! [Boolean] default = False + ! Read the depth list from a file if it exists or + ! create that file otherwise. +DEPTH_LIST_MIN_INC = 1.0E-06 ! [m] default = 1.0E-10 + ! The minimum increment between the depths of the + ! entries in the depth-list file. + +! === module MOM_surface_forcing === +VARIABLE_WINDS = False ! [Boolean] default = True + ! If true, the winds vary in time after the initialization. +VARIABLE_BUOYFORCE = False ! [Boolean] default = True + ! If true, the buoyancy forcing varies in time after the + ! initialization of the model. +BUOY_CONFIG = "zero" ! + ! The character string that indicates how buoyancy forcing + ! is specified. Valid options include (file), (zero), + ! (linear), (USER), (BFB) and (NONE). +WIND_CONFIG = "2gyre" ! + ! The character string that indicates how wind forcing + ! is specified. Valid options include (file), (2gyre), + ! (1gyre), (gyres), (zero), and (USER). + +! === module MOM_restart === + +! === module MOM_main (MOM_driver) === +DT_FORCING = 2400.0 ! [s] default = 1200.0 + ! The time step for changing forcing, coupling with other + ! components, or potentially writing certain diagnostics. + ! The default value is given by DT. +DAYMAX = 1460.0 ! [days] + ! The final time of the whole simulation, in units of + ! TIMEUNIT seconds. This also sets the potential end + ! time of the present run segment if the end time is + ! not set via ocean_solo_nml in input.nml. +RESTART_CONTROL = 3 ! default = 1 + ! An integer whose bits encode which restart files are + ! written. Add 2 (bit 1) for a time-stamped file, and odd + ! (bit 0) for a non-time-stamped file. A non-time-stamped + ! restart file is saved at the end of the run segment + ! for any non-negative value. + +! === module MOM_write_cputime === + +! === module MOM_file_parser === diff --git a/tests/test_configs/tagged_tests/correct/example_input.i b/tests/test_configs/tagged_tests/correct/example_input.i new file mode 100644 index 000000000..099485e11 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/example_input.i @@ -0,0 +1,118 @@ +[Mesh] + file = reactor.e + # Let's assign human friendly names to the blocks on the fly + block_id = '1 2' + block_name = 'fuel deflector' + + boundary_id = '4 5' + boundary_name = 'bottom top' +[] + +[Variables] + [./diffused] + order = SECOND + family = LAGRANGE + initial_condition = 0.5 # shortcut/convenience for setting constant initial condition + [../] + + [./convected] + order = FIRST + family = LAGRANGE + initial_condition = 0.0 # shortcut/convenience for setting constant initial condition + [../] +[] + +[Kernels] + # This Kernel consumes a real-gradient material property from the active material + [./convection] + type = ExampleConvection + variable = convected + [../] + + [./diff_convected] + type = Diffusion + variable = convected + [../] + + [./example_diff] + # This Kernel uses "diffusivity" from the active material + type = ExampleDiffusion + variable = diffused + [../] + + [./time_deriv_diffused] + type = TimeDerivative + variable = diffused + [../] + + [./time_deriv_convected] + type = TimeDerivative + variable = convected + [../] +[] + +[BCs] + [./bottom_diffused] + type = DirichletBC + variable = diffused + boundary = 'bottom' + value = 0 + [../] + + [./top_diffused] + type = DirichletBC + variable = diffused + boundary = 'top' + value = 5 + [../] + + [./bottom_convected] + type = DirichletBC + variable = convected + boundary = 'bottom' + value = 0 + [../] + + [./top_convected] + type = NeumannBC + variable = convected + boundary = 'top' + value = 1 + [../] +[] + +[Materials] + [./example] + type = ExampleMaterial + block = 'fuel' + diffusion_gradient = 'diffused' + + # Approximate Parabolic Diffusivity + independent_vals = '0 0.25 0.5 0.75 1.0' + dependent_vals = '1e-2 5e-3 1e-3 5e-3 1e-2' + [../] + + [./example1] + type = ExampleMaterial + block = 'deflector' + diffusion_gradient = 'diffused' + + # Constant Diffusivity + independent_vals = '0 1.0' + dependent_vals = '1e-1 1e-1' + [../] +[] + +[Executioner] + type = Transient + solve_type = 'PJFNK' + petsc_options_iname = '-pc_type -pc_hypre_type' + petsc_options_value = 'hypre boomeramg' + dt = 0.1 + num_steps = 10 +[] + +[Outputs] + execute_on = 'timestep_end' + exodus = true +[] diff --git a/tests/test_configs/tagged_tests/correct/in.airebo b/tests/test_configs/tagged_tests/correct/in.airebo new file mode 100644 index 000000000..56ff000c3 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/in.airebo @@ -0,0 +1,22 @@ +# AIREBO polyethelene benchmark + +units metal +atom_style atomic + +read_data data.airebo + +replicate 20 16 2 + +neighbor 0.5 bin +neigh_modify delay 5 every 1 + +pair_style airebo 3.0 1 1 +pair_coeff * * CH.airebo C H + +velocity all create 300.0 761341 + +fix 1 all nve +timestep 0.0005 + +thermo 10 +run 100 diff --git a/tests/test_configs/tagged_tests/correct/in.atm b/tests/test_configs/tagged_tests/correct/in.atm new file mode 100644 index 000000000..055c1986b --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/in.atm @@ -0,0 +1,31 @@ +# Axilrod-Teller-Muto potential example + +variable x index 1 +variable y index 1 +variable z index 1 + +variable xx equal 10*$x +variable yy equal 10*$y +variable zz equal 10*$z + +units lj +atom_style atomic + +lattice fcc 0.70 +region box block 0 ${xx} 0 ${yy} 0 ${zz} +create_box 1 box +create_atoms 1 box + +pair_style hybrid/overlay lj/cut 4.5 atm 4.5 2.5 +pair_coeff * * lj/cut 1.0 1.0 +pair_coeff * * atm * 0.072 + +mass * 1.0 +velocity all create 1.033 12345678 loop geom + +fix 1 all nvt temp 1.033 1.033 0.05 + +timestep 0.002 +thermo 5 + +run 25 diff --git a/tests/test_configs/tagged_tests/correct/in.crack b/tests/test_configs/tagged_tests/correct/in.crack new file mode 100644 index 000000000..150b84fc9 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/in.crack @@ -0,0 +1,78 @@ +# 2d LJ crack simulation + +dimension 2 +boundary s s p + +atom_style atomic +neighbor 0.3 bin +neigh_modify delay 5 + +# create geometry + +lattice hex 0.93 +region box block 0 100 0 40 -0.25 0.25 +create_box 5 box +create_atoms 1 box + +mass 1 1.0 +mass 2 1.0 +mass 3 1.0 +mass 4 1.0 +mass 5 1.0 + +# LJ potentials + +pair_style lj/cut 2.5 +pair_coeff * * 1.0 1.0 2.5 + +# define groups + +region 1 block INF INF INF 1.25 INF INF +group lower region 1 +region 2 block INF INF 38.75 INF INF INF +group upper region 2 +group boundary union lower upper +group mobile subtract all boundary +group leftupper region + +region leftupper block INF 20 20 INF INF INF +region leftlower block INF 20 INF 20 INF INF +group leftupper region leftupper +group leftlower region leftlower + +set group leftupper type 2 +set group leftlower type 3 +set group lower type 4 +set group upper type 5 + +# initial velocities + +compute new mobile temp +velocity mobile create 0.01 887723 temp new +velocity upper set 0.0 0.3 0.0 +velocity mobile ramp vy 0.0 0.3 y 1.25 38.75 sum yes + +# fixes + +fix 1 all nve +fix 2 boundary setforce NULL 0.0 0.0 + +# run + +timestep 0.003 +thermo 200 +thermo_modify temp new + +neigh_modify exclude type 2 3 + +#dump 1 all atom 500 dump.crack + +#dump 2 all image 250 image.*.jpg type type & +# zoom 1.6 adiam 1.5 +#dump_modify 2 pad 4 + +#dump 3 all movie 250 movie.mpg type type & +# zoom 1.6 adiam 1.5 +#dump_modify 3 pad 4 + +run 5000 diff --git a/tests/test_configs/tagged_tests/correct/in.ellipse.gayberne b/tests/test_configs/tagged_tests/correct/in.ellipse.gayberne new file mode 100644 index 000000000..fe783ac6d --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/in.ellipse.gayberne @@ -0,0 +1,66 @@ +# GayBerne ellipsoids in LJ background fluid + +units lj +atom_style ellipsoid +dimension 2 + +lattice sq 0.02 +region box block 0 20 0 20 -0.5 0.5 +create_box 2 box +create_atoms 1 box + +set group all type/fraction 2 0.1 95392 +set type 1 mass 1.0 +set type 2 mass 1.5 +set type 1 shape 1 1 1 +set type 2 shape 3 1 1 +set group all quat/random 18238 + +compute rot all temp/asphere +group spheroid type 1 +variable dof equal count(spheroid)+2 +compute_modify rot extra/dof ${dof} + +velocity all create 2.4 87287 loop geom + +pair_style gayberne 1.0 3.0 1.0 4.0 +pair_coeff 1 1 3.0 1.0 1 1 1 1 1 1 2.5 +pair_coeff 1 2 3.0 1.0 1 1 1 0 0 0 +pair_coeff 2 2 1.0 1.0 1 1 0.2 0 0 0 + +neighbor 0.8 bin + +thermo_style custom step c_rot epair etotal press vol +thermo 100 + +timestep 0.002 + +compute q all property/atom quatw quati quatj quatk + +#dump 1 all custom 100 dump.ellipse.gayberne & +# id type x y z c_q[1] c_q[2] c_q[3] c_q[4] + +#dump 2 all image 100 image.*.jpg type type & +# zoom 1.6 center d 0.5 0.5 0.5 +#dump_modify 2 pad 4 adiam 1 1.0 adiam 2 2.0 + +#dump 3 all movie 100 movie.mpg type type & +# zoom 1.6 center d 0.5 0.5 0.5 +#dump_modify 3 pad 4 adiam 1 1.0 adiam 2 2.0 + +fix 1 all npt/asphere temp 2.0 2.0 0.1 iso 0.0 1.0 1.0 & + mtk no pchain 0 tchain 1 +fix 2 all enforce2d + +compute_modify 1_temp extra/dof ${dof} + +# equilibrate to shrink box around dilute system + +run 2000 + +# run dynamics on dense system + +unfix 1 +fix 1 all nve/asphere + +run 2000 diff --git a/tests/test_configs/tagged_tests/correct/input-file.inp b/tests/test_configs/tagged_tests/correct/input-file.inp new file mode 100644 index 000000000..f135ecec1 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/input-file.inp @@ -0,0 +1,245 @@ +&FORCE_EVAL + METHOD Quickstep + &DFT + &QS + METHOD PM6 + EPS_DEFAULT 1.0E-10 + &SE + ANALYTICAL_GRADIENTS T + &COULOMB + CUTOFF [angstrom] 12.0 + RC_RANGE [bohr] 1.0 + &END + &EXCHANGE + CUTOFF [angstrom] 4.9325 + RC_TAPER [angstrom] 12.0 + RC_RANGE [bohr] 1.0 + &END + &END + &END QS + &SCF + &PRINT + &RESTART OFF + &END + &RESTART_HISTORY OFF + &END + &END + SCF_GUESS MOPAC + EPS_SCF 1.0E-5 + MAX_SCF 45 + &OT + PRECONDITIONER FULL_SINGLE_INVERSE + MINIMIZER DIIS + N_DIIS 9 + &END + &OUTER_SCF + MAX_SCF 8 + EPS_SCF 1.0E-5 + &END + &END SCF + &XC + &XC_FUNCTIONAL PBE + &END XC_FUNCTIONAL + &END XC + &END DFT + &SUBSYS + &CELL + PERIODIC NONE + ABC 16.0000 16.0000000 16.0000 + &END CELL + &COLVAR + &REACTION_PATH + LAMBDA 5.0 + STEP_SIZE 0.01 + RANGE -2.0 37.0 + VARIABLE T + FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T + &MAP + RANGE 2 12 + RANGE 2 12 + GRID_SPACING 0.5 + GRID_SPACING 0.5 + &END + &COLVAR + &DISTANCE + ATOMS 26 32 + &END + &END + FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T + &COLVAR + &DISTANCE + ATOMS 26 30 + &END + &END + &END + &END + &COLVAR + &DISTANCE_FROM_PATH + LAMBDA 15.0 + STEP_SIZE 0.01 + RANGE -2.0 37.0 + VARIABLE T + FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T + &MAP + RANGE 2 12 + RANGE 2 12 + GRID_SPACING 0.5 + GRID_SPACING 0.5 + &END + &COLVAR + &DISTANCE + ATOMS 26 32 + &END + &END + FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T + &COLVAR + &DISTANCE + ATOMS 26 30 + &END + &END + &END + &END + &COORD + O -4.4247387624 -0.5309572750 0.5697230862 + H -5.0183448733 -2.6944585590 1.3203736786 + C -3.6109170457 -1.3741775205 0.8680427212 + O -4.0237717839 -2.6214903530 1.2703514467 + C -2.1306370189 -1.2791406640 0.8535019283 + C -1.3120456446 -2.3978600881 1.1275197410 + H -1.7701653707 -3.3666124305 1.3548030036 + C 0.0688545253 -2.2437189267 1.1156801180 + H 0.7574075334 -3.0761272058 1.3486991675 + N 0.6738559410 -1.0371313768 0.8361136511 + C -0.1389885304 0.0574156107 0.5692682480 + C -1.5454296273 -0.0457404425 0.5761779393 + H -2.1704627231 0.8321461706 0.3801763635 + C 0.5913111673 1.2918018616 0.2805097546 + C -0.0299604321 2.4779270180 -0.1639273497 + H -1.1228367654 2.5372055901 -0.2444243606 + C 0.7663086277 3.5666879578 -0.5045075502 + C 0.1920272309 4.8496216484 -0.9668879613 + O -1.1836917368 4.8023669936 -0.9803620784 + O 0.7656701641 5.8524002130 -1.3220490954 + C 2.1762122923 3.4640311228 -0.4070263504 + H 2.8012745440 4.3073721391 -0.7188446821 + C 2.7340753851 2.2923512744 0.0769505212 + H 3.8188537022 2.1726101379 0.1961014861 + N 1.9703552650 1.1968447154 0.4447247525 + Ru 2.6808139034 -0.6268135255 1.0491515077 + N 3.2563830464 -2.5532275954 1.5027135434 + C 3.4490476757 -3.6669628938 1.8370067213 + S 3.7025486662 -5.2038677765 2.2394078258 + N 2.1955375549 -0.3650270770 3.0173819208 + C 1.6049936707 0.3098036981 3.7855301635 + S 0.8289774632 1.2452756100 4.8086474146 + N 3.3853640961 -0.9960228944 -0.8720138955 + C 2.6610011074 -1.5613188486 -1.8986641089 + H 1.5915143243 -1.7170221188 -1.6810559568 + C 3.2453301524 -1.9340252113 -3.1029040105 + H 2.6589980673 -2.4156892719 -3.8913029786 + C 4.6278759505 -1.7104040677 -3.2859858733 + C 5.3108087788 -2.0657932775 -4.5535542391 + O 6.4687902929 -1.8960108056 -4.8530811803 + H 10.0999394091 1.8836409660 2.4997876783 + O 4.4387949377 -2.6355643612 -5.4502144227 + C 5.3859781105 -1.1418894308 -2.2656271240 + H 6.4625500813 -0.9854422725 -2.3976463282 + C 4.7478203106 -0.7953695233 -1.0558453820 + C 5.4376024832 -0.2248315007 0.1002998824 + C 6.7974932647 0.1530791814 0.1052796969 + H 7.4226310419 0.0133029862 -0.7846209826 + C 7.3399526057 0.7040767770 1.2625106413 + C 8.7635758427 1.1048269295 1.2588255996 + O 9.5655997786 1.0448892034 0.3558301760 + O 9.1376353505 1.6165130959 2.4790228433 + C 6.5282331717 0.8715673047 2.4115537092 + H 6.9561129325 1.2952014265 3.3275244791 + C 5.2007233179 0.4774776840 2.3595740688 + H 4.5207036878 0.5633558687 3.2285882081 + N 4.6296913260 -0.0725021202 1.2253520131 + H 4.8874702054 -2.8845712381 -6.3088218514 + H -1.5910323092 5.6705894857 -1.2601791678 + &END COORD + &END SUBSYS +&END FORCE_EVAL +&GLOBAL + PROJECT N3_rp_dp_md + RUN_TYPE MD + PRINT_LEVEL LOW +&END GLOBAL +&MOTION + &FREE_ENERGY + &METADYN + DO_HILLS T + LAGRANGE + NT_HILLS 10 + WW 1.0e-4 + TEMPERATURE 100. + TEMP_TOL 50 + &METAVAR + COLVAR 1 + LAMBDA 0.5 + MASS 10.0 + SCALE 0.03 + &END + &METAVAR + COLVAR 2 + LAMBDA 4.5 + MASS 2.0 + SCALE 0.03 + &END + &PRINT + &COLVAR + COMMON_ITERATION_LEVELS 3 + &EACH + MD 1 + &END + &END + &HILLS + COMMON_ITERATION_LEVELS 3 + &EACH + MD 1 + &END + &END + &END + &END METADYN + &END FREE_ENERGY + &MD + ENSEMBLE NVT + STEPS 10 + TIMESTEP 1 + TEMPERATURE 300.0 + TEMP_TOL 50 + &THERMOSTAT + TYPE NOSE + &NOSE + LENGTH 6 + YOSHIDA 3 + TIMECON 1500 + MTS 2 + &END NOSE + &END THERMOSTAT + &PRINT + &ENERGY + &EACH + MD 10 + &END + ADD_LAST NUMERIC + &END + &END PRINT + &END MD + &PRINT + &TRAJECTORY + &EACH + MD 10 + &END + &END + &RESTART + &EACH + MD 100 + &END + &END + &RESTART_HISTORY OFF + &END + &END +&END MOTION diff --git a/tests/test_configs/tagged_tests/correct/input.nml b/tests/test_configs/tagged_tests/correct/input.nml new file mode 100644 index 000000000..12b131307 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/input.nml @@ -0,0 +1,22 @@ + &MOM_input_nml + output_directory = './', + input_filename = 'n' + restart_input_dir = 'INPUT/', + restart_output_dir = 'RESTART/', + parameter_filename = 'MOM_input', + 'MOM_override' / +&fms_nml + domains_stack_size = 552960 + / + + &diag_manager_nml + / + + &ocean_solo_nml + months = 120 + days = 0 + date_init = 1,1,1,0,0,0 + hours = 0 + minutes = 0 + seconds = 0 + calendar = 'noleap' / diff --git a/tests/test_configs/tagged_tests/correct/simple-H20.xml b/tests/test_configs/tagged_tests/correct/simple-H20.xml new file mode 100644 index 000000000..120843be5 --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/simple-H20.xml @@ -0,0 +1,86 @@ + + + + + Simple Example of moleculear H2O + + + + + + -1 + + 2.9151687332e-01 -6.5123272502e-01 -1.2188463918e-01 + 5.8423636048e-01 4.2730406357e-01 -4.5964306231e-03 + 3.5228575807e-01 -3.5027014639e-01 5.2644808295e-01 + -5.1686250912e-01 -1.6648002292e+00 6.5837023441e-01 + + + + -1 + + 3.1443445436e-01 6.5068682609e-01 -4.0983449009e-02 + -3.8686061749e-01 -9.3744432997e-02 -6.0456005388e-01 + 2.4978241724e-02 -3.2862514649e-02 -7.2266047173e-01 + -4.0352404772e-01 1.1927734805e+00 5.5610824921e-01 + + + + + + 6 + 4 + 8 + + + 1 + 1 + 1 + + + 0.0000000000e+00 0.0000000000e+00 0.0000000000e+00 + 0.0000000000e+00 -1.4308249289e+00 1.1078707576e+00 + 0.0000000000e+00 1.4308249289e+00 1.1078707576e+00 + + + O H H + + + + + + + + + + + + + + + + + + + + 1 + 64 + 1 + 5 + 100 + 1 + 0.3 + no + + + + 128 + no + 100 + 0.005 + 10 + 200 + yes + + + diff --git a/tests/test_configs/tagged_tests/marked/MOM_input b/tests/test_configs/tagged_tests/marked/MOM_input new file mode 100644 index 000000000..27b9336b6 --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/MOM_input @@ -0,0 +1,363 @@ +! This file was written by the model and records the non-default parameters used at run-time. + +! === module MOM === + +! === module MOM_unit_scaling === +! Parameters for doing unit scaling of variables. +ENABLE_THERMODYNAMICS = False ! [Boolean] default = True + ! If true, Temperature and salinity are used as state + ! variables. +ADIABATIC = True ! [Boolean] default = False + ! There are no diapycnal mass fluxes if ADIABATIC is + ! true. This assumes that KD = KDML = 0.0 and that + ! there is no buoyancy forcing, but makes the model + ! faster by eliminating subroutine calls. +THICKNESSDIFFUSE = True ! [Boolean] default = False + ! If true, interface heights are diffused with a + ! coefficient of KHTH. +DT = 1200.0 ! [s] + ! The (baroclinic) dynamics time step. The time-step that + ! is actually used will be an integer fraction of the + ! forcing time-step (DT_FORCING in ocean-only mode or the + ! coupling timestep in coupled mode.) +DTBT_RESET_PERIOD = -1.0 ! [s] default = 1200.0 + ! The period between recalculations of DTBT (if DTBT <= 0). + ! If DTBT_RESET_PERIOD is negative, DTBT is set based + ! only on information available at initialization. If 0, + ! DTBT will be set every dynamics time step. The default + ! is set by DT_THERM. This is only used if SPLIT is true. + +! === module MOM_domains === +REENTRANT_X = False ! [Boolean] default = True + ! If true, the domain is zonally reentrant. +NIGLOBAL = 80 ! + ! The total number of thickness grid points in the + ! x-direction in the physical domain. With STATIC_MEMORY_ + ! this is set in MOM_memory.h at compile time. +NJGLOBAL = 40 ! + ! The total number of thickness grid points in the + ! y-direction in the physical domain. With STATIC_MEMORY_ + ! this is set in MOM_memory.h at compile time. + +! === module MOM_hor_index === +! Sets the horizontal array index types. + +! === module MOM_verticalGrid === +! Parameters providing information about the vertical grid. +NK = ;5; ! [nondim] + ! The number of model layers. + +! === module MOM_fixed_initialization === +INPUTDIR = "INPUT" ! default = "." + ! The directory in which input files are found. + +! === module MOM_grid_init === +GRID_CONFIG = "spherical" ! + ! A character string that determines the method for + ! defining the horizontal grid. Current options are: + ! mosaic - read the grid from a mosaic (supergrid) + ! file set by GRID_FILE. + ! cartesian - use a (flat) Cartesian grid. + ! spherical - use a simple spherical grid. + ! mercator - use a Mercator spherical grid. +SOUTHLAT = 30.0 ! [degrees] + ! The southern latitude of the domain. +LENLAT = 20.0 ! [degrees] + ! The latitudinal length of the domain. +LENLON = 40.0 ! [degrees] + ! The longitudinal length of the domain. +TOPO_CONFIG = "spoon" ! + ! This specifies how bathymetry is specified: + ! file - read bathymetric information from the file + ! specified by (TOPO_FILE). + ! flat - flat bottom set to MAXIMUM_DEPTH. + ! bowl - an analytically specified bowl-shaped basin + ! ranging between MAXIMUM_DEPTH and MINIMUM_DEPTH. + ! spoon - a similar shape to 'bowl', but with an vertical + ! wall at the southern face. + ! halfpipe - a zonally uniform channel with a half-sine + ! profile in the meridional direction. + ! benchmark - use the benchmark test case topography. + ! Neverland - use the Neverland test case topography. + ! DOME - use a slope and channel configuration for the + ! DOME sill-overflow test case. + ! ISOMIP - use a slope and channel configuration for the + ! ISOMIP test case. + ! DOME2D - use a shelf and slope configuration for the + ! DOME2D gravity current/overflow test case. + ! Kelvin - flat but with rotated land mask. + ! seamount - Gaussian bump for spontaneous motion test case. + ! dumbbell - Sloshing channel with reservoirs on both ends. + ! shelfwave - exponential slope for shelfwave test case. + ! Phillips - ACC-like idealized topography used in the Phillips config. + ! dense - Denmark Strait-like dense water formation and overflow. + ! USER - call a user modified routine. +MINIMUM_DEPTH = 1.0 ! [m] default = 0.0 + ! The minimum depth of the ocean. +MAXIMUM_DEPTH = 2000.0 ! [m] + ! The maximum depth of the ocean. + +! === module MOM_open_boundary === +! Controls where open boundaries are located, what kind of boundary condition to impose, and what data to apply, if any. + +! === module MOM_tracer_registry === + +! === module MOM_restart === + +! === module MOM_tracer_flow_control === +USE_REGIONAL_DYES = True ! [Boolean] default = False + ! If true, use the regional_dyes tracer package. + +! === module regional_dyes === +NUM_DYE_TRACERS = 3 ! default = 0 + ! The number of dye tracers in this run. Each tracer + ! should have a separate region. +DYE_SOURCE_MINLON = 3*10.0 ! [not defined] + ! This is the starting longitude at which we start injecting dyes. +DYE_SOURCE_MAXLON = 3*12.0 ! [not defined] + ! This is the ending longitude at which we finish injecting dyes. +DYE_SOURCE_MINLAT = 33.0, 39.0, 45.0 ! [not defined] + ! This is the starting latitude at which we start injecting dyes. +DYE_SOURCE_MAXLAT = 35.0, 41.0, 47.0 ! [not defined] + ! This is the ending latitude at which we finish injecting dyes. +DYE_SOURCE_MINDEPTH = 3*0.0 ! [m] + ! This is the minumum depth at which we inject dyes. +DYE_SOURCE_MAXDEPTH = 3*2000.0 ! [m] + ! This is the maximum depth at which we inject dyes. + +! === module MOM_coord_initialization === +COORD_CONFIG = "gprime" ! + ! This specifies how layers are to be defined: + ! ALE or none - used to avoid defining layers in ALE mode + ! file - read coordinate information from the file + ! specified by (COORD_FILE). + ! BFB - Custom coords for buoyancy-forced basin case + ! based on SST_S, T_BOT and DRHO_DT. + ! linear - linear based on interfaces not layers + ! layer_ref - linear based on layer densities + ! ts_ref - use reference temperature and salinity + ! ts_range - use range of temperature and salinity + ! (T_REF and S_REF) to determine surface density + ! and GINT calculate internal densities. + ! gprime - use reference density (RHO_0) for surface + ! density and GINT calculate internal densities. + ! ts_profile - use temperature and salinity profiles + ! (read from COORD_FILE) to set layer densities. + ! USER - call a user modified routine. +GFS = 0.98 ! [m s-2] default = 9.8 + ! The reduced gravity at the free surface. +GINT = 0.004 ! [m s-2] + ! The reduced gravity across internal interfaces. + +! === module MOM_grid === +! Parameters providing information about the lateral grid. + +! === module MOM_state_initialization === +THICKNESS_CONFIG = "uniform" ! + ! A string that determines how the initial layer + ! thicknesses are specified for a new run: + ! file - read interface heights from the file specified + ! thickness_file - read thicknesses from the file specified + ! by (THICKNESS_FILE). + ! coord - determined by ALE coordinate. + ! uniform - uniform thickness layers evenly distributed + ! between the surface and MAXIMUM_DEPTH. + ! list - read a list of positive interface depths. + ! DOME - use a slope and channel configuration for the + ! DOME sill-overflow test case. + ! ISOMIP - use a configuration for the + ! ISOMIP test case. + ! benchmark - use the benchmark test case thicknesses. + ! Neverland - use the Neverland test case thicknesses. + ! search - search a density profile for the interface + ! densities. This is not yet implemented. + ! circle_obcs - the circle_obcs test case is used. + ! DOME2D - 2D version of DOME initialization. + ! adjustment2d - 2D lock exchange thickness ICs. + ! sloshing - sloshing gravity thickness ICs. + ! seamount - no motion test with seamount ICs. + ! dumbbell - sloshing channel ICs. + ! soliton - Equatorial Rossby soliton. + ! rossby_front - a mixed layer front in thermal wind balance. + ! USER - call a user modified routine. + +! === module MOM_diag_mediator === + +! === module MOM_MEKE === + +! === module MOM_lateral_mixing_coeffs === + +! === module MOM_set_visc === +LINEAR_DRAG = True ! [Boolean] default = False + ! If LINEAR_DRAG and BOTTOMDRAGLAW are defined the drag + ! law is cdrag*DRAG_BG_VEL*u. +HBBL = 10.0 ! [m] + ! The thickness of a bottom boundary layer with a + ! viscosity of KVBBL if BOTTOMDRAGLAW is not defined, or + ! the thickness over which near-bottom velocities are + ! averaged for the drag law if BOTTOMDRAGLAW is defined + ! but LINEAR_DRAG is not. +DRAG_BG_VEL = 0.1 ! [m s-1] default = 0.0 + ! DRAG_BG_VEL is either the assumed bottom velocity (with + ! LINEAR_DRAG) or an unresolved velocity that is + ! combined with the resolved velocity to estimate the + ! velocity magnitude. DRAG_BG_VEL is only used when + ! BOTTOMDRAGLAW is defined. +BBL_THICK_MIN = 0.1 ! [m] default = 0.0 + ! The minimum bottom boundary layer thickness that can be + ! used with BOTTOMDRAGLAW. This might be + ! Kv / (cdrag * drag_bg_vel) to give Kv as the minimum + ! near-bottom viscosity. +KV = 1.0E-04 ! [m2 s-1] + ! The background kinematic viscosity in the interior. + ! The molecular value, ~1e-6 m2 s-1, may be used. + +! === module MOM_continuity === + +! === module MOM_continuity_PPM === +ETA_TOLERANCE = 1.0E-12 ! [m] default = 2.5E-10 + ! The tolerance for the differences between the + ! barotropic and baroclinic estimates of the sea surface + ! height due to the fluxes through each face. The total + ! tolerance for SSH is 4 times this value. The default + ! is 0.5*NK*ANGSTROM, and this should not be set less x + ! than about 10^-15*MAXIMUM_DEPTH. + +! === module MOM_CoriolisAdv === +BOUND_CORIOLIS = True ! [Boolean] default = False + ! If true, the Coriolis terms at u-points are bounded by + ! the four estimates of (f+rv)v from the four neighboring + ! v-points, and similarly at v-points. This option would + ! have no effect on the SADOURNY Coriolis scheme if it + ! were possible to use centered difference thickness fluxes. + +! === module MOM_PressureForce === + +! === module MOM_PressureForce_AFV === + +! === module MOM_hor_visc === +SMAGORINSKY_AH = True ! [Boolean] default = False + ! If true, use a biharmonic Smagorinsky nonlinear eddy + ! viscosity. +SMAG_BI_CONST = 0.06 ! [nondim] default = 0.0 + ! The nondimensional biharmonic Smagorinsky constant, + ! typically 0.015 - 0.06. + +! === module MOM_vert_friction === +DIRECT_STRESS = True ! [Boolean] default = False + ! If true, the wind stress is distributed over the + ! topmost HMIX_STRESS of fluid (like in HYCOM), and KVML + ! may be set to a very small value. +HARMONIC_VISC = True ! [Boolean] default = False + ! If true, use the harmonic mean thicknesses for + ! calculating the vertical viscosity. +HMIX_FIXED = 20.0 ! [m] + ! The prescribed depth over which the near-surface + ! viscosity and diffusivity are elevated when the bulk + ! mixed layer is not used. +KVML = 0.01 ! [m2 s-1] default = 1.0E-04 + ! The kinematic viscosity in the mixed layer. A typical + ! value is ~1e-2 m2 s-1. KVML is not used if + ! BULKMIXEDLAYER is true. The default is set by KV. +MAXVEL = 6.0 ! [m s-1] default = 3.0E+08 + ! The maximum velocity allowed before the velocity + ! components are truncated. + +! === module MOM_barotropic === +BOUND_BT_CORRECTION = True ! [Boolean] default = False + ! If true, the corrective pseudo mass-fluxes into the + ! barotropic solver are limited to values that require + ! less than maxCFL_BT_cont to be accommodated. +BT_PROJECT_VELOCITY = True ! [Boolean] default = False + ! If true, step the barotropic velocity first and project + ! out the velocity tendancy by 1+BEBT when calculating the + ! transport. The default (false) is to use a predictor + ! continuity step to find the pressure field, and then + ! to do a corrector continuity step using a weighted + ! average of the old and new velocities, with weights + ! of (1-BEBT) and BEBT. +BEBT = 0.2 ! [nondim] default = 0.1 + ! BEBT determines whether the barotropic time stepping + ! uses the forward-backward time-stepping scheme or a + ! backward Euler scheme. BEBT is valid in the range from + ! 0 (for a forward-backward treatment of nonrotating + ! gravity waves) to 1 (for a backward Euler treatment). + ! In practice, BEBT must be greater than about 0.05. +DTBT = -0.9 ! [s or nondim] default = -0.98 + ! The barotropic time step, in s. DTBT is only used with + ! the split explicit time stepping. To set the time step + ! automatically based the maximum stable value use 0, or + ! a negative value gives the fraction of the stable value. + ! Setting DTBT to 0 is the same as setting it to -0.98. + ! The value of DTBT that will actually be used is an + ! integer fraction of DT, rounding down. + +! === module MOM_thickness_diffuse === +KHTH = 0. ! [m2 s-1] default = 0.0 + ! The background horizontal thickness diffusivity. +KHTH_USE_FGNV_STREAMFUNCTION = False ! [Boolean] default = False + ! If true, use the streamfunction formulation of + ! Ferrari et al., 2010, which effectively emphasizes + ! graver vertical modes by smoothing in the vertical. + +! === module MOM_mixed_layer_restrat === + +! === module MOM_diag_to_Z === + +! === module MOM_diabatic_driver === +! The following parameters are used for diabatic processes. + +! === module MOM_tracer_advect === + +! === module MOM_tracer_hor_diff === + +! === module MOM_neutral_diffusion === +! This module implements neutral diffusion of tracers + +! === module MOM_sum_output === +DATE_STAMPED_STDOUT = False ! [Boolean] default = True + ! If true, use dates (not times) in messages to stdout +READ_DEPTH_LIST = True ! [Boolean] default = False + ! Read the depth list from a file if it exists or + ! create that file otherwise. +DEPTH_LIST_MIN_INC = 1.0E-06 ! [m] default = 1.0E-10 + ! The minimum increment between the depths of the + ! entries in the depth-list file. + +! === module MOM_surface_forcing === +VARIABLE_WINDS = False ! [Boolean] default = True + ! If true, the winds vary in time after the initialization. +VARIABLE_BUOYFORCE = False ! [Boolean] default = True + ! If true, the buoyancy forcing varies in time after the + ! initialization of the model. +BUOY_CONFIG = "zero" ! + ! The character string that indicates how buoyancy forcing + ! is specified. Valid options include (file), (zero), + ! (linear), (USER), (BFB) and (NONE). +WIND_CONFIG = "2gyre" ! + ! The character string that indicates how wind forcing + ! is specified. Valid options include (file), (2gyre), + ! (1gyre), (gyres), (zero), and (USER). + +! === module MOM_restart === + +! === module MOM_main (MOM_driver) === +DT_FORCING = 2400.0 ! [s] default = 1200.0 + ! The time step for changing forcing, coupling with other + ! components, or potentially writing certain diagnostics. + ! The default value is given by DT. +DAYMAX = 1460.0 ! [days] + ! The final time of the whole simulation, in units of + ! TIMEUNIT seconds. This also sets the potential end + ! time of the present run segment if the end time is + ! not set via ocean_solo_nml in input.nml. +RESTART_CONTROL = 3 ! default = 1 + ! An integer whose bits encode which restart files are + ! written. Add 2 (bit 1) for a time-stamped file, and odd + ! (bit 0) for a non-time-stamped file. A non-time-stamped + ! restart file is saved at the end of the run segment + ! for any non-negative value. + +! === module MOM_write_cputime === + +! === module MOM_file_parser === diff --git a/tests/test_configs/tagged_tests/marked/example_input.i b/tests/test_configs/tagged_tests/marked/example_input.i new file mode 100644 index 000000000..97157a2f3 --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/example_input.i @@ -0,0 +1,118 @@ +[Mesh] + file = reactor.e + # Let's assign human friendly names to the blocks on the fly + block_id = '1 2' + block_name = 'fuel deflector' + + boundary_id = '4 5' + boundary_name = 'bottom top' +[] + +[Variables] + [./diffused] + order = ;FIRST; + family = LAGRANGE + initial_condition = 0.5 # shortcut/convenience for setting constant initial condition + [../] + + [./convected] + order = FIRST + family = LAGRANGE + initial_condition = 0.0 # shortcut/convenience for setting constant initial condition + [../] +[] + +[Kernels] + # This Kernel consumes a real-gradient material property from the active material + [./convection] + type = ExampleConvection + variable = convected + [../] + + [./diff_convected] + type = Diffusion + variable = convected + [../] + + [./example_diff] + # This Kernel uses "diffusivity" from the active material + type = ExampleDiffusion + variable = diffused + [../] + + [./time_deriv_diffused] + type = TimeDerivative + variable = diffused + [../] + + [./time_deriv_convected] + type = TimeDerivative + variable = convected + [../] +[] + +[BCs] + [./bottom_diffused] + type = DirichletBC + variable = diffused + boundary = 'bottom' + value = 0 + [../] + + [./top_diffused] + type = DirichletBC + variable = diffused + boundary = 'top' + value = 5 + [../] + + [./bottom_convected] + type = DirichletBC + variable = convected + boundary = 'bottom' + value = 0 + [../] + + [./top_convected] + type = NeumannBC + variable = convected + boundary = 'top' + value = 1 + [../] +[] + +[Materials] + [./example] + type = ExampleMaterial + block = 'fuel' + diffusion_gradient = 'diffused' + + # Approximate Parabolic Diffusivity + independent_vals = '0 0.25 0.5 0.75 1.0' + dependent_vals = '1e-2 5e-3 1e-3 5e-3 1e-2' + [../] + + [./example1] + type = ExampleMaterial + block = 'deflector' + diffusion_gradient = 'diffused' + + # Constant Diffusivity + independent_vals = '0 1.0' + dependent_vals = '1e-1 1e-1' + [../] +[] + +[Executioner] + type = Transient + solve_type = 'PJFNK' + petsc_options_iname = '-pc_type -pc_hypre_type' + petsc_options_value = 'hypre boomeramg' + dt = 0.1 + num_steps = 10 +[] + +[Outputs] + execute_on = 'timestep_end' + exodus = true +[] diff --git a/tests/test_configs/tagged_tests/marked/in.airebo b/tests/test_configs/tagged_tests/marked/in.airebo new file mode 100644 index 000000000..7523a8e77 --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/in.airebo @@ -0,0 +1,22 @@ +# AIREBO polyethelene benchmark + +units metal +atom_style atomic + +read_data data.airebo + +replicate ;17; 16 2 + +neighbor 0.5 bin +neigh_modify delay 5 every 1 + +pair_style airebo 3.0 1 1 +pair_coeff * * CH.airebo C H + +velocity all create 300.0 761341 + +fix 1 all nve +timestep 0.0005 + +thermo 10 +run 100 diff --git a/tests/test_configs/tagged_tests/marked/in.atm b/tests/test_configs/tagged_tests/marked/in.atm new file mode 100644 index 000000000..c162fa7ee --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/in.atm @@ -0,0 +1,31 @@ +# Axilrod-Teller-Muto potential example + +variable x index 1 +variable y index 1 +variable z index 1 + +variable xx equal 10*$x +variable yy equal 10*$y +variable zz equal 10*$z + +units lj +atom_style atomic + +lattice fcc 0.;65; +region box block 0 ${xx} 0 ${yy} 0 ${zz} +create_box 1 box +create_atoms 1 box + +pair_style hybrid/overlay lj/cut 4.5 atm 4.5 2.5 +pair_coeff * * lj/cut 1.0 1.0 +pair_coeff * * atm * 0.072 + +mass * 1.0 +velocity all create 1.033 12345678 loop geom + +fix 1 all nvt temp 1.033 1.033 0.05 + +timestep 0.002 +thermo 5 + +run 25 diff --git a/tests/test_configs/tagged_tests/marked/in.crack b/tests/test_configs/tagged_tests/marked/in.crack new file mode 100644 index 000000000..e9e70240d --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/in.crack @@ -0,0 +1,78 @@ +# 2d LJ crack simulation + +dimension 2 +boundary s s p + +atom_style atomic +neighbor 0.3 bin +neigh_modify delay 5 + +# create geometry + +lattice hex 0.93 +region box block 0 100 0 40 -0.25 0.25 +create_box 5 box +create_atoms 1 box + +mass 1 1.0 +mass 2 1.0 +mass 3 1.0 +mass 4 1.0 +mass 5 1.0 + +# LJ potentials + +pair_style lj/cut 2.5 +pair_coeff * * 1.0 1.0 2.5 + +# define groups + +region 1 block INF INF INF 1.25 INF INF +group lower region 1 +region 2 block INF INF 38.75 INF INF INF +group upper region 2 +group boundary union lower upper +group mobile subtract all boundary +;placeholder; + +region leftupper block INF 20 20 INF INF INF +region leftlower block INF 20 INF 20 INF INF +group leftupper region leftupper +group leftlower region leftlower + +set group leftupper type 2 +set group leftlower type 3 +set group lower type 4 +set group upper type 5 + +# initial velocities + +compute new mobile temp +velocity mobile create 0.01 887723 temp new +velocity upper set 0.0 0.3 0.0 +velocity mobile ramp vy 0.0 0.3 y 1.25 38.75 sum yes + +# fixes + +fix 1 all nve +fix 2 boundary setforce NULL 0.0 0.0 + +# run + +timestep 0.003 +thermo 200 +thermo_modify temp new + +neigh_modify exclude type 2 3 + +#dump 1 all atom 500 dump.crack + +#dump 2 all image 250 image.*.jpg type type & +# zoom 1.6 adiam 1.5 +#dump_modify 2 pad 4 + +#dump 3 all movie 250 movie.mpg type type & +# zoom 1.6 adiam 1.5 +#dump_modify 3 pad 4 + +run 5000 diff --git a/tests/test_configs/tagged_tests/marked/in.ellipse.gayberne b/tests/test_configs/tagged_tests/marked/in.ellipse.gayberne new file mode 100644 index 000000000..fe783ac6d --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/in.ellipse.gayberne @@ -0,0 +1,66 @@ +# GayBerne ellipsoids in LJ background fluid + +units lj +atom_style ellipsoid +dimension 2 + +lattice sq 0.02 +region box block 0 20 0 20 -0.5 0.5 +create_box 2 box +create_atoms 1 box + +set group all type/fraction 2 0.1 95392 +set type 1 mass 1.0 +set type 2 mass 1.5 +set type 1 shape 1 1 1 +set type 2 shape 3 1 1 +set group all quat/random 18238 + +compute rot all temp/asphere +group spheroid type 1 +variable dof equal count(spheroid)+2 +compute_modify rot extra/dof ${dof} + +velocity all create 2.4 87287 loop geom + +pair_style gayberne 1.0 3.0 1.0 4.0 +pair_coeff 1 1 3.0 1.0 1 1 1 1 1 1 2.5 +pair_coeff 1 2 3.0 1.0 1 1 1 0 0 0 +pair_coeff 2 2 1.0 1.0 1 1 0.2 0 0 0 + +neighbor 0.8 bin + +thermo_style custom step c_rot epair etotal press vol +thermo 100 + +timestep 0.002 + +compute q all property/atom quatw quati quatj quatk + +#dump 1 all custom 100 dump.ellipse.gayberne & +# id type x y z c_q[1] c_q[2] c_q[3] c_q[4] + +#dump 2 all image 100 image.*.jpg type type & +# zoom 1.6 center d 0.5 0.5 0.5 +#dump_modify 2 pad 4 adiam 1 1.0 adiam 2 2.0 + +#dump 3 all movie 100 movie.mpg type type & +# zoom 1.6 center d 0.5 0.5 0.5 +#dump_modify 3 pad 4 adiam 1 1.0 adiam 2 2.0 + +fix 1 all npt/asphere temp 2.0 2.0 0.1 iso 0.0 1.0 1.0 & + mtk no pchain 0 tchain 1 +fix 2 all enforce2d + +compute_modify 1_temp extra/dof ${dof} + +# equilibrate to shrink box around dilute system + +run 2000 + +# run dynamics on dense system + +unfix 1 +fix 1 all nve/asphere + +run 2000 diff --git a/tests/test_configs/tagged_tests/marked/input-file.inp b/tests/test_configs/tagged_tests/marked/input-file.inp new file mode 100644 index 000000000..f135ecec1 --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/input-file.inp @@ -0,0 +1,245 @@ +&FORCE_EVAL + METHOD Quickstep + &DFT + &QS + METHOD PM6 + EPS_DEFAULT 1.0E-10 + &SE + ANALYTICAL_GRADIENTS T + &COULOMB + CUTOFF [angstrom] 12.0 + RC_RANGE [bohr] 1.0 + &END + &EXCHANGE + CUTOFF [angstrom] 4.9325 + RC_TAPER [angstrom] 12.0 + RC_RANGE [bohr] 1.0 + &END + &END + &END QS + &SCF + &PRINT + &RESTART OFF + &END + &RESTART_HISTORY OFF + &END + &END + SCF_GUESS MOPAC + EPS_SCF 1.0E-5 + MAX_SCF 45 + &OT + PRECONDITIONER FULL_SINGLE_INVERSE + MINIMIZER DIIS + N_DIIS 9 + &END + &OUTER_SCF + MAX_SCF 8 + EPS_SCF 1.0E-5 + &END + &END SCF + &XC + &XC_FUNCTIONAL PBE + &END XC_FUNCTIONAL + &END XC + &END DFT + &SUBSYS + &CELL + PERIODIC NONE + ABC 16.0000 16.0000000 16.0000 + &END CELL + &COLVAR + &REACTION_PATH + LAMBDA 5.0 + STEP_SIZE 0.01 + RANGE -2.0 37.0 + VARIABLE T + FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T + &MAP + RANGE 2 12 + RANGE 2 12 + GRID_SPACING 0.5 + GRID_SPACING 0.5 + &END + &COLVAR + &DISTANCE + ATOMS 26 32 + &END + &END + FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T + &COLVAR + &DISTANCE + ATOMS 26 30 + &END + &END + &END + &END + &COLVAR + &DISTANCE_FROM_PATH + LAMBDA 15.0 + STEP_SIZE 0.01 + RANGE -2.0 37.0 + VARIABLE T + FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T + &MAP + RANGE 2 12 + RANGE 2 12 + GRID_SPACING 0.5 + GRID_SPACING 0.5 + &END + &COLVAR + &DISTANCE + ATOMS 26 32 + &END + &END + FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T + &COLVAR + &DISTANCE + ATOMS 26 30 + &END + &END + &END + &END + &COORD + O -4.4247387624 -0.5309572750 0.5697230862 + H -5.0183448733 -2.6944585590 1.3203736786 + C -3.6109170457 -1.3741775205 0.8680427212 + O -4.0237717839 -2.6214903530 1.2703514467 + C -2.1306370189 -1.2791406640 0.8535019283 + C -1.3120456446 -2.3978600881 1.1275197410 + H -1.7701653707 -3.3666124305 1.3548030036 + C 0.0688545253 -2.2437189267 1.1156801180 + H 0.7574075334 -3.0761272058 1.3486991675 + N 0.6738559410 -1.0371313768 0.8361136511 + C -0.1389885304 0.0574156107 0.5692682480 + C -1.5454296273 -0.0457404425 0.5761779393 + H -2.1704627231 0.8321461706 0.3801763635 + C 0.5913111673 1.2918018616 0.2805097546 + C -0.0299604321 2.4779270180 -0.1639273497 + H -1.1228367654 2.5372055901 -0.2444243606 + C 0.7663086277 3.5666879578 -0.5045075502 + C 0.1920272309 4.8496216484 -0.9668879613 + O -1.1836917368 4.8023669936 -0.9803620784 + O 0.7656701641 5.8524002130 -1.3220490954 + C 2.1762122923 3.4640311228 -0.4070263504 + H 2.8012745440 4.3073721391 -0.7188446821 + C 2.7340753851 2.2923512744 0.0769505212 + H 3.8188537022 2.1726101379 0.1961014861 + N 1.9703552650 1.1968447154 0.4447247525 + Ru 2.6808139034 -0.6268135255 1.0491515077 + N 3.2563830464 -2.5532275954 1.5027135434 + C 3.4490476757 -3.6669628938 1.8370067213 + S 3.7025486662 -5.2038677765 2.2394078258 + N 2.1955375549 -0.3650270770 3.0173819208 + C 1.6049936707 0.3098036981 3.7855301635 + S 0.8289774632 1.2452756100 4.8086474146 + N 3.3853640961 -0.9960228944 -0.8720138955 + C 2.6610011074 -1.5613188486 -1.8986641089 + H 1.5915143243 -1.7170221188 -1.6810559568 + C 3.2453301524 -1.9340252113 -3.1029040105 + H 2.6589980673 -2.4156892719 -3.8913029786 + C 4.6278759505 -1.7104040677 -3.2859858733 + C 5.3108087788 -2.0657932775 -4.5535542391 + O 6.4687902929 -1.8960108056 -4.8530811803 + H 10.0999394091 1.8836409660 2.4997876783 + O 4.4387949377 -2.6355643612 -5.4502144227 + C 5.3859781105 -1.1418894308 -2.2656271240 + H 6.4625500813 -0.9854422725 -2.3976463282 + C 4.7478203106 -0.7953695233 -1.0558453820 + C 5.4376024832 -0.2248315007 0.1002998824 + C 6.7974932647 0.1530791814 0.1052796969 + H 7.4226310419 0.0133029862 -0.7846209826 + C 7.3399526057 0.7040767770 1.2625106413 + C 8.7635758427 1.1048269295 1.2588255996 + O 9.5655997786 1.0448892034 0.3558301760 + O 9.1376353505 1.6165130959 2.4790228433 + C 6.5282331717 0.8715673047 2.4115537092 + H 6.9561129325 1.2952014265 3.3275244791 + C 5.2007233179 0.4774776840 2.3595740688 + H 4.5207036878 0.5633558687 3.2285882081 + N 4.6296913260 -0.0725021202 1.2253520131 + H 4.8874702054 -2.8845712381 -6.3088218514 + H -1.5910323092 5.6705894857 -1.2601791678 + &END COORD + &END SUBSYS +&END FORCE_EVAL +&GLOBAL + PROJECT N3_rp_dp_md + RUN_TYPE MD + PRINT_LEVEL LOW +&END GLOBAL +&MOTION + &FREE_ENERGY + &METADYN + DO_HILLS T + LAGRANGE + NT_HILLS 10 + WW 1.0e-4 + TEMPERATURE 100. + TEMP_TOL 50 + &METAVAR + COLVAR 1 + LAMBDA 0.5 + MASS 10.0 + SCALE 0.03 + &END + &METAVAR + COLVAR 2 + LAMBDA 4.5 + MASS 2.0 + SCALE 0.03 + &END + &PRINT + &COLVAR + COMMON_ITERATION_LEVELS 3 + &EACH + MD 1 + &END + &END + &HILLS + COMMON_ITERATION_LEVELS 3 + &EACH + MD 1 + &END + &END + &END + &END METADYN + &END FREE_ENERGY + &MD + ENSEMBLE NVT + STEPS 10 + TIMESTEP 1 + TEMPERATURE 300.0 + TEMP_TOL 50 + &THERMOSTAT + TYPE NOSE + &NOSE + LENGTH 6 + YOSHIDA 3 + TIMECON 1500 + MTS 2 + &END NOSE + &END THERMOSTAT + &PRINT + &ENERGY + &EACH + MD 10 + &END + ADD_LAST NUMERIC + &END + &END PRINT + &END MD + &PRINT + &TRAJECTORY + &EACH + MD 10 + &END + &END + &RESTART + &EACH + MD 100 + &END + &END + &RESTART_HISTORY OFF + &END + &END +&END MOTION diff --git a/tests/test_configs/tagged_tests/marked/input.nml b/tests/test_configs/tagged_tests/marked/input.nml new file mode 100644 index 000000000..4ab606fd8 --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/input.nml @@ -0,0 +1,22 @@ + &MOM_input_nml + output_directory = './', + input_filename = 'n' + restart_input_dir = 'INPUT/', + restart_output_dir = 'RESTART/', + parameter_filename = 'MOM_input', + 'MOM_override' / +&fms_nml + domains_stack_size = 552960 + / + + &diag_manager_nml + / + + &ocean_solo_nml + months = ;1200; + days = 0 + date_init = 1,1,1,0,0,0 + hours = 0 + minutes = 0 + seconds = 0 + calendar = 'noleap' / diff --git a/tests/test_configs/tagged_tests/marked/simple-H20.xml b/tests/test_configs/tagged_tests/marked/simple-H20.xml new file mode 100644 index 000000000..120843be5 --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/simple-H20.xml @@ -0,0 +1,86 @@ + + + + + Simple Example of moleculear H2O + + + + + + -1 + + 2.9151687332e-01 -6.5123272502e-01 -1.2188463918e-01 + 5.8423636048e-01 4.2730406357e-01 -4.5964306231e-03 + 3.5228575807e-01 -3.5027014639e-01 5.2644808295e-01 + -5.1686250912e-01 -1.6648002292e+00 6.5837023441e-01 + + + + -1 + + 3.1443445436e-01 6.5068682609e-01 -4.0983449009e-02 + -3.8686061749e-01 -9.3744432997e-02 -6.0456005388e-01 + 2.4978241724e-02 -3.2862514649e-02 -7.2266047173e-01 + -4.0352404772e-01 1.1927734805e+00 5.5610824921e-01 + + + + + + 6 + 4 + 8 + + + 1 + 1 + 1 + + + 0.0000000000e+00 0.0000000000e+00 0.0000000000e+00 + 0.0000000000e+00 -1.4308249289e+00 1.1078707576e+00 + 0.0000000000e+00 1.4308249289e+00 1.1078707576e+00 + + + O H H + + + + + + + + + + + + + + + + + + + + 1 + 64 + 1 + 5 + 100 + 1 + 0.3 + no + + + + 128 + no + 100 + 0.005 + 10 + 200 + yes + + + From 53b05292f4541879466c20df88aa7432d3ca31d8 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 24 Jun 2024 19:07:08 -0500 Subject: [PATCH 02/17] addressing amanda comments, bug fixes --- smartsim/_core/generation/commandgenerator.py | 135 ++++++--- tests/test_command_generation.py | 267 ++++++++++++++---- 2 files changed, 304 insertions(+), 98 deletions(-) diff --git a/smartsim/_core/generation/commandgenerator.py b/smartsim/_core/generation/commandgenerator.py index 6870f07f9..d8260ee73 100644 --- a/smartsim/_core/generation/commandgenerator.py +++ b/smartsim/_core/generation/commandgenerator.py @@ -24,71 +24,128 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# TODO: rename and make all the descriptions accurate and good -# including the params and stuff +import typing as t +from os import path + + +def move_op(to_move: str, dest_file: str): + """Write a python script to moves a file + + :param to_move: Path to a source file to be copied + :param dest_file: Path to a file to copy the contents from the source file into + :return: string of python code to perform operation that can be executed + """ + return f"import shutil;shutil.move('{to_move}','{dest_file}')" + def delete_op(to_delete: str): - """Produce string of a script that deletes a file when executed. + """Write a python script that deletes a file when executed. - :param to_delete: path to the file to be deleted - :return: string to be shipped as a command that deletes the given file + :param to_delete: Path to the file to be deleted + :return: string of python code to perform operation that can be executed """ - return f"import os;os.remove('{to_delete}')" + if path.isfile(to_delete): + return rf""" +import os +os.remove('{to_delete}') + """ + else: + raise FileNotFoundError(f"{to_delete} is not a valid path") -def copy_op(source_file: str, dest_file: str): - """return command that can be popened""" - return f"import shutil;shutil.copyfile('{source_file}', '{dest_file}')" +def copy_op(to_copy: str, entity_path: str): + """ + Write a python script to copy the entity files and directories attached + to this entity into an entity directory + + :param to_copy: Path to directory, or path to file to copy into an entity directory + :param entity_path: Path to a directory of an entity + :return: string of python code to perform operation that can be executed + """ + + if not path.exists(entity_path): + raise FileNotFoundError(f"{entity_path} is not a valid path") + elif not path.exists(to_copy): + raise FileNotFoundError(f"{to_copy} is not a valid path") + + else: + + return rf"""import shutil +from distutils import dir_util +from os import path +dst_path = path.join('{entity_path}', path.basename('{to_copy}')) + + +if path.isdir('{to_copy}'): + dir_util.copy_tree('{to_copy}', '{entity_path}') +else: + shutil.copyfile('{to_copy}', dst_path) +""" -def symlink_op(dest_file: str, source_file: str): - """ - Make the source path a symlink pointing to the destination path. - :param source_file: - :param dest_file: - :return: +def symlink_op(to_link: str, entity_path: str): + """ + Write a python script to make the to_link path a symlink pointing to the entity path. + + :param to_link: path to link + :param entity_path: + :return: string of python code to perform operation that can be executed + """ + return rf"""import os +from os import path +dest_path = path.join('{entity_path}', path.basename('{to_link}')) + +os.symlink('{to_link}', dest_path) """ - return f"import os;os.symlink(str('{source_file}'), str('{dest_file}'))" -def move_op(source_file, dest_file): - """Execute a script that moves a file""" - return f"import shutil;shutil.move('{source_file}','{dest_file}')" +def configure_op( + to_configure: str, params: t.Dict[str, str], tag_delimiter: t.Optional[str] = None +): + """Write a python script to set, search and replace the tagged parameters for the configure operation + within tagged files attached to an entity. -def configure_op(file_path, param_dict): - """configure file with params - TODO: clean this up""" + :param to configure: + :param params: A dict of parameter names and values set for the file + :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" + :return: string of python code to perform operation that can be executed + """ - return fr""" -import re + return rf"""import re import collections -edited = [] -used_params = {{}} -files_to_tags = {{}} -params = {param_dict} -regex = "(;.+;)" -tag = ";" - def _get_prev_value(tagged_line: str) -> str: - split_tag = tagged_line.split(tag) + split_tag = tagged_line.split(tag_delimiter) return split_tag[1] def _is_ensemble_spec( tagged_line: str, model_params: dict ) -> bool: - split_tag = tagged_line.split(tag) + split_tag = tagged_line.split(tag_delimiter) prev_val = split_tag[1] if prev_val in model_params.keys(): return True return False -with open('{file_path}','r+', encoding='utf-8') as file_stream: +edited = [] +used_params = {{}} +params = {params} + +tag_delimiter = '{tag_delimiter}' + +if tag_delimiter == 'False': + tag_delimiter = ';' + +regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) + +# Set the lines to iterate over +with open('{to_configure}','r+', encoding='utf-8') as file_stream: lines = file_stream.readlines() unused_tags = collections.defaultdict(list) +# read lines in file for i, line in enumerate(lines, 1): while search := re.search(regex, line): tagged_line = search.group(0) @@ -98,18 +155,18 @@ def _is_ensemble_spec( line = re.sub(regex, new_val, line, 1) used_params[previous_value] = new_val - # if a tag is found but is not in this model's configurations + # if a tag_delimiter is found but is not in this model's configurations # put in placeholder value else: - tag = tagged_line.split(tag)[1] - unused_tags[tag].append(i) + tag_delimiter_ = tagged_line.split(tag_delimiter)[1] + unused_tags[tag_delimiter_].append(i) line = re.sub(regex, previous_value, line) break edited.append(line) lines = edited -with open('{file_path}', "w+", encoding="utf-8") as file_stream: +with open('{to_configure}', "w+", encoding="utf-8") as file_stream: for line in lines: file_stream.write(line) -""" + """ diff --git a/tests/test_command_generation.py b/tests/test_command_generation.py index 0680d8895..933858480 100644 --- a/tests/test_command_generation.py +++ b/tests/test_command_generation.py @@ -25,99 +25,199 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import filecmp -from glob import glob import os import pathlib import subprocess import sys import typing as t +from distutils import dir_util +from glob import glob from os import path as osp -from distutils import dir_util +import pytest from smartsim._core.generation import commandgenerator test_path = os.path.dirname(os.path.abspath(__file__)) - -@staticmethod -def make_test_file( - file_name: str, file_dir: str, file_content: t.Optional[str] = None -) -> str: - """Create a dummy file in the test output directory. - - :param file_name: name of file to create, e.g. "file.txt" - :param file_dir: path - :return: String path to test output file - """ - file_path = os.path.join(file_dir, file_name) - os.makedirs(file_dir) - with open(file_path, "w+", encoding="utf-8") as dummy_file: - if not file_content: - dummy_file.write("dummy\n") - else: - dummy_file.write(file_content) - return file_path +test_output_root = os.path.join(test_path, "tests", "test_output") -def test_symlink(): +def test_symlink_entity(): """ - Test symlink. Make path a symlink pointing to the given path + Test the execution of the python script created by symlink_op """ - # Prep test directory - test_output_root = os.path.join(test_path, "tests", "test_output") - target = pathlib.Path(test_output_root) / "sym_target_dir" / "target" + source = pathlib.Path(test_output_root) / "sym_source" + os.mkdir(source) - # Build the command - cmd = commandgenerator.symlink_op(source, target) + # entity_path to be the dest dir + entity_path = os.path.join(test_output_root, "entity_name") + os.mkdir(entity_path) - # execute the command + target = pathlib.Path(test_output_root) / "entity_name" / "sym_source" + + # # Build the command + cmd = commandgenerator.symlink_op(source, entity_path) + # # execute the command subprocess.run([sys.executable, "-c", cmd]) - # Assert the two files are the same file - assert source.is_symlink() - assert os.readlink(source) == str(target) + # # Assert the two files are the same file + assert target.is_symlink() + assert os.readlink(target) == str(source) # Clean up the test directory - os.unlink(source) + os.unlink(target) + os.rmdir(pathlib.Path(test_output_root) / "entity_name") + os.rmdir(pathlib.Path(test_output_root) / "sym_source") -def test_copy(): - """Copy the content of the source file to the destination file.""" +def test_copy_op_file(): + """Test the execution of the python script created by copy_op + Test the operation to copy the content of the source file to the destination path + with an empty file of the same name already in the directory""" - test_output_root = os.path.join(test_path, "tests", "test_output") + to_copy = os.path.join(test_output_root, "to_copy") + os.mkdir(to_copy) + + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + entity_path = os.path.join(test_output_root, "entity_name") + os.mkdir(entity_path) + + dest_file = os.path.join(test_output_root, "entity_name", "copy_file.txt") + with open(dest_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("") + + # Execute copy + cmd = commandgenerator.copy_op(source_file, entity_path) + subprocess.run([sys.executable, "-c", cmd]) + + # clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.rmdir(pathlib.Path(test_output_root) / "to_copy") + + os.remove(pathlib.Path(entity_path) / "copy_file.txt") + os.rmdir(pathlib.Path(test_output_root) / "entity_name") + + +def test_copy_op_dirs(): + """Test the execution of the python script created by copy_op + Test the oeprations that copies an entire directory tree source to a new location destination + """ + + to_copy = os.path.join(test_output_root, "to_copy") + os.mkdir(to_copy) + + # write some test files in the dir + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + + source_file_2 = pathlib.Path(to_copy) / "copy_file_2.txt" + with open(source_file_2, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy2") + + # entity_path to be the dest dir + entity_path = os.path.join(test_output_root, "entity_name") + os.mkdir(entity_path) + # copy those files? + + cmd = commandgenerator.copy_op(to_copy, entity_path) + subprocess.run([sys.executable, "-c", cmd]) + + # Clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.remove(pathlib.Path(to_copy) / "copy_file_2.txt") + os.rmdir(pathlib.Path(test_output_root) / "to_copy") + os.remove(pathlib.Path(entity_path) / "copy_file.txt") + os.remove(pathlib.Path(entity_path) / "copy_file_2.txt") + os.rmdir(pathlib.Path(test_output_root) / "entity_name") + + +def test_copy_op(fileutils): + """Test the execution of the python script created by copy_op + Test the operation to copy the content of the source file to the destination file. + """ # make a test file with some contents - source_file = make_test_file( - "source.txt", pathlib.Path(test_output_root) / "source", "dummy" + to_copy_file = fileutils.make_test_file( + "to_copy.txt", pathlib.Path(test_output_root) / "to_copy", "dummy" ) - dest_file = make_test_file("dest.txt", pathlib.Path(test_output_root) / "dest", "") + + entity_path = os.path.join(test_output_root, "entity_name") + + os.mkdir(entity_path) # assert that source file exists, has correct contents - assert osp.exists(source_file) - with open(source_file, "r", encoding="utf-8") as dummy_file: + assert osp.exists(to_copy_file) + with open(to_copy_file, "r", encoding="utf-8") as dummy_file: assert dummy_file.read() == "dummy" - cmd = commandgenerator.copy_op(source_file, dest_file) + cmd = commandgenerator.copy_op(to_copy_file, entity_path) subprocess.run([sys.executable, "-c", cmd]) - with open(dest_file, "r", encoding="utf-8") as dummy_file: - assert dummy_file.read() == "dummy" + entity_file = os.path.join(test_output_root, "entity_name", "to_copy.txt") + # asser that the entity_path now has the source file with the correct contents + with open(entity_file, "r", encoding="utf-8") as dummy_file: + assert "dummy" in dummy_file.read() # Clean up the test directory - os.remove(source_file) - os.remove(dest_file) - os.rmdir(pathlib.Path(test_output_root) / "source") - os.rmdir(pathlib.Path(test_output_root) / "dest") + os.remove(to_copy_file) + os.remove(entity_file) + os.rmdir(pathlib.Path(test_output_root) / "to_copy") + os.rmdir(pathlib.Path(test_output_root) / "entity_name") -def test_move(): - """Test to move command execution""" - test_output_root = os.path.join(test_path, "tests", "test_output") +def test_copy_op_bad_source_file(): + """Test the execution of the python script created by copy_op + Test that a FileNotFoundError is raised when there is a bad source file + """ + + to_copy = os.path.join(test_output_root, "to_copy") + os.mkdir(to_copy) + entity_path = os.path.join(test_output_root, "entity_name") + os.mkdir(entity_path) + + # Execute copy + with pytest.raises(FileNotFoundError) as ex: + commandgenerator.copy_op("/not/a/real/path", entity_path) + assert "is not a valid path" in ex.value.args[0] + # clean up + os.rmdir(pathlib.Path(test_output_root) / "to_copy") + os.rmdir(pathlib.Path(test_output_root) / "entity_name") + + +def test_copy_op_bad_dest_path(): + """Test the execution of the python script created by copy_op. + Test that a FileNotFoundError is raised when there is a bad destination file.""" + + to_copy = os.path.join(test_output_root, "to_copy") + os.mkdir(to_copy) + + source_file = pathlib.Path(to_copy) / "copy_file.txt" + entity_path = os.path.join(test_output_root, "entity_name") + os.mkdir(entity_path) + + with pytest.raises(FileNotFoundError) as ex: + commandgenerator.copy_op(source_file, "/not/a/real/path") + assert "is not a valid path" in ex.value.args[0] + + # clean up + os.rmdir(pathlib.Path(test_output_root) / "to_copy") + os.rmdir(pathlib.Path(test_output_root) / "entity_name") + + +def test_move_op(): + """Test the execution of the python script created by move_op. + Test the operation to move a file""" + source_dir = os.path.join(test_output_root, "from_here") os.mkdir(source_dir) dest_dir = os.path.join(test_output_root, "to_here") os.mkdir(dest_dir) + dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") source_file = pathlib.Path(source_dir) / "app_move.txt" @@ -128,11 +228,10 @@ def test_move(): with open(source_file, "r", encoding="utf-8") as dummy_file: assert dummy_file.read() == "dummy" - cmd = commandgenerator.move_op(source_file, dest_file) subprocess.run([sys.executable, "-c", cmd]) - # assert that the move was successful + # Assert that the move was successful assert not osp.exists(source_file) assert osp.exists(dest_file) with open(dest_file, "r", encoding="utf-8") as dummy_file: @@ -144,9 +243,9 @@ def test_move(): os.rmdir(dest_dir) -def test_delete(): - """Test python inline command to delete a file""" - test_output_root = os.path.join(test_path, "tests", "test_output") +def test_delete_op(): + """Test the execution of the python script created by delete_op. + Test the operation to delete a file""" # Make a test file with dummy text to_del = pathlib.Path(test_output_root) / "app_del.txt" @@ -164,8 +263,58 @@ def test_delete(): assert not osp.exists(to_del) -def test_configure(test_dir, fileutils): - """ test configure param operations """ +def test_delete_op_bad_path(): + """Test the execution of the python script created by delete_op. + Test that FileNotFoundError is raised when a bad path is given to the + soperation to delete a file""" + + test_output_root = os.path.join(test_path, "tests", "test_output") + to_del = pathlib.Path(test_output_root) / "not_real.txt" + + with pytest.raises(FileNotFoundError) as ex: + commandgenerator.delete_op(to_del) + assert "is not a valid path" in ex.value.args[0] + + +def test_configure_op(test_dir, fileutils): + """Test the execution of the python script created by configure_op. + Test configure param operations with a tag parameter given""" + + # the param dict for configure operations + param_dict = { + "5": 10, # MOM_input + "FIRST": "SECOND", # example_input.i + "17": 20, # in.airebo + "65": "70", # in.atm + "placeholder": "group leftupper region", # in.crack + "1200": "120", # input.nml + } + tag = ";" + # retreive tagged files + conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) + # retrieve files to compare after test + correct_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "correct/")) + + # copy files to test directory + dir_util.copy_tree(conf_path, test_dir) + assert osp.isdir(test_dir) + + tagged_files = sorted(glob(test_dir + "/*")) + correct_files = sorted(glob(correct_path + "*")) + + # Run configure op on test files + for tagged_file in tagged_files: + cmd = commandgenerator.configure_op(tagged_file, param_dict, tag) + subprocess.run([sys.executable, "-c", cmd]) + + # check that files and correct files are the same + for written, correct in zip(tagged_files, correct_files): + assert filecmp.cmp(written, correct) + + +def test_configure_op_no_tag(test_dir, fileutils): + """Test the execution of the python script created by configure_op. + Test configure param operations with no tag parameter given""" # the param dict for configure operations param_dict = { @@ -190,7 +339,7 @@ def test_configure(test_dir, fileutils): # Run configure op on test files for tagged_file in tagged_files: - cmd = commandgenerator.configure_op(tagged_file, param_dict) + cmd = commandgenerator.configure_op(tagged_file, param_dict, False) subprocess.run([sys.executable, "-c", cmd]) # check that files and correct files are the same From 7f32004dd3a7d7ca422faeea032cbeba7dc59e5d Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Wed, 3 Jul 2024 15:05:50 -0500 Subject: [PATCH 03/17] WIP --- .../_core/entrypoints/command_generator.py | 395 ++++++++++++++++++ .../_core/entrypoints/telemetrymonitor.py | 3 +- smartsim/_core/generation/commandgenerator.py | 232 +++++++--- tests/test_command_generation.py | 370 ++++++++++++---- tests/test_telemetry_monitor.py | 1 + 5 files changed, 857 insertions(+), 144 deletions(-) create mode 100644 smartsim/_core/entrypoints/command_generator.py diff --git a/smartsim/_core/entrypoints/command_generator.py b/smartsim/_core/entrypoints/command_generator.py new file mode 100644 index 000000000..a480a65f3 --- /dev/null +++ b/smartsim/_core/entrypoints/command_generator.py @@ -0,0 +1,395 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024 Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import os +import typing as t +import shutil +from distutils import dir_util +import re +import collections + + +def get_dst_path(application_path, input_file): + return os.path.join(application_path, os.path.basename(input_file)) + + +def _check_path(file_path: str) -> str: + """Given a user provided path-like str, find the actual path to + the directory or file and create a full path. + + :param file_path: path to a specific file or directory + :raises FileNotFoundError: if file or directory does not exist + :return: full path to file or directory + """ + full_path = os.path.abspath(file_path) + if os.path.isfile(full_path): + return full_path + if os.path.isdir(full_path): + return full_path + raise FileNotFoundError(f"File or Directory {file_path} not found") + + + +def move_op(to_move: str, dst_path: str): + """Move a file + + :param to_move: Path to a source file to be copied + :param dst_path: Path to a file to copy the contents from the source file into + :return: string of python code to perform operation that can be executed + """ + + shutil.move(to_move, dst_path) + + +def remove_op(to_delete: str): + """Write a python script that removes a file when executed. + + :param to_delete: Path to the file to be deleted + :return: string of python code to perform operation that can be executed + """ + _check_path(to_delete) + + os.remove(to_delete) + + +# TODO: remove the entity path things and keep it as a helper funtion that +# gives the user to add the application, and then it would take the applciation path and do the copy + + +def copy_op(source_path: str, dest_path: str): + """ + Write a python script to copy the entity files and directories attached + to this entity into an entity directory + + :param source_path: Path to directory, or path to file to copy into an entity directory + :param dest_path: Path to destination directory or path to destination file to copy + :return: string of python code to perform operation that can be executed + """ + + #_check_path(to_copy) + + if os.path.isdir(source_path): + dir_util.copy_tree(source_path, dest_path) + else: + shutil.copyfile(source_path, dest_path) + + + +def symlink_op(source: str, link: str): + """ + Create a symbolic link pointing to the exisiting source file + named link + + :param src_file: the exisiting source path + :param dst_file: target name where the symlink will be created. + """ + + _check_path(source) + + os.symlink(source, link) + + + +def configure_op( + to_configure: str, dst_path: t.Optional[str], param_dict: t.Dict[str, str], tag_delimiter: str = ';' +): + """Write a python script to set, search and replace the tagged parameters for the configure operation + within tagged files attached to an entity. + + User-formatted files can be attached using the `to_configure` argument. These files will be modified +during ``Model`` generation to replace tagged sections in the user-formatted files with +values from the `params` initializer argument used during ``Model`` creation: + + :param to_configure: The tagged files the search and replace operations to be performed upon + :param dest: Optional destination for configured files to be written to + :param param_dict: A dict of parameter names and values set for the file + :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" + :return: string of python code to perform operation that can be executed + """ + + _check_path(to_configure) + _check_path(dst_path) + + def _get_prev_value(tagged_line: str) -> str: + split_tag = tagged_line.split(tag_delimiter) + return split_tag[1] + + def _is_ensemble_spec( + tagged_line: str, model_params: dict + ) -> bool: + split_tag = tagged_line.split(tag_delimiter) + prev_val = split_tag[1] + if prev_val in model_params.keys(): + return True + return False + + edited = [] + used_params = {} + #param_dict = {param_dict} + + # tag_delimiter = tag_delimiter + + # Set the tag for the modelwriter to search for within + # tagged files attached to an entity. + regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) + + # Set the lines to iterate over + with open(to_configure,'r+', encoding='utf-8') as file_stream: + lines = file_stream.readlines() + + unused_tags = collections.defaultdict(list) + + # Replace the tagged parameters within the file attached to this + # model. The tag defaults to ";" + for i, line in enumerate(lines, 1): + while search := re.search(regex, line): + tagged_line = search.group(0) + previous_value = _get_prev_value(tagged_line) + if _is_ensemble_spec(tagged_line, params): + new_val = str(params[previous_value]) + line = re.sub(regex, new_val, line, 1) + used_params[previous_value] = new_val + + # if a tag_delimiter is found but is not in this model's configurations + # put in placeholder value + else: + tag_delimiter_ = tagged_line.split(tag_delimiter)[1] + unused_tags[tag_delimiter_].append(i) + line = re.sub(regex, previous_value, line) + break + edited.append(line) + + lines = edited + + # write configured file to destination specified. Default is an overwrite + if dst_path: + file_stream = dst_path + + with open(to_configure, "w+", encoding="utf-8") as file_stream: + for line in lines: + file_stream.write(line) + + +# python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path +# python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path +# python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path +# python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path +# python _core/entrypoints/file_operations.py configure /absolte/file/src/path /absolte/file/dest/path tagged_deliminator params # base64 encoded dictionary for params + +import json +def get_parser() -> argparse.ArgumentParser: + """Instantiate a parser to process command line arguments + + :returns: An argument parser ready to accept required command generator parameters + """ + arg_parser = argparse.ArgumentParser(description="Command Generator") + + + + subparsers = arg_parser.add_subparsers(help='file_operations') + + # subparser for move op + move_parser = subparsers.add_parser("move") + move_parser.add_argument("src_path") + move_parser.add_argument("dest_path") + + #usage: pytest move [-h] operation src_path dest_path + + # subparser for remove op + remove_parser = subparsers.add_parser("remove") + remove_parser.add_argument( + "to_remove", + type = str) + + # subparser for copy op + copy_parser = subparsers.add_parser("copy") + copy_parser.add_argument( + "source_path", + type = str) + copy_parser.add_argument( + "dest_path", + type = str) + # subparser for symlink op + symlink_parser = subparsers.add_parser("symlink") + symlink_parser.add_argument( + "source_path", + type = str) + symlink_parser.add_argument( + "dest_path", + type = str) + + # subparser for configure op + configure_parser = subparsers.add_parser("configure") + configure_parser.add_argument( + "source_path", + type = str) + configure_parser.add_argument( + "dest_path", + type = str) + configure_parser.add_argument( + "tag_delimiter", + type = str) + configure_parser.add_argument( + "param_dict", + type = str) + +# parser.add_argument('-m', '--my-dict', type=str) +# args = parser.parse_args() + +# import json +# my_dictionary = json.loads(args.my_dict) + + + + + + # arg_parser.add_argument( + # "move", + # type=str, + # help="move files", + # # required=True, + # default="" + # ) + # arg_parser.add_argument( + # "remove", + # type=str, + # help="delete files", + # #required=True, + # default="" + # ) + # arg_parser.add_argument( + # "symlink", + # type=str, + # help="symlink a file", + # #required=True, + # default="" + # ) + # arg_parser.add_argument( + # "copy", + # type=str, + # help="copy a file or files from one destination to another", + # #required=True, + # default="" + # ) + # arg_parser.add_argument( + # "configure", + # type=str, + # help="configure a file with tagged parameters", + # #required=True, + # default="" + + # arg_parser.add_argument( + # "-exp_dir", + # type=str, + # help="Experiment root directory", + # required=True, + # ) + # arg_parser.add_argument( + # "-frequency", + # type=float, + # help="Frequency of telemetry updates (in seconds))", + # required=True, + # ) + # arg_parser.add_argument( + # "-cooldown", + # type=int, + # help="Default lifetime of telemetry monitor (in seconds) before auto-shutdown", + # default=cfg.CONFIG.telemetry_cooldown, + # ) + # arg_parser.add_argument( + # "-loglevel", + # type=int, + # help="Logging level", + # default=logging.INFO, + # ) + return arg_parser + + +def parse_arguments():# -> TelemetryMonitorArgs: + """Parse the command line arguments and return an instance + of TelemetryMonitorArgs populated with the CLI inputs + + :returns: `TelemetryMonitorArgs` instance populated with command line arguments + """ + parser = get_parser() + parsed_args = parser.parse_args() + print(parsed_args) + # return TelemetryMonitorArgs( + # parsed_args.exp_dir, + # parsed_args.frequency, + # parsed_args.cooldown, + # parsed_args.loglevel, + # ) + + +if __name__ == "__main__": + """Prepare the telemetry monitor process using command line arguments. + + Sample usage: + python -m smartsim._core.entrypoints.telemetrymonitor -exp_dir + -frequency 30 -cooldown 90 -loglevel INFO + The experiment id is generated during experiment startup + and can be found in the manifest.json in /.smartsim/telemetry + """ + os.environ["PYTHONUNBUFFERED"] = "1" + + args = parse_arguments() #JPNOTE get from here - pos args, first one in, the rest will get fed into the rest of the functions + #sys args? - some number of strings that come after + #configure_logger(logger, args.log_level, args.exp_dir) + + print("IN MAIN HERE") + + if args[1] == 'remove': + + to_remove = args[2] + remove_op(to_remove) + + if args[1] == 'move': + to_move = args [2] + entity_path = args[3] + move_op(to_move, entity_path) + + if args[1] == 'symlink': + + symlink_op() + + if args[1] == 'copy': + copy_op() + + if args[1] == 'configure': + configure_op() + + + import json + my_dictionary = json.loads(args.my_dict) + + + + + + + # sys.exit(1) # do I need? diff --git a/smartsim/_core/entrypoints/telemetrymonitor.py b/smartsim/_core/entrypoints/telemetrymonitor.py index 5ed1a0c91..95cf8b4d7 100644 --- a/smartsim/_core/entrypoints/telemetrymonitor.py +++ b/smartsim/_core/entrypoints/telemetrymonitor.py @@ -148,7 +148,8 @@ def configure_logger(logger_: logging.Logger, log_level_: int, exp_dir: str) -> """ os.environ["PYTHONUNBUFFERED"] = "1" - args = parse_arguments() + args = parse_arguments() #JPNOTE get from here - pos args, first one in, the rest will get fed into the rest of the functions + #sys args? - some number of strings that come after configure_logger(logger, args.log_level, args.exp_dir) telemetry_monitor = TelemetryMonitor(args) diff --git a/smartsim/_core/generation/commandgenerator.py b/smartsim/_core/generation/commandgenerator.py index d8260ee73..2bf2acf7e 100644 --- a/smartsim/_core/generation/commandgenerator.py +++ b/smartsim/_core/generation/commandgenerator.py @@ -28,6 +28,31 @@ from os import path +# ## + +# # module -- named function things? +# # open file locally, +# # open up module and run that module + +# - each of these going + +# one single entry point file +# with a main function with an arg parser (takes in at least one arg (the op), all the other args for that function) + +# move_op, arg1, arg2 + +# - no smartsim imports + +# core/entrypoints/file_operations.py + + + + + + + + + def move_op(to_move: str, dest_file: str): """Write a python script to moves a file @@ -84,7 +109,9 @@ def copy_op(to_copy: str, entity_path: str): """ -def symlink_op(to_link: str, entity_path: str): + + +def symlink_op(source: str, dest: str): """ Write a python script to make the to_link path a symlink pointing to the entity path. @@ -92,17 +119,35 @@ def symlink_op(to_link: str, entity_path: str): :param entity_path: :return: string of python code to perform operation that can be executed """ - return rf"""import os -from os import path -dest_path = path.join('{entity_path}', path.basename('{to_link}')) + # return rf"""import os + import os + + + os.symlink(source, dest) + #""" + + + # dest_path = path.join('{entity_path}', path.basename('{to_link}')) + # os.symlink('{to_link}', dest_path) + + -os.symlink('{to_link}', dest_path) - """ + + + + +# TODO: needs a destination path -- is there a dest file? + +# TODO: make sure im not missing anything +# TODO: application writer - should be captured in the entry point + def configure_op( - to_configure: str, params: t.Dict[str, str], tag_delimiter: t.Optional[str] = None + to_configure: str, dest: t.Optional[str], params: t.Dict[str, str], tag_delimiter: str = ';' ): + + """Write a python script to set, search and replace the tagged parameters for the configure operation within tagged files attached to an entity. @@ -112,61 +157,118 @@ def configure_op( :return: string of python code to perform operation that can be executed """ - return rf"""import re -import collections - -def _get_prev_value(tagged_line: str) -> str: - split_tag = tagged_line.split(tag_delimiter) - return split_tag[1] - -def _is_ensemble_spec( - tagged_line: str, model_params: dict - ) -> bool: - split_tag = tagged_line.split(tag_delimiter) - prev_val = split_tag[1] - if prev_val in model_params.keys(): - return True - return False - -edited = [] -used_params = {{}} -params = {params} - -tag_delimiter = '{tag_delimiter}' - -if tag_delimiter == 'False': - tag_delimiter = ';' - -regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) - -# Set the lines to iterate over -with open('{to_configure}','r+', encoding='utf-8') as file_stream: - lines = file_stream.readlines() - -unused_tags = collections.defaultdict(list) - -# read lines in file -for i, line in enumerate(lines, 1): - while search := re.search(regex, line): - tagged_line = search.group(0) - previous_value = _get_prev_value(tagged_line) - if _is_ensemble_spec(tagged_line, params): - new_val = str(params[previous_value]) - line = re.sub(regex, new_val, line, 1) - used_params[previous_value] = new_val - - # if a tag_delimiter is found but is not in this model's configurations - # put in placeholder value - else: - tag_delimiter_ = tagged_line.split(tag_delimiter)[1] - unused_tags[tag_delimiter_].append(i) - line = re.sub(regex, previous_value, line) - break - edited.append(line) - -lines = edited - -with open('{to_configure}', "w+", encoding="utf-8") as file_stream: - for line in lines: - file_stream.write(line) - """ + import re + import collections + + def _get_prev_value(tagged_line: str) -> str: + split_tag = tagged_line.split(tag_delimiter) + return split_tag[1] + + def _is_ensemble_spec( + tagged_line: str, model_params: dict + ) -> bool: + split_tag = tagged_line.split(tag_delimiter) + prev_val = split_tag[1] + if prev_val in model_params.keys(): + return True + return False + + edited = [] + used_params = {} + # params = params + + tag_delimiter = tag_delimiter + + regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) + + # Set the lines to iterate over + with open(to_configure,'r+', encoding='utf-8') as file_stream: + lines = file_stream.readlines() + + unused_tags = collections.defaultdict(list) + + # read lines in file + for i, line in enumerate(lines, 1): + while search := re.search(regex, line): + tagged_line = search.group(0) + previous_value = _get_prev_value(tagged_line) + if _is_ensemble_spec(tagged_line, params): + new_val = str(params[previous_value]) + line = re.sub(regex, new_val, line, 1) + used_params[previous_value] = new_val + + # if a tag_delimiter is found but is not in this model's configurations + # put in placeholder value + else: + tag_delimiter_ = tagged_line.split(tag_delimiter)[1] + unused_tags[tag_delimiter_].append(i) + line = re.sub(regex, previous_value, line) + break + edited.append(line) + + lines = edited + + # write to file + if dest: + file_stream = dest + + with open(to_configure, "w+", encoding="utf-8") as file_stream: + for line in lines: + file_stream.write(line) + + +# return rf"""import re +# import collections + +# def _get_prev_value(tagged_line: str) -> str: +# split_tag = tagged_line.split(tag_delimiter) +# return split_tag[1] + +# def _is_ensemble_spec( +# tagged_line: str, model_params: dict +# ) -> bool: +# split_tag = tagged_line.split(tag_delimiter) +# prev_val = split_tag[1] +# if prev_val in model_params.keys(): +# return True +# return False + +# edited = [] +# used_params = {{}} +# params = {params} + +# tag_delimiter = '{tag_delimiter}' + +# regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) + +# # Set the lines to iterate over +# with open('{to_configure}','r+', encoding='utf-8') as file_stream: +# lines = file_stream.readlines() + +# unused_tags = collections.defaultdict(list) + +# # read lines in file +# for i, line in enumerate(lines, 1): +# while search := re.search(regex, line): +# tagged_line = search.group(0) +# previous_value = _get_prev_value(tagged_line) +# if _is_ensemble_spec(tagged_line, params): +# new_val = str(params[previous_value]) +# line = re.sub(regex, new_val, line, 1) +# used_params[previous_value] = new_val + +# # if a tag_delimiter is found but is not in this model's configurations +# # put in placeholder value +# else: +# tag_delimiter_ = tagged_line.split(tag_delimiter)[1] +# unused_tags[tag_delimiter_].append(i) +# line = re.sub(regex, previous_value, line) +# break +# edited.append(line) + +# lines = edited + +# with open('{to_configure}', "w+", encoding="utf-8") as file_stream: +# for line in lines: +# file_stream.write(line) +# """ diff --git a/tests/test_command_generation.py b/tests/test_command_generation.py index 933858480..e285a4a35 100644 --- a/tests/test_command_generation.py +++ b/tests/test_command_generation.py @@ -27,8 +27,7 @@ import filecmp import os import pathlib -import subprocess -import sys + import typing as t from distutils import dir_util from glob import glob @@ -37,43 +36,63 @@ import pytest from smartsim._core.generation import commandgenerator +from smartsim._core.entrypoints import command_generator test_path = os.path.dirname(os.path.abspath(__file__)) test_output_root = os.path.join(test_path, "tests", "test_output") -def test_symlink_entity(): +def test_symlink_files(): """ - Test the execution of the python script created by symlink_op + Test operation to symlink files """ source = pathlib.Path(test_output_root) / "sym_source" os.mkdir(source) - + source_file = pathlib.Path(source) / "sym_source.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("") # entity_path to be the dest dir entity_path = os.path.join(test_output_root, "entity_name") - os.mkdir(entity_path) - target = pathlib.Path(test_output_root) / "entity_name" / "sym_source" + command_generator.symlink_op(source_file, entity_path) + + link = pathlib.Path(test_output_root) / "entity_name" + # Assert the two files are the same file + assert link.is_symlink() + assert os.readlink(link) == str(source_file) + + # Clean up the test directory + os.unlink(link) + os.remove(pathlib.Path(source) / "sym_source.txt") + os.rmdir(pathlib.Path(test_output_root) / "sym_source") + + +def test_symlink_dir(): + """ + Test operation to symlink directories + """ + + source = pathlib.Path(test_output_root) / "sym_source" + os.mkdir(source) + + # entity_path to be the dest dir + entity_path = os.path.join(test_output_root, "entity_name") - # # Build the command - cmd = commandgenerator.symlink_op(source, entity_path) - # # execute the command - subprocess.run([sys.executable, "-c", cmd]) + command_generator.symlink_op(source, entity_path) - # # Assert the two files are the same file - assert target.is_symlink() - assert os.readlink(target) == str(source) + link = pathlib.Path(test_output_root) / "entity_name" + # Assert the two files are the same file + assert link.is_symlink() + assert os.readlink(link) == str(source) # Clean up the test directory - os.unlink(target) - os.rmdir(pathlib.Path(test_output_root) / "entity_name") - os.rmdir(pathlib.Path(test_output_root) / "sym_source") + os.unlink(link) + os.rmdir(pathlib.Path(test_output_root) / "sym_source") def test_copy_op_file(): - """Test the execution of the python script created by copy_op - Test the operation to copy the content of the source file to the destination path + """Test the operation to copy the content of the source file to the destination path with an empty file of the same name already in the directory""" to_copy = os.path.join(test_output_root, "to_copy") @@ -91,8 +110,7 @@ def test_copy_op_file(): dummy_file.write("") # Execute copy - cmd = commandgenerator.copy_op(source_file, entity_path) - subprocess.run([sys.executable, "-c", cmd]) + command_generator.copy_op(source_file, dest_file) # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -103,8 +121,7 @@ def test_copy_op_file(): def test_copy_op_dirs(): - """Test the execution of the python script created by copy_op - Test the oeprations that copies an entire directory tree source to a new location destination + """Test the oeprations that copies an entire directory tree source to a new location destination """ to_copy = os.path.join(test_output_root, "to_copy") @@ -124,8 +141,8 @@ def test_copy_op_dirs(): os.mkdir(entity_path) # copy those files? - cmd = commandgenerator.copy_op(to_copy, entity_path) - subprocess.run([sys.executable, "-c", cmd]) + cmd = command_generator.copy_op(to_copy, entity_path) + #subprocess.run([sys.executable, "-c", cmd]) # Clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -137,8 +154,7 @@ def test_copy_op_dirs(): def test_copy_op(fileutils): - """Test the execution of the python script created by copy_op - Test the operation to copy the content of the source file to the destination file. + """Test the operation to copy the content of the source file to the destination file. """ # make a test file with some contents @@ -155,8 +171,8 @@ def test_copy_op(fileutils): with open(to_copy_file, "r", encoding="utf-8") as dummy_file: assert dummy_file.read() == "dummy" - cmd = commandgenerator.copy_op(to_copy_file, entity_path) - subprocess.run([sys.executable, "-c", cmd]) + cmd = command_generator.copy_op(to_copy_file, entity_path) + #subprocess.run([sys.executable, "-c", cmd]) entity_file = os.path.join(test_output_root, "entity_name", "to_copy.txt") # asser that the entity_path now has the source file with the correct contents @@ -171,8 +187,7 @@ def test_copy_op(fileutils): def test_copy_op_bad_source_file(): - """Test the execution of the python script created by copy_op - Test that a FileNotFoundError is raised when there is a bad source file + """Test that a FileNotFoundError is raised when there is a bad source file """ to_copy = os.path.join(test_output_root, "to_copy") @@ -182,7 +197,7 @@ def test_copy_op_bad_source_file(): # Execute copy with pytest.raises(FileNotFoundError) as ex: - commandgenerator.copy_op("/not/a/real/path", entity_path) + command_generator.copy_op("/not/a/real/path", entity_path) assert "is not a valid path" in ex.value.args[0] # clean up os.rmdir(pathlib.Path(test_output_root) / "to_copy") @@ -190,8 +205,8 @@ def test_copy_op_bad_source_file(): def test_copy_op_bad_dest_path(): - """Test the execution of the python script created by copy_op. - Test that a FileNotFoundError is raised when there is a bad destination file.""" + """Test that a FileNotFoundError is raised when there is a bad destination file. + """ to_copy = os.path.join(test_output_root, "to_copy") os.mkdir(to_copy) @@ -201,7 +216,7 @@ def test_copy_op_bad_dest_path(): os.mkdir(entity_path) with pytest.raises(FileNotFoundError) as ex: - commandgenerator.copy_op(source_file, "/not/a/real/path") + command_generator.copy_op(source_file, "/not/a/real/path") assert "is not a valid path" in ex.value.args[0] # clean up @@ -210,8 +225,7 @@ def test_copy_op_bad_dest_path(): def test_move_op(): - """Test the execution of the python script created by move_op. - Test the operation to move a file""" + """Test the operation to move a file""" source_dir = os.path.join(test_output_root, "from_here") os.mkdir(source_dir) @@ -228,8 +242,8 @@ def test_move_op(): with open(source_file, "r", encoding="utf-8") as dummy_file: assert dummy_file.read() == "dummy" - cmd = commandgenerator.move_op(source_file, dest_file) - subprocess.run([sys.executable, "-c", cmd]) + cmd = command_generator.move_op(source_file, dest_file) + #subprocess.run([sys.executable, "-c", cmd]) # Assert that the move was successful assert not osp.exists(source_file) @@ -243,42 +257,10 @@ def test_move_op(): os.rmdir(dest_dir) -def test_delete_op(): - """Test the execution of the python script created by delete_op. - Test the operation to delete a file""" - - # Make a test file with dummy text - to_del = pathlib.Path(test_output_root) / "app_del.txt" - with open(to_del, "w+", encoding="utf-8") as dummy_file: - dummy_file.write("dummy") - - assert osp.exists(to_del) - with open(to_del, "r", encoding="utf-8") as dummy_file: - assert dummy_file.read() == "dummy" - - cmd = commandgenerator.delete_op(to_del) - subprocess.run([sys.executable, "-c", cmd]) - - # Assert file has been deleted - assert not osp.exists(to_del) - - -def test_delete_op_bad_path(): - """Test the execution of the python script created by delete_op. - Test that FileNotFoundError is raised when a bad path is given to the - soperation to delete a file""" - - test_output_root = os.path.join(test_path, "tests", "test_output") - to_del = pathlib.Path(test_output_root) / "not_real.txt" - - with pytest.raises(FileNotFoundError) as ex: - commandgenerator.delete_op(to_del) - assert "is not a valid path" in ex.value.args[0] def test_configure_op(test_dir, fileutils): - """Test the execution of the python script created by configure_op. - Test configure param operations with a tag parameter given""" + """Test configure param operations with a tag parameter given""" # the param dict for configure operations param_dict = { @@ -304,8 +286,8 @@ def test_configure_op(test_dir, fileutils): # Run configure op on test files for tagged_file in tagged_files: - cmd = commandgenerator.configure_op(tagged_file, param_dict, tag) - subprocess.run([sys.executable, "-c", cmd]) + cmd = command_generator.configure_op(tagged_file,dest=tagged_file, params=param_dict,tag_delimiter=tag) + #subprocess.run([sys.executable, "-c", cmd]) # check that files and correct files are the same for written, correct in zip(tagged_files, correct_files): @@ -313,8 +295,7 @@ def test_configure_op(test_dir, fileutils): def test_configure_op_no_tag(test_dir, fileutils): - """Test the execution of the python script created by configure_op. - Test configure param operations with no tag parameter given""" + """Test configure param operations with no tag parameter given""" # the param dict for configure operations param_dict = { @@ -339,9 +320,242 @@ def test_configure_op_no_tag(test_dir, fileutils): # Run configure op on test files for tagged_file in tagged_files: - cmd = commandgenerator.configure_op(tagged_file, param_dict, False) - subprocess.run([sys.executable, "-c", cmd]) + cmd = command_generator.configure_op(tagged_file,dest=None, params=param_dict) + # subprocess.run([sys.executable, "-c", cmd]) # check that files and correct files are the same for written, correct in zip(tagged_files, correct_files): assert filecmp.cmp(written, correct) + +def test_delete_op(): + """Test the operation to delete a file""" + + # Make a test file with dummy text + to_del = pathlib.Path(test_output_root) / "app_del.txt" + with open(to_del, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + assert osp.exists(to_del) + with open(to_del, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + cmd = command_generator.delete_op(to_del) + #subprocess.run([sys.executable, "-c", cmd]) + + # Assert file has been deleted + assert not osp.exists(to_del) + + +def test_delete_op_bad_path(): + """Test that FileNotFoundError is raised when a bad path is given to the + soperation to delete a file""" + + test_output_root = os.path.join(test_path, "tests", "test_output") + to_del = pathlib.Path(test_output_root) / "not_real.txt" + + with pytest.raises(FileNotFoundError) as ex: + command_generator.delete_op(to_del) + assert "is not a valid path" in ex.value.args[0] + + +from smartsim._core.entrypoints.telemetrymonitor import get_parser +from smartsim._core.entrypoints.command_generator import get_parser + +def test_parser_remove(): + """Test that the parser succeeds when receiving expected args""" + + + parser = get_parser() + + # Make a test file with dummy text + to_remove = pathlib.Path(test_output_root) / "app_del.txt" + with open(to_remove, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + # parser.add_argument('integers', metavar='N', type=int, nargs='+', + # help='an integer for the accumulator') + # parser.add_argument('--sum', dest='accumulate', action='store_const', + # const=sum, default=max, + # help='sum the integers (default: find the max)') + + #args = parser.parse_args() + + + + # test_dir = "/foo/bar" + # test_freq = 123 + + + + # cmd = f"-exp_dir {test_dir} -frequency {test_freq}" + cmd = f"_core/entrypoints/file_operations.py remove {to_remove}" + args = cmd.split() + + print(args) + + #ns = parser.parse_args(args) + + # assert ns.exp_dir == test_dir + # assert ns.frequency == test_freq + + #assert not osp.exists(to_del) + + + +ALL_ARGS = {"-exp_dir", "-frequency"} + +@pytest.mark.parametrize( + ["cmd", "missing"], + [ + pytest.param("", {"-exp_dir", "-frequency"}, id="no args"), + pytest.param("-exp_dir /foo/bar", {"-frequency"}, id="no freq"), + pytest.param("-frequency 123", {"-exp_dir"}, id="no dir"), + ], +) + +def test_parser_reqd_args(capsys, cmd, missing): + """Test that the parser reports any missing required arguments""" + parser = get_parser() + + args = cmd.split() + + captured = capsys.readouterr() # throw away existing output + with pytest.raises(SystemExit) as ex: + ns = parser.parse_args(args) + + captured = capsys.readouterr() + assert "the following arguments are required" in captured.err + err_desc = captured.err.split("the following arguments are required:")[-1] + for arg in missing: + assert arg in err_desc + + expected = ALL_ARGS - missing + for exp in expected: + assert exp not in err_desc + + +def test_parser(): + """Test that the parser succeeds when receiving expected args""" + parser = get_parser() + + test_dir = "/foo/bar" + test_freq = 123 + + cmd = f"-exp_dir {test_dir} -frequency {test_freq}" + args = cmd.split() + + ns = parser.parse_args(args) + print(ns) + + assert ns.exp_dir == test_dir + assert ns.frequency == test_freq + + + + + +def test_parser_move(): + """Test that the parser succeeds when receiving expected args""" + parser = get_parser() + + + src_path = "/absolute/file/src/path" + dest_path = "/absolute/file/dest/path" + + cmd = f"move {src_path} {dest_path}" # must be absolute path + # python + args = cmd.split() + print(args) + ns = parser.parse_args(args) + print(ns) + + assert ns.src_path == src_path + assert ns.dest_path == dest_path + +def test_parser_remove(): + """Test that the parser succeeds when receiving expected args""" + parser = get_parser() + + file_path = "/absolute/file/path" + cmd = f"remove {file_path}" + + args = cmd.split() + ns = parser.parse_args(args) + assert ns.to_remove == file_path + +def test_parser_symlink(): + """Test that the parser succeeds when receiving expected args""" + parser = get_parser() + + src_path = "/absolute/file/src/path" + dest_path = "/absolute/file/dest/path" + cmd = f"symlink {src_path} {dest_path}" + + args = cmd.split() + + ns = parser.parse_args(args) + + assert ns.source_path == src_path + assert ns.dest_path == dest_path + +def test_parser_copy(): + """Test that the parser succeeds when receiving expected args""" + parser = get_parser() + + src_path = "/absolute/file/src/path" + dest_path = "/absolute/file/dest/path" + + cmd = f"copy {src_path} {dest_path}" # must be absolute path + # python + args = cmd.split() + + ns = parser.parse_args(args) + print(ns) + + assert ns.source_path == src_path + assert ns.dest_path == dest_path + +import json + +def test_parser_configure(): + """Test that the parser succeeds when receiving expected args""" + parser = get_parser() + + src_path = "/absolute/file/src/path" + dest_path = "/absolute/file/dest/path" + tag_delimiter = ";" + params = '{"5":10}' + +# '{"5": 10}' + +# >>> +# {u'value1': u'key1'} + + cmd = f"configure {src_path} {dest_path} {tag_delimiter} {params}" # must be absolute path + # python + args = cmd.split() + + ns = parser.parse_args(args) + print(ns) + + +#parser.add_argument('-m', '--my-dict', type=str) +#args = parser.parse_args() + print("HIHIHI") +#import json + my_dictionary = json.loads(ns.param_dict) + print(my_dictionary) + print("is thre a type",type(my_dictionary)) + + + assert ns.source_path == src_path + assert ns.dest_path == dest_path + assert ns.tag_delimiter == tag_delimiter + assert ns.param_dict == params + + +def test_command_generator_entrypoint(): + ... + parser = get_parser() + + # from smartsim._core.generation import commandgenerator diff --git a/tests/test_telemetry_monitor.py b/tests/test_telemetry_monitor.py index c1bfe2719..cf2db9696 100644 --- a/tests/test_telemetry_monitor.py +++ b/tests/test_telemetry_monitor.py @@ -149,6 +149,7 @@ def test_parser(): args = cmd.split() ns = parser.parse_args(args) + print(ns) assert ns.exp_dir == test_dir assert ns.frequency == test_freq From c55acb3de56428a6f1cee1afdfcba27e79d9cfc3 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Wed, 3 Jul 2024 22:42:42 -0500 Subject: [PATCH 04/17] WIP, todo: configure tests --- .../_core/entrypoints/command_generator.py | 308 ++++++------- smartsim/_core/entrypoints/file_operations.py | 361 +++++++++++++++ tests/test_command_generation.py | 419 ++++++++---------- 3 files changed, 672 insertions(+), 416 deletions(-) create mode 100644 smartsim/_core/entrypoints/file_operations.py diff --git a/smartsim/_core/entrypoints/command_generator.py b/smartsim/_core/entrypoints/command_generator.py index a480a65f3..d5de62122 100644 --- a/smartsim/_core/entrypoints/command_generator.py +++ b/smartsim/_core/entrypoints/command_generator.py @@ -25,19 +25,51 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse +import ast import os import typing as t import shutil from distutils import dir_util import re import collections +import json +from pathlib import Path + -def get_dst_path(application_path, input_file): - return os.path.join(application_path, os.path.basename(input_file)) +# TODO: remove the entity path things and keep it as a helper funtion that +# gives the user to add the application, and then it would take the applciation path and do the copy +# def get_dest_path(application_path, input_file) -> Path: +# return os.path.join(application_path, os.path.basename(input_file)) -def _check_path(file_path: str) -> str: +def _str_to_dict(string): + # remove the curly braces from the string + print("inside str to dict") + #"{'Nikhil", "lastname'", ':', '1,', "'Akshat'", ':', '2,', "'Akash'", ':', '3}' + #'{"Nikhil" : 1, "Akshat" : 2, "Akash" : 3}' + + string = str(string)[1:-1] + print(string) + string = string.strip('{}') + print(string) + + # split the string into key-value pairs + pairs = string.split(", ") + print(pairs) + for pair in pairs: + print(pair) + # "{'Nikhil", "lastname'", ':', '1,', "'Akshat'", ':', '2,', "'Akash'", ':', '3}' + # ['"{\'Nikhil"', '"lastname\'"', "':'", "'1,'", '"\'Akshat\'"', "':'", "'2,'", '"\'Akash\'"', "':'", "'3}'"] + + #"{'Nikhil", "lastname'", ':', '1,', "'Akshat'", ':', '2,', "'Akash'", ':', '3}' + + # use a dictionary comprehension to create + # the dictionary, converting the values to + # integers and removing the quotes from the keys + return {key[1:-2]: int(value) for key, value in (pair.split(': ') for pair in pairs)} + +def _check_path(file_path: str) -> Path: """Given a user provided path-like str, find the actual path to the directory or file and create a full path. @@ -53,86 +85,112 @@ def _check_path(file_path: str) -> str: raise FileNotFoundError(f"File or Directory {file_path} not found") - -def move_op(to_move: str, dst_path: str): +def move(args) -> None: """Move a file + eg. python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path + :param to_move: Path to a source file to be copied :param dst_path: Path to a file to copy the contents from the source file into - :return: string of python code to perform operation that can be executed """ - - shutil.move(to_move, dst_path) + _check_path(args.source) + _check_path(args.dest) + shutil.move(args.source, args.dest) -def remove_op(to_delete: str): +def remove(args) -> None: """Write a python script that removes a file when executed. - :param to_delete: Path to the file to be deleted - :return: string of python code to perform operation that can be executed - """ - _check_path(to_delete) - - os.remove(to_delete) + eg. python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path - -# TODO: remove the entity path things and keep it as a helper funtion that -# gives the user to add the application, and then it would take the applciation path and do the copy + :param to_remove: Path to the file to be deleted + """ + _check_path(args.to_remove) + os.remove(args.to_remove) -def copy_op(source_path: str, dest_path: str): +def copy(args): + # args.source_path: str, dest_path: str): """ Write a python script to copy the entity files and directories attached to this entity into an entity directory + eg. python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path + :param source_path: Path to directory, or path to file to copy into an entity directory :param dest_path: Path to destination directory or path to destination file to copy - :return: string of python code to perform operation that can be executed """ + _check_path(args.source) + _check_path(args.dest) - #_check_path(to_copy) - - if os.path.isdir(source_path): - dir_util.copy_tree(source_path, dest_path) + if os.path.isdir(args.source): + dir_util.copy_tree(args.source, args.dest) else: - shutil.copyfile(source_path, dest_path) + shutil.copyfile(args.source, args.dest) - -def symlink_op(source: str, link: str): +def symlink(args): """ Create a symbolic link pointing to the exisiting source file named link + eg. python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path + :param src_file: the exisiting source path :param dst_file: target name where the symlink will be created. """ + _check_path(args.source) - _check_path(source) - - os.symlink(source, link) - + os.symlink(args.source, args.dest) -def configure_op( - to_configure: str, dst_path: t.Optional[str], param_dict: t.Dict[str, str], tag_delimiter: str = ';' -): +def configure(args): """Write a python script to set, search and replace the tagged parameters for the configure operation within tagged files attached to an entity. User-formatted files can be attached using the `to_configure` argument. These files will be modified -during ``Model`` generation to replace tagged sections in the user-formatted files with -values from the `params` initializer argument used during ``Model`` creation: + during ``Model`` generation to replace tagged sections in the user-formatted files with + values from the `params` initializer argument used during ``Model`` creation: + + eg. python _core/entrypoints/file_operations.py configure /absolte/file/src/path /absolte/file/dest/path tagged_deliminator params # base64 encoded dictionary for params :param to_configure: The tagged files the search and replace operations to be performed upon :param dest: Optional destination for configured files to be written to :param param_dict: A dict of parameter names and values set for the file :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" - :return: string of python code to perform operation that can be executed """ + #args.to_configure: str, args.dest_path: t.Optional[str], args.param_dict: t.Dict[str, str], args.tag_delimiter: str = ';' + + + print(args.tag_delimiter) + print(args.param_dict) + print(args.source) + print(args.dest) + + print(type(args.param_dict)) + + param_dict = str(args.param_dict) + + #param_dict = eval(args.param_dict) + + #param_dict = ''.join(args.param_dict) + + #param_dict = json.loads(args.param_dict) + param_dict = _str_to_dict(args.param_dict) + + print("the param dict", param_dict) + + tag_delimiter = ";" + if args.tag_delimiter: + tag_delimiter = args.tag_delimiter + - _check_path(to_configure) - _check_path(dst_path) + + _check_path(args.source) + if args.dest: + _check_path(args.dest) + + def _get_prev_value(tagged_line: str) -> str: split_tag = tagged_line.split(tag_delimiter) @@ -158,7 +216,7 @@ def _is_ensemble_spec( regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) # Set the lines to iterate over - with open(to_configure,'r+', encoding='utf-8') as file_stream: + with open(args.source,'r+', encoding='utf-8') as file_stream: lines = file_stream.readlines() unused_tags = collections.defaultdict(list) @@ -169,8 +227,8 @@ def _is_ensemble_spec( while search := re.search(regex, line): tagged_line = search.group(0) previous_value = _get_prev_value(tagged_line) - if _is_ensemble_spec(tagged_line, params): - new_val = str(params[previous_value]) + if _is_ensemble_spec(tagged_line, param_dict): + new_val = str(param_dict[previous_value]) line = re.sub(regex, new_val, line, 1) used_params[previous_value] = new_val @@ -186,21 +244,14 @@ def _is_ensemble_spec( lines = edited # write configured file to destination specified. Default is an overwrite - if dst_path: - file_stream = dst_path + if args.dest: + file_stream = args.dest - with open(to_configure, "w+", encoding="utf-8") as file_stream: + with open(args.source, "w+", encoding="utf-8") as file_stream: for line in lines: file_stream.write(line) -# python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path -# python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path -# python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path -# python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path -# python _core/entrypoints/file_operations.py configure /absolte/file/src/path /absolte/file/dest/path tagged_deliminator params # base64 encoded dictionary for params - -import json def get_parser() -> argparse.ArgumentParser: """Instantiate a parser to process command line arguments @@ -208,16 +259,13 @@ def get_parser() -> argparse.ArgumentParser: """ arg_parser = argparse.ArgumentParser(description="Command Generator") - - subparsers = arg_parser.add_subparsers(help='file_operations') # subparser for move op move_parser = subparsers.add_parser("move") - move_parser.add_argument("src_path") - move_parser.add_argument("dest_path") - - #usage: pytest move [-h] operation src_path dest_path + move_parser.set_defaults(func=move) + move_parser.add_argument("source") + move_parser.add_argument("dest") # subparser for remove op remove_parser = subparsers.add_parser("remove") @@ -227,123 +275,54 @@ def get_parser() -> argparse.ArgumentParser: # subparser for copy op copy_parser = subparsers.add_parser("copy") + copy_parser.set_defaults(func=copy) copy_parser.add_argument( - "source_path", + "source", type = str) copy_parser.add_argument( - "dest_path", + "dest", type = str) + # subparser for symlink op symlink_parser = subparsers.add_parser("symlink") + symlink_parser.set_defaults(func=symlink) symlink_parser.add_argument( - "source_path", + "source", type = str) symlink_parser.add_argument( - "dest_path", + "dest", type = str) # subparser for configure op configure_parser = subparsers.add_parser("configure") + configure_parser.set_defaults(func=configure) configure_parser.add_argument( - "source_path", + "source", type = str) configure_parser.add_argument( - "dest_path", + "dest", type = str) configure_parser.add_argument( "tag_delimiter", type = str) configure_parser.add_argument( "param_dict", - type = str) - -# parser.add_argument('-m', '--my-dict', type=str) -# args = parser.parse_args() - -# import json -# my_dictionary = json.loads(args.my_dict) - + type = str, + nargs='+') - - - - # arg_parser.add_argument( - # "move", - # type=str, - # help="move files", - # # required=True, - # default="" - # ) - # arg_parser.add_argument( - # "remove", - # type=str, - # help="delete files", - # #required=True, - # default="" - # ) - # arg_parser.add_argument( - # "symlink", - # type=str, - # help="symlink a file", - # #required=True, - # default="" - # ) - # arg_parser.add_argument( - # "copy", - # type=str, - # help="copy a file or files from one destination to another", - # #required=True, - # default="" - # ) - # arg_parser.add_argument( - # "configure", - # type=str, - # help="configure a file with tagged parameters", - # #required=True, - # default="" - - # arg_parser.add_argument( - # "-exp_dir", - # type=str, - # help="Experiment root directory", - # required=True, - # ) - # arg_parser.add_argument( - # "-frequency", - # type=float, - # help="Frequency of telemetry updates (in seconds))", - # required=True, - # ) - # arg_parser.add_argument( - # "-cooldown", - # type=int, - # help="Default lifetime of telemetry monitor (in seconds) before auto-shutdown", - # default=cfg.CONFIG.telemetry_cooldown, - # ) - # arg_parser.add_argument( - # "-loglevel", - # type=int, - # help="Logging level", - # default=logging.INFO, - # ) return arg_parser -def parse_arguments():# -> TelemetryMonitorArgs: - """Parse the command line arguments and return an instance - of TelemetryMonitorArgs populated with the CLI inputs +def parse_arguments() -> str: + """Parse the command line arguments - :returns: `TelemetryMonitorArgs` instance populated with command line arguments + # :returns: the parsed command line arguments """ parser = get_parser() parsed_args = parser.parse_args() - print(parsed_args) - # return TelemetryMonitorArgs( - # parsed_args.exp_dir, - # parsed_args.frequency, - # parsed_args.cooldown, - # parsed_args.loglevel, - # ) + parsed_args.func(parsed_args) + return parsed_args + if __name__ == "__main__": @@ -355,41 +334,8 @@ def parse_arguments():# -> TelemetryMonitorArgs: The experiment id is generated during experiment startup and can be found in the manifest.json in /.smartsim/telemetry """ - os.environ["PYTHONUNBUFFERED"] = "1" - - args = parse_arguments() #JPNOTE get from here - pos args, first one in, the rest will get fed into the rest of the functions - #sys args? - some number of strings that come after - #configure_logger(logger, args.log_level, args.exp_dir) - - print("IN MAIN HERE") - - if args[1] == 'remove': - - to_remove = args[2] - remove_op(to_remove) - - if args[1] == 'move': - to_move = args [2] - entity_path = args[3] - move_op(to_move, entity_path) - - if args[1] == 'symlink': - - symlink_op() - - if args[1] == 'copy': - copy_op() - - if args[1] == 'configure': - configure_op() - - - import json - my_dictionary = json.loads(args.my_dict) - - - - + os.environ["PYTHONUNBUFFERED"] = "1" #? + args = parse_arguments() - # sys.exit(1) # do I need? + # sys.exit(1) #? diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py new file mode 100644 index 000000000..48c125ba0 --- /dev/null +++ b/smartsim/_core/entrypoints/file_operations.py @@ -0,0 +1,361 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024 Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import os +import typing as t +import shutil +from distutils import dir_util +import re +import collections + + +def get_dst_path(application_path, input_file): + return os.path.join(application_path, os.path.basename(input_file)) + + +def _check_path(file_path: str) -> str: + """Given a user provided path-like str, find the actual path to + the directory or file and create a full path. + + :param file_path: path to a specific file or directory + :raises FileNotFoundError: if file or directory does not exist + :return: full path to file or directory + """ + full_path = os.path.abspath(file_path) + if os.path.isfile(full_path): + return full_path + if os.path.isdir(full_path): + return full_path + raise FileNotFoundError(f"File or Directory {file_path} not found") + + + +def move(source: str, dest: str): + """Move a file + + :param to_move: Path to a source file to be copied + :param dst_path: Path to a file to copy the contents from the source file into + :return: string of python code to perform operation that can be executed + """ + + _check_path(source) + + shutil.move(source, dest) + + +def remove(to_delete: str): + """Write a python script that removes a file when executed. + + :param to_delete: Path to the file to be deleted + :return: string of python code to perform operation that can be executed + """ + _check_path(to_delete) + + os.remove(to_delete) + + +# TODO: remove the entity path things and keep it as a helper funtion that +# gives the user to add the application, and then it would take the applciation path and do the copy + + +def copy(args): + # args.source_path: str, dest_path: str): + """ + Write a python script to copy the entity files and directories attached + to this entity into an entity directory + + :param source_path: Path to directory, or path to file to copy into an entity directory + :param dest_path: Path to destination directory or path to destination file to copy + :return: string of python code to perform operation that can be executed + """ + + print("MADE IT INTO THE COPY FUNCTION") + _check_path(args.source_path) + _check_path(args.dest_path) + + if os.path.isdir(args.source_path): + dir_util.copy_tree(args.source_path, args.dest_path) + else: + shutil.copyfile(args.source_path, args.dest_path) + + + +def symlink(source: str, link: str): + """ + Create a symbolic link pointing to the exisiting source file + named link + + :param src_file: the exisiting source path + :param dst_file: target name where the symlink will be created. + """ + + _check_path(source) + + os.symlink(source, link) + + + +def configure( + to_configure: str, dest_path: t.Optional[str], param_dict: t.Dict[str, str], tag_delimiter: str = ';' +): + """Write a python script to set, search and replace the tagged parameters for the configure operation + within tagged files attached to an entity. + + User-formatted files can be attached using the `to_configure` argument. These files will be modified +during ``Model`` generation to replace tagged sections in the user-formatted files with +values from the `params` initializer argument used during ``Model`` creation: + + :param to_configure: The tagged files the search and replace operations to be performed upon + :param dest: Optional destination for configured files to be written to + :param param_dict: A dict of parameter names and values set for the file + :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" + :return: string of python code to perform operation that can be executed + """ + + _check_path(to_configure) + if dest_path: + _check_path(dest_path) + + def _get_prev_value(tagged_line: str) -> str: + split_tag = tagged_line.split(tag_delimiter) + return split_tag[1] + + def _is_ensemble_spec( + tagged_line: str, model_params: dict + ) -> bool: + split_tag = tagged_line.split(tag_delimiter) + prev_val = split_tag[1] + if prev_val in model_params.keys(): + return True + return False + + edited = [] + used_params = {} + #param_dict = {param_dict} + + # tag_delimiter = tag_delimiter + + # Set the tag for the modelwriter to search for within + # tagged files attached to an entity. + regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) + + # Set the lines to iterate over + with open(to_configure,'r+', encoding='utf-8') as file_stream: + lines = file_stream.readlines() + + unused_tags = collections.defaultdict(list) + + # Replace the tagged parameters within the file attached to this + # model. The tag defaults to ";" + for i, line in enumerate(lines, 1): + while search := re.search(regex, line): + tagged_line = search.group(0) + previous_value = _get_prev_value(tagged_line) + if _is_ensemble_spec(tagged_line, param_dict): + new_val = str(param_dict[previous_value]) + line = re.sub(regex, new_val, line, 1) + used_params[previous_value] = new_val + + # if a tag_delimiter is found but is not in this model's configurations + # put in placeholder value + else: + tag_delimiter_ = tagged_line.split(tag_delimiter)[1] + unused_tags[tag_delimiter_].append(i) + line = re.sub(regex, previous_value, line) + break + edited.append(line) + + lines = edited + + # write configured file to destination specified. Default is an overwrite + if dest_path: + file_stream = dest_path + + with open(to_configure, "w+", encoding="utf-8") as file_stream: + for line in lines: + file_stream.write(line) + + +# python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path +# python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path +# python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path +# python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path +# python _core/entrypoints/file_operations.py configure /absolte/file/src/path /absolte/file/dest/path tagged_deliminator params # base64 encoded dictionary for params + + +import json +def get_parser() -> argparse.ArgumentParser: + """Instantiate a parser to process command line arguments + + :returns: An argument parser ready to accept required command generator parameters + """ + arg_parser = argparse.ArgumentParser(description="Command Generator") + + subparsers = arg_parser.add_subparsers(help='file_operations') + + # subparser for move op + move_parser = subparsers.add_parser("move") + move_parser.set_defaults(func=move) + move_parser.add_argument("src_path") + move_parser.add_argument("dest_path") + + # subparser for remove op + remove_parser = subparsers.add_parser("remove") + remove_parser.add_argument( + "to_remove", + type = str) + + # subparser for copy op + copy_parser = subparsers.add_parser("copy") + copy_parser.set_defaults(func=copy) + copy_parser.add_argument( + "source_path", + type = str) + copy_parser.add_argument( + "dest_path", + type = str) + + # subparser for symlink op + symlink_parser = subparsers.add_parser("symlink") + symlink_parser.set_defaults(func=symlink) + symlink_parser.add_argument( + "source_path", + type = str) + symlink_parser.add_argument( + "dest_path", + type = str) + + # subparser for configure op + configure_parser = subparsers.add_parser("configure") + configure_parser.set_defaults(func=configure) + configure_parser.add_argument( + "source_path", + type = str) + configure_parser.add_argument( + "dest_path", + type = str) + configure_parser.add_argument( + "tag_delimiter", + type = str) + configure_parser.add_argument( + "param_dict", + type = str) + + return arg_parser + + +def parse_arguments() -> str: # -> TelemetryMonitorArgs: + # """Parse the command line arguments and return an instance + # of TelemetryMonitorArgs populated with the CLI inputs + + # :returns: `TelemetryMonitorArgs` instance populated with command line arguments + #""" + parser = get_parser() + parsed_args = parser.parse_args() + + parsed_args.func(parsed_args) + #print(parsed_args) + # return TelemetryMonitorArgs( + # parsed_args.exp_dir, + # parsed_args.frequency, + # parsed_args.cooldown, + # parsed_args.loglevel, + # ) + #parsed_args.func(parsed_args) + return parsed_args + + + + +if __name__ == "__main__": + """Prepare the telemetry monitor process using command line arguments. + + Sample usage: + python -m smartsim._core.entrypoints.telemetrymonitor -exp_dir + -frequency 30 -cooldown 90 -loglevel INFO + The experiment id is generated during experiment startup + and can be found in the manifest.json in /.smartsim/telemetry + """ + os.environ["PYTHONUNBUFFERED"] = "1" + + args = parse_arguments() #JPNOTE get from here - pos args, first one in, the rest will get fed into the rest of the functions + #sys args? - some number of strings that come after + #configure_logger(logger, args.log_level, args.exp_dir) + + print(args) + + print("IN MAIN HERE") + import json + + + + # if arg.copy + # my_dictionary = json.loads(args.my_dict) + + # args = parser.parse_args() + + + # def command1(args): + # print("command1: %s" % args.name) + + # def command2(args): + # print("comamnd2: %s" % args.frequency) + + # if __name__ == '__main__': + # main() + + #ns = parser.parse_args(args) + + # if args[1] == 'remove': + + # to_remove = args[2] + # remove_op(to_remove) + + # if args[1] == 'move': + # to_move = args [2] + # entity_path = args[3] + # move_op(to_move, entity_path) + + # if args[1] == 'symlink': + + # symlink_op() + + # if args[1] == 'copy': + # copy_op() + + # if args[1] == 'configure': + # configure_op() + + + + + + + + + + # sys.exit(1) # do I need? diff --git a/tests/test_command_generation.py b/tests/test_command_generation.py index e285a4a35..1daece861 100644 --- a/tests/test_command_generation.py +++ b/tests/test_command_generation.py @@ -34,9 +34,8 @@ from os import path as osp import pytest - -from smartsim._core.generation import commandgenerator from smartsim._core.entrypoints import command_generator +from smartsim._core.entrypoints.command_generator import get_parser test_path = os.path.dirname(os.path.abspath(__file__)) test_output_root = os.path.join(test_path, "tests", "test_output") @@ -55,13 +54,19 @@ def test_symlink_files(): # entity_path to be the dest dir entity_path = os.path.join(test_output_root, "entity_name") - command_generator.symlink_op(source_file, entity_path) + + parser = get_parser() + cmd = f"symlink {source_file} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + command_generator.symlink(ns) link = pathlib.Path(test_output_root) / "entity_name" # Assert the two files are the same file assert link.is_symlink() assert os.readlink(link) == str(source_file) - + # Clean up the test directory os.unlink(link) os.remove(pathlib.Path(source) / "sym_source.txt") @@ -79,7 +84,12 @@ def test_symlink_dir(): # entity_path to be the dest dir entity_path = os.path.join(test_output_root, "entity_name") - command_generator.symlink_op(source, entity_path) + parser = get_parser() + cmd = f"symlink {source} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + command_generator.symlink(ns) link = pathlib.Path(test_output_root) / "entity_name" # Assert the two files are the same file @@ -109,8 +119,14 @@ def test_copy_op_file(): with open(dest_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("") + + parser = get_parser() + cmd = f"copy {source_file} {dest_file}" + args = cmd.split() + ns = parser.parse_args(args) + # Execute copy - command_generator.copy_op(source_file, dest_file) + command_generator.copy(ns) # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -139,10 +155,14 @@ def test_copy_op_dirs(): # entity_path to be the dest dir entity_path = os.path.join(test_output_root, "entity_name") os.mkdir(entity_path) - # copy those files? - cmd = command_generator.copy_op(to_copy, entity_path) - #subprocess.run([sys.executable, "-c", cmd]) + parser = get_parser() + cmd = f"copy {to_copy} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + # Execute copy + command_generator.copy(ns) # Clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -153,39 +173,6 @@ def test_copy_op_dirs(): os.rmdir(pathlib.Path(test_output_root) / "entity_name") -def test_copy_op(fileutils): - """Test the operation to copy the content of the source file to the destination file. - """ - - # make a test file with some contents - to_copy_file = fileutils.make_test_file( - "to_copy.txt", pathlib.Path(test_output_root) / "to_copy", "dummy" - ) - - entity_path = os.path.join(test_output_root, "entity_name") - - os.mkdir(entity_path) - - # assert that source file exists, has correct contents - assert osp.exists(to_copy_file) - with open(to_copy_file, "r", encoding="utf-8") as dummy_file: - assert dummy_file.read() == "dummy" - - cmd = command_generator.copy_op(to_copy_file, entity_path) - #subprocess.run([sys.executable, "-c", cmd]) - - entity_file = os.path.join(test_output_root, "entity_name", "to_copy.txt") - # asser that the entity_path now has the source file with the correct contents - with open(entity_file, "r", encoding="utf-8") as dummy_file: - assert "dummy" in dummy_file.read() - - # Clean up the test directory - os.remove(to_copy_file) - os.remove(entity_file) - os.rmdir(pathlib.Path(test_output_root) / "to_copy") - os.rmdir(pathlib.Path(test_output_root) / "entity_name") - - def test_copy_op_bad_source_file(): """Test that a FileNotFoundError is raised when there is a bad source file """ @@ -195,10 +182,18 @@ def test_copy_op_bad_source_file(): entity_path = os.path.join(test_output_root, "entity_name") os.mkdir(entity_path) + bad_path = "/not/a/real/path" # Execute copy + + parser = get_parser() + cmd = f"copy {bad_path} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + with pytest.raises(FileNotFoundError) as ex: - command_generator.copy_op("/not/a/real/path", entity_path) - assert "is not a valid path" in ex.value.args[0] + command_generator.copy(ns) + assert f"File or Directory {bad_path} not found" in ex.value.args[0] + # clean up os.rmdir(pathlib.Path(test_output_root) / "to_copy") os.rmdir(pathlib.Path(test_output_root) / "entity_name") @@ -212,14 +207,24 @@ def test_copy_op_bad_dest_path(): os.mkdir(to_copy) source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") entity_path = os.path.join(test_output_root, "entity_name") os.mkdir(entity_path) + bad_path = "/not/a/real/path" + + parser = get_parser() + cmd = f"copy {source_file} {bad_path}" + args = cmd.split() + ns = parser.parse_args(args) + with pytest.raises(FileNotFoundError) as ex: - command_generator.copy_op(source_file, "/not/a/real/path") - assert "is not a valid path" in ex.value.args[0] + command_generator.copy(ns) + assert f"File or Directory {bad_path} not found" in ex.value.args[0] # clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") os.rmdir(pathlib.Path(test_output_root) / "to_copy") os.rmdir(pathlib.Path(test_output_root) / "entity_name") @@ -232,18 +237,25 @@ def test_move_op(): dest_dir = os.path.join(test_output_root, "to_here") os.mkdir(dest_dir) - dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") + #dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") + dest_file = pathlib.Path(dest_dir) / "to_here.txt" + with open(dest_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write(" ") + source_file = pathlib.Path(source_dir) / "app_move.txt" - with open(source_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("dummy") assert osp.exists(source_file) with open(source_file, "r", encoding="utf-8") as dummy_file: assert dummy_file.read() == "dummy" + + parser = get_parser() + cmd = f"move {source_file} {dest_file}" + args = cmd.split() + ns = parser.parse_args(args) - cmd = command_generator.move_op(source_file, dest_file) - #subprocess.run([sys.executable, "-c", cmd]) + command_generator.move(ns) # Assert that the move was successful assert not osp.exists(source_file) @@ -256,22 +268,64 @@ def test_move_op(): os.remove(dest_file) os.rmdir(dest_dir) +def test_remove_op(): + """Test the operation to delete a file""" + + # Make a test file with dummy text + to_del = pathlib.Path(test_output_root) / "app_del.txt" + with open(to_del, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + assert osp.exists(to_del) + with open(to_del, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + + command_generator.remove(ns) + + # Assert file has been deleted + assert not osp.exists(to_del) + + +def test_remove_op_bad_path(): + """Test that FileNotFoundError is raised when a bad path is given to the + soperation to delete a file""" + + test_output_root = os.path.join(test_path, "tests", "test_output") + to_del = pathlib.Path(test_output_root) / "not_real.txt" + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + with pytest.raises(FileNotFoundError) as ex: + command_generator.remove(ns) + assert f"File or Directory {to_del} not found" in ex.value.args[0] def test_configure_op(test_dir, fileutils): """Test configure param operations with a tag parameter given""" # the param dict for configure operations - param_dict = { - "5": 10, # MOM_input - "FIRST": "SECOND", # example_input.i - "17": 20, # in.airebo - "65": "70", # in.atm - "placeholder": "group leftupper region", # in.crack - "1200": "120", # input.nml - } + # param_dict = { + # "5": 10, # MOM_input + # "FIRST": "SECOND", # example_input.i + # "17": 20, # in.airebo + # "65": "70", # in.atm + # "placeholder": "group leftupper region", # in.crack + # "1200": "120", # input.nml + # } + #"{\"error\":\"Wrong account\"}" + param_dict = '{"5":10,"FIRST":"SECOND","17":20,"65":"70","placeholder":\"group leftupper region\","1200":"120"}' tag = ";" + + # json loads --> needs to be ' with double quotes inside ' + # but for the arg parse -- needs to have double quotes around everything # retreive tagged files conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) # retrieve files to compare after test @@ -284,9 +338,23 @@ def test_configure_op(test_dir, fileutils): tagged_files = sorted(glob(test_dir + "/*")) correct_files = sorted(glob(correct_path + "*")) - # Run configure op on test files + + parser = get_parser() + cmd = f" configure {tagged_files[0]} {tagged_files[0]} {tag} {param_dict}" + args = cmd.split() + ns = parser.parse_args(args) + print(ns) + + #command_generator.configure(ns) + #Run configure op on test files for tagged_file in tagged_files: - cmd = command_generator.configure_op(tagged_file,dest=tagged_file, params=param_dict,tag_delimiter=tag) + #cmd = f" configure {tagged_file} {tagged_file} {param_dict} {tag}" + parser = get_parser() + cmd = f" configure {tagged_file} {tagged_file} {tag} {param_dict}" + args = cmd.split() + ns = parser.parse_args(args) + command_generator.configure(ns) + #tagged_file, dest=tagged_file, param_dict=param_dict,tag_delimiter=tag #subprocess.run([sys.executable, "-c", cmd]) # check that files and correct files are the same @@ -298,14 +366,15 @@ def test_configure_op_no_tag(test_dir, fileutils): """Test configure param operations with no tag parameter given""" # the param dict for configure operations - param_dict = { - "5": 10, # MOM_input - "FIRST": "SECOND", # example_input.i - "17": 20, # in.airebo - "65": "70", # in.atm - "placeholder": "group leftupper region", # in.crack - "1200": "120", # input.nml - } + # param_dict = { + # "5": 10, # MOM_input + # "FIRST": "SECOND", # example_input.i + # "17": 20, # in.airebo + # "65": "70", # in.atm + # "placeholder": "group leftupper region", # in.crack + # "1200": "120", # input.nml + # } + param_dict = '{"5":10,"FIRST":"SECOND","17":20,"65":"70","placeholder":"group_leftupper_region","1200":"120"}' # retreive tagged files conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) # retrieve files to compare after test @@ -320,157 +389,27 @@ def test_configure_op_no_tag(test_dir, fileutils): # Run configure op on test files for tagged_file in tagged_files: - cmd = command_generator.configure_op(tagged_file,dest=None, params=param_dict) + cmd = command_generator.configure(tagged_file,dest_path=None, param_dict=param_dict) # subprocess.run([sys.executable, "-c", cmd]) # check that files and correct files are the same for written, correct in zip(tagged_files, correct_files): assert filecmp.cmp(written, correct) -def test_delete_op(): - """Test the operation to delete a file""" - - # Make a test file with dummy text - to_del = pathlib.Path(test_output_root) / "app_del.txt" - with open(to_del, "w+", encoding="utf-8") as dummy_file: - dummy_file.write("dummy") - - assert osp.exists(to_del) - with open(to_del, "r", encoding="utf-8") as dummy_file: - assert dummy_file.read() == "dummy" - - cmd = command_generator.delete_op(to_del) - #subprocess.run([sys.executable, "-c", cmd]) - - # Assert file has been deleted - assert not osp.exists(to_del) - - -def test_delete_op_bad_path(): - """Test that FileNotFoundError is raised when a bad path is given to the - soperation to delete a file""" - - test_output_root = os.path.join(test_path, "tests", "test_output") - to_del = pathlib.Path(test_output_root) / "not_real.txt" - - with pytest.raises(FileNotFoundError) as ex: - command_generator.delete_op(to_del) - assert "is not a valid path" in ex.value.args[0] - - -from smartsim._core.entrypoints.telemetrymonitor import get_parser -from smartsim._core.entrypoints.command_generator import get_parser - -def test_parser_remove(): - """Test that the parser succeeds when receiving expected args""" - - - parser = get_parser() - - # Make a test file with dummy text - to_remove = pathlib.Path(test_output_root) / "app_del.txt" - with open(to_remove, "w+", encoding="utf-8") as dummy_file: - dummy_file.write("dummy") - - # parser.add_argument('integers', metavar='N', type=int, nargs='+', - # help='an integer for the accumulator') - # parser.add_argument('--sum', dest='accumulate', action='store_const', - # const=sum, default=max, - # help='sum the integers (default: find the max)') - - #args = parser.parse_args() - - - - # test_dir = "/foo/bar" - # test_freq = 123 - - - - # cmd = f"-exp_dir {test_dir} -frequency {test_freq}" - cmd = f"_core/entrypoints/file_operations.py remove {to_remove}" - args = cmd.split() - - print(args) - - #ns = parser.parse_args(args) - - # assert ns.exp_dir == test_dir - # assert ns.frequency == test_freq - - #assert not osp.exists(to_del) - - - -ALL_ARGS = {"-exp_dir", "-frequency"} - -@pytest.mark.parametrize( - ["cmd", "missing"], - [ - pytest.param("", {"-exp_dir", "-frequency"}, id="no args"), - pytest.param("-exp_dir /foo/bar", {"-frequency"}, id="no freq"), - pytest.param("-frequency 123", {"-exp_dir"}, id="no dir"), - ], -) - -def test_parser_reqd_args(capsys, cmd, missing): - """Test that the parser reports any missing required arguments""" - parser = get_parser() - - args = cmd.split() - - captured = capsys.readouterr() # throw away existing output - with pytest.raises(SystemExit) as ex: - ns = parser.parse_args(args) - - captured = capsys.readouterr() - assert "the following arguments are required" in captured.err - err_desc = captured.err.split("the following arguments are required:")[-1] - for arg in missing: - assert arg in err_desc - - expected = ALL_ARGS - missing - for exp in expected: - assert exp not in err_desc - - -def test_parser(): - """Test that the parser succeeds when receiving expected args""" - parser = get_parser() - - test_dir = "/foo/bar" - test_freq = 123 - - cmd = f"-exp_dir {test_dir} -frequency {test_freq}" - args = cmd.split() - - ns = parser.parse_args(args) - print(ns) - - assert ns.exp_dir == test_dir - assert ns.frequency == test_freq - - - - def test_parser_move(): """Test that the parser succeeds when receiving expected args""" parser = get_parser() - src_path = "/absolute/file/src/path" dest_path = "/absolute/file/dest/path" - cmd = f"move {src_path} {dest_path}" # must be absolute path - # python + cmd = f"move {src_path} {dest_path}" args = cmd.split() - print(args) ns = parser.parse_args(args) - print(ns) - assert ns.src_path == src_path - assert ns.dest_path == dest_path + assert ns.source == src_path + assert ns.dest == dest_path def test_parser_remove(): """Test that the parser succeeds when receiving expected args""" @@ -481,6 +420,7 @@ def test_parser_remove(): args = cmd.split() ns = parser.parse_args(args) + assert ns.to_remove == file_path def test_parser_symlink(): @@ -495,8 +435,8 @@ def test_parser_symlink(): ns = parser.parse_args(args) - assert ns.source_path == src_path - assert ns.dest_path == dest_path + assert ns.source == src_path + assert ns.dest == dest_path def test_parser_copy(): """Test that the parser succeeds when receiving expected args""" @@ -505,19 +445,16 @@ def test_parser_copy(): src_path = "/absolute/file/src/path" dest_path = "/absolute/file/dest/path" - cmd = f"copy {src_path} {dest_path}" # must be absolute path - # python - args = cmd.split() + cmd = f"copy {src_path} {dest_path}" + args = cmd.split() ns = parser.parse_args(args) - print(ns) - assert ns.source_path == src_path - assert ns.dest_path == dest_path + assert ns.source == src_path + assert ns.dest == dest_path -import json -def test_parser_configure(): +def test_parser_configure_parse(): """Test that the parser succeeds when receiving expected args""" parser = get_parser() @@ -525,37 +462,49 @@ def test_parser_configure(): dest_path = "/absolute/file/dest/path" tag_delimiter = ";" params = '{"5":10}' - -# '{"5": 10}' -# >>> -# {u'value1': u'key1'} + #params = '{"5":10,"FIRST":"SECOND","17":20,"65":"70","placeholder":"group_leftupper_region","1200":"120"}' - cmd = f"configure {src_path} {dest_path} {tag_delimiter} {params}" # must be absolute path - # python + cmd = f"configure {src_path} {dest_path} {tag_delimiter} {params}" args = cmd.split() - ns = parser.parse_args(args) - print(ns) + + assert ns.source == src_path + assert ns.dest == dest_path + assert ns.tag_delimiter == tag_delimiter + assert ns.param_dict == params -#parser.add_argument('-m', '--my-dict', type=str) -#args = parser.parse_args() - print("HIHIHI") -#import json - my_dictionary = json.loads(ns.param_dict) - print(my_dictionary) - print("is thre a type",type(my_dictionary)) - assert ns.source_path == src_path - assert ns.dest_path == dest_path - assert ns.tag_delimiter == tag_delimiter - assert ns.param_dict == params +# TODO: should I add +ALL_ARGS = {"-exp_dir", "-frequency"} +@pytest.mark.parametrize( + ["cmd", "missing"], + [ + pytest.param("", {"-exp_dir", "-frequency"}, id="no args"), + pytest.param("-exp_dir /foo/bar", {"-frequency"}, id="no freq"), + pytest.param("-frequency 123", {"-exp_dir"}, id="no dir"), + ], +) -def test_command_generator_entrypoint(): - ... +def test_parser_reqd_args(capsys, cmd, missing): + """Test that the parser reports any missing required arguments""" parser = get_parser() - - # from smartsim._core.generation import commandgenerator + + args = cmd.split() + + captured = capsys.readouterr() # throw away existing output + with pytest.raises(SystemExit) as ex: + ns = parser.parse_args(args) + + captured = capsys.readouterr() + assert "the following arguments are required" in captured.err + err_desc = captured.err.split("the following arguments are required:")[-1] + for arg in missing: + assert arg in err_desc + + expected = ALL_ARGS - missing + for exp in expected: + assert exp not in err_desc \ No newline at end of file From 578f86b7d608867ae04720e19582b4cd7434b424 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 11 Jul 2024 16:20:57 -0500 Subject: [PATCH 05/17] file operations --- .../_core/entrypoints/command_generator.py | 341 ------------------ smartsim/_core/entrypoints/file_operations.py | 280 +++++--------- smartsim/_core/generation/commandgenerator.py | 274 -------------- ..._generation.py => test_file_operations.py} | 263 ++++++-------- 4 files changed, 198 insertions(+), 960 deletions(-) delete mode 100644 smartsim/_core/entrypoints/command_generator.py delete mode 100644 smartsim/_core/generation/commandgenerator.py rename tests/{test_command_generation.py => test_file_operations.py} (67%) diff --git a/smartsim/_core/entrypoints/command_generator.py b/smartsim/_core/entrypoints/command_generator.py deleted file mode 100644 index d5de62122..000000000 --- a/smartsim/_core/entrypoints/command_generator.py +++ /dev/null @@ -1,341 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2024 Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse -import ast -import os -import typing as t -import shutil -from distutils import dir_util -import re -import collections -import json -from pathlib import Path - - - -# TODO: remove the entity path things and keep it as a helper funtion that -# gives the user to add the application, and then it would take the applciation path and do the copy - -# def get_dest_path(application_path, input_file) -> Path: -# return os.path.join(application_path, os.path.basename(input_file)) - -def _str_to_dict(string): - # remove the curly braces from the string - print("inside str to dict") - #"{'Nikhil", "lastname'", ':', '1,', "'Akshat'", ':', '2,', "'Akash'", ':', '3}' - #'{"Nikhil" : 1, "Akshat" : 2, "Akash" : 3}' - - string = str(string)[1:-1] - print(string) - string = string.strip('{}') - print(string) - - # split the string into key-value pairs - pairs = string.split(", ") - print(pairs) - for pair in pairs: - print(pair) - # "{'Nikhil", "lastname'", ':', '1,', "'Akshat'", ':', '2,', "'Akash'", ':', '3}' - # ['"{\'Nikhil"', '"lastname\'"', "':'", "'1,'", '"\'Akshat\'"', "':'", "'2,'", '"\'Akash\'"', "':'", "'3}'"] - - #"{'Nikhil", "lastname'", ':', '1,', "'Akshat'", ':', '2,', "'Akash'", ':', '3}' - - # use a dictionary comprehension to create - # the dictionary, converting the values to - # integers and removing the quotes from the keys - return {key[1:-2]: int(value) for key, value in (pair.split(': ') for pair in pairs)} - -def _check_path(file_path: str) -> Path: - """Given a user provided path-like str, find the actual path to - the directory or file and create a full path. - - :param file_path: path to a specific file or directory - :raises FileNotFoundError: if file or directory does not exist - :return: full path to file or directory - """ - full_path = os.path.abspath(file_path) - if os.path.isfile(full_path): - return full_path - if os.path.isdir(full_path): - return full_path - raise FileNotFoundError(f"File or Directory {file_path} not found") - - -def move(args) -> None: - """Move a file - - eg. python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path - - :param to_move: Path to a source file to be copied - :param dst_path: Path to a file to copy the contents from the source file into - """ - _check_path(args.source) - _check_path(args.dest) - shutil.move(args.source, args.dest) - - -def remove(args) -> None: - """Write a python script that removes a file when executed. - - eg. python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path - - :param to_remove: Path to the file to be deleted - """ - _check_path(args.to_remove) - os.remove(args.to_remove) - - -def copy(args): - # args.source_path: str, dest_path: str): - """ - Write a python script to copy the entity files and directories attached - to this entity into an entity directory - - eg. python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path - - :param source_path: Path to directory, or path to file to copy into an entity directory - :param dest_path: Path to destination directory or path to destination file to copy - """ - _check_path(args.source) - _check_path(args.dest) - - if os.path.isdir(args.source): - dir_util.copy_tree(args.source, args.dest) - else: - shutil.copyfile(args.source, args.dest) - - -def symlink(args): - """ - Create a symbolic link pointing to the exisiting source file - named link - - eg. python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path - - :param src_file: the exisiting source path - :param dst_file: target name where the symlink will be created. - """ - _check_path(args.source) - - os.symlink(args.source, args.dest) - - -def configure(args): - """Write a python script to set, search and replace the tagged parameters for the configure operation - within tagged files attached to an entity. - - User-formatted files can be attached using the `to_configure` argument. These files will be modified - during ``Model`` generation to replace tagged sections in the user-formatted files with - values from the `params` initializer argument used during ``Model`` creation: - - eg. python _core/entrypoints/file_operations.py configure /absolte/file/src/path /absolte/file/dest/path tagged_deliminator params # base64 encoded dictionary for params - - :param to_configure: The tagged files the search and replace operations to be performed upon - :param dest: Optional destination for configured files to be written to - :param param_dict: A dict of parameter names and values set for the file - :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" - """ - #args.to_configure: str, args.dest_path: t.Optional[str], args.param_dict: t.Dict[str, str], args.tag_delimiter: str = ';' - - - print(args.tag_delimiter) - print(args.param_dict) - print(args.source) - print(args.dest) - - print(type(args.param_dict)) - - param_dict = str(args.param_dict) - - #param_dict = eval(args.param_dict) - - #param_dict = ''.join(args.param_dict) - - #param_dict = json.loads(args.param_dict) - param_dict = _str_to_dict(args.param_dict) - - print("the param dict", param_dict) - - tag_delimiter = ";" - if args.tag_delimiter: - tag_delimiter = args.tag_delimiter - - - - _check_path(args.source) - if args.dest: - _check_path(args.dest) - - - - def _get_prev_value(tagged_line: str) -> str: - split_tag = tagged_line.split(tag_delimiter) - return split_tag[1] - - def _is_ensemble_spec( - tagged_line: str, model_params: dict - ) -> bool: - split_tag = tagged_line.split(tag_delimiter) - prev_val = split_tag[1] - if prev_val in model_params.keys(): - return True - return False - - edited = [] - used_params = {} - #param_dict = {param_dict} - - # tag_delimiter = tag_delimiter - - # Set the tag for the modelwriter to search for within - # tagged files attached to an entity. - regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) - - # Set the lines to iterate over - with open(args.source,'r+', encoding='utf-8') as file_stream: - lines = file_stream.readlines() - - unused_tags = collections.defaultdict(list) - - # Replace the tagged parameters within the file attached to this - # model. The tag defaults to ";" - for i, line in enumerate(lines, 1): - while search := re.search(regex, line): - tagged_line = search.group(0) - previous_value = _get_prev_value(tagged_line) - if _is_ensemble_spec(tagged_line, param_dict): - new_val = str(param_dict[previous_value]) - line = re.sub(regex, new_val, line, 1) - used_params[previous_value] = new_val - - # if a tag_delimiter is found but is not in this model's configurations - # put in placeholder value - else: - tag_delimiter_ = tagged_line.split(tag_delimiter)[1] - unused_tags[tag_delimiter_].append(i) - line = re.sub(regex, previous_value, line) - break - edited.append(line) - - lines = edited - - # write configured file to destination specified. Default is an overwrite - if args.dest: - file_stream = args.dest - - with open(args.source, "w+", encoding="utf-8") as file_stream: - for line in lines: - file_stream.write(line) - - -def get_parser() -> argparse.ArgumentParser: - """Instantiate a parser to process command line arguments - - :returns: An argument parser ready to accept required command generator parameters - """ - arg_parser = argparse.ArgumentParser(description="Command Generator") - - subparsers = arg_parser.add_subparsers(help='file_operations') - - # subparser for move op - move_parser = subparsers.add_parser("move") - move_parser.set_defaults(func=move) - move_parser.add_argument("source") - move_parser.add_argument("dest") - - # subparser for remove op - remove_parser = subparsers.add_parser("remove") - remove_parser.add_argument( - "to_remove", - type = str) - - # subparser for copy op - copy_parser = subparsers.add_parser("copy") - copy_parser.set_defaults(func=copy) - copy_parser.add_argument( - "source", - type = str) - copy_parser.add_argument( - "dest", - type = str) - - # subparser for symlink op - symlink_parser = subparsers.add_parser("symlink") - symlink_parser.set_defaults(func=symlink) - symlink_parser.add_argument( - "source", - type = str) - symlink_parser.add_argument( - "dest", - type = str) - - # subparser for configure op - configure_parser = subparsers.add_parser("configure") - configure_parser.set_defaults(func=configure) - configure_parser.add_argument( - "source", - type = str) - configure_parser.add_argument( - "dest", - type = str) - configure_parser.add_argument( - "tag_delimiter", - type = str) - configure_parser.add_argument( - "param_dict", - type = str, - nargs='+') - - return arg_parser - - -def parse_arguments() -> str: - """Parse the command line arguments - - # :returns: the parsed command line arguments - """ - parser = get_parser() - parsed_args = parser.parse_args() - parsed_args.func(parsed_args) - return parsed_args - - - -if __name__ == "__main__": - """Prepare the telemetry monitor process using command line arguments. - - Sample usage: - python -m smartsim._core.entrypoints.telemetrymonitor -exp_dir - -frequency 30 -cooldown 90 -loglevel INFO - The experiment id is generated during experiment startup - and can be found in the manifest.json in /.smartsim/telemetry - """ - os.environ["PYTHONUNBUFFERED"] = "1" #? - - args = parse_arguments() - - # sys.exit(1) #? diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 48c125ba0..0ddd9f8e3 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -25,19 +25,17 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse +import base64 +import collections import os -import typing as t +import pickle +import re import shutil from distutils import dir_util -import re -import collections +from pathlib import Path -def get_dst_path(application_path, input_file): - return os.path.join(application_path, os.path.basename(input_file)) - - -def _check_path(file_path: str) -> str: +def _check_path(file_path: str) -> Path: """Given a user provided path-like str, find the actual path to the directory or file and create a full path. @@ -53,100 +51,106 @@ def _check_path(file_path: str) -> str: raise FileNotFoundError(f"File or Directory {file_path} not found") - -def move(source: str, dest: str): +def move(args) -> None: """Move a file + Sample usage: + python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path + :param to_move: Path to a source file to be copied :param dst_path: Path to a file to copy the contents from the source file into - :return: string of python code to perform operation that can be executed """ - - _check_path(source) - - shutil.move(source, dest) + _check_path(args.source) + _check_path(args.dest) + shutil.move(args.source, args.dest) -def remove(to_delete: str): +def remove(args) -> None: """Write a python script that removes a file when executed. - :param to_delete: Path to the file to be deleted - :return: string of python code to perform operation that can be executed - """ - _check_path(to_delete) - - os.remove(to_delete) - + Sample usage: + python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path -# TODO: remove the entity path things and keep it as a helper funtion that -# gives the user to add the application, and then it would take the applciation path and do the copy + :param to_remove: Path to the file to be deleted + """ + _check_path(args.to_remove) + os.remove(args.to_remove) def copy(args): - # args.source_path: str, dest_path: str): """ Write a python script to copy the entity files and directories attached to this entity into an entity directory + Sample usage: + python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path + :param source_path: Path to directory, or path to file to copy into an entity directory - :param dest_path: Path to destination directory or path to destination file to copy - :return: string of python code to perform operation that can be executed + :param dest_path: Path to destination directory or path to destination file to copy """ + _check_path(args.source) + _check_path(args.dest) - print("MADE IT INTO THE COPY FUNCTION") - _check_path(args.source_path) - _check_path(args.dest_path) - - if os.path.isdir(args.source_path): - dir_util.copy_tree(args.source_path, args.dest_path) + if os.path.isdir(args.source): + dir_util.copy_tree(args.source, args.dest) else: - shutil.copyfile(args.source_path, args.dest_path) - + shutil.copyfile(args.source, args.dest) -def symlink(source: str, link: str): +def symlink(args): """ Create a symbolic link pointing to the exisiting source file named link + Sample usage: + python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path + :param src_file: the exisiting source path - :param dst_file: target name where the symlink will be created. + :param dst_file: target name where the symlink will be created. """ + _check_path(args.source) - _check_path(source) + os.symlink(args.source, args.dest) - os.symlink(source, link) - - -def configure( - to_configure: str, dest_path: t.Optional[str], param_dict: t.Dict[str, str], tag_delimiter: str = ';' -): +def configure(args): """Write a python script to set, search and replace the tagged parameters for the configure operation within tagged files attached to an entity. - User-formatted files can be attached using the `to_configure` argument. These files will be modified -during ``Model`` generation to replace tagged sections in the user-formatted files with -values from the `params` initializer argument used during ``Model`` creation: + User-formatted files can be attached using the `configure` argument. These files will be modified + during ``Model`` generation to replace tagged sections in the user-formatted files with + values from the `params` initializer argument used during ``Model`` creation: + + Sample usage: + python _core/entrypoints/file_operations.py configure /absolute/file/src/path /absolute/file/dest/path tag_deliminator param_dict :param to_configure: The tagged files the search and replace operations to be performed upon :param dest: Optional destination for configured files to be written to :param param_dict: A dict of parameter names and values set for the file :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" - :return: string of python code to perform operation that can be executed """ - _check_path(to_configure) - if dest_path: - _check_path(dest_path) + _check_path(args.source) + if args.dest: + _check_path(args.dest) + + tag_delimiter = ";" + if args.tag_delimiter: + tag_delimiter = args.tag_delimiter + + decoded_dict = base64.b64decode(eval(args.param_dict)) + param_dict = pickle.loads(decoded_dict) + + if not param_dict: + raise ValueError("param dictionary is empty") + if not isinstance(param_dict, dict): + raise TypeError("param dict is not a valid dictionary") def _get_prev_value(tagged_line: str) -> str: split_tag = tagged_line.split(tag_delimiter) return split_tag[1] - def _is_ensemble_spec( - tagged_line: str, model_params: dict - ) -> bool: + def _is_ensemble_spec(tagged_line: str, model_params: dict) -> bool: split_tag = tagged_line.split(tag_delimiter) prev_val = split_tag[1] if prev_val in model_params.keys(): @@ -155,16 +159,13 @@ def _is_ensemble_spec( edited = [] used_params = {} - #param_dict = {param_dict} - - # tag_delimiter = tag_delimiter # Set the tag for the modelwriter to search for within # tagged files attached to an entity. regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) # Set the lines to iterate over - with open(to_configure,'r+', encoding='utf-8') as file_stream: + with open(args.source, "r+", encoding="utf-8") as file_stream: lines = file_stream.readlines() unused_tags = collections.defaultdict(list) @@ -192,170 +193,69 @@ def _is_ensemble_spec( lines = edited # write configured file to destination specified. Default is an overwrite - if dest_path: - file_stream = dest_path + if args.dest: + file_stream = args.dest - with open(to_configure, "w+", encoding="utf-8") as file_stream: + with open(args.source, "w+", encoding="utf-8") as file_stream: for line in lines: file_stream.write(line) -# python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path -# python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path -# python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path -# python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path -# python _core/entrypoints/file_operations.py configure /absolte/file/src/path /absolte/file/dest/path tagged_deliminator params # base64 encoded dictionary for params - - -import json def get_parser() -> argparse.ArgumentParser: """Instantiate a parser to process command line arguments :returns: An argument parser ready to accept required command generator parameters """ arg_parser = argparse.ArgumentParser(description="Command Generator") - - subparsers = arg_parser.add_subparsers(help='file_operations') - - # subparser for move op + + subparsers = arg_parser.add_subparsers(help="file_operations") + + # Subparser for move op move_parser = subparsers.add_parser("move") move_parser.set_defaults(func=move) - move_parser.add_argument("src_path") - move_parser.add_argument("dest_path") + move_parser.add_argument("source") + move_parser.add_argument("dest") - # subparser for remove op + # Subparser for remove op remove_parser = subparsers.add_parser("remove") - remove_parser.add_argument( - "to_remove", - type = str) + remove_parser.add_argument("to_remove", type=str) - # subparser for copy op + # Subparser for copy op copy_parser = subparsers.add_parser("copy") copy_parser.set_defaults(func=copy) - copy_parser.add_argument( - "source_path", - type = str) - copy_parser.add_argument( - "dest_path", - type = str) - - # subparser for symlink op + copy_parser.add_argument("source", type=str) + copy_parser.add_argument("dest", type=str) + + # Subparser for symlink op symlink_parser = subparsers.add_parser("symlink") symlink_parser.set_defaults(func=symlink) - symlink_parser.add_argument( - "source_path", - type = str) - symlink_parser.add_argument( - "dest_path", - type = str) - - # subparser for configure op + symlink_parser.add_argument("source", type=str) + symlink_parser.add_argument("dest", type=str) + + # Subparser for configure op configure_parser = subparsers.add_parser("configure") configure_parser.set_defaults(func=configure) - configure_parser.add_argument( - "source_path", - type = str) - configure_parser.add_argument( - "dest_path", - type = str) - configure_parser.add_argument( - "tag_delimiter", - type = str) - configure_parser.add_argument( - "param_dict", - type = str) + configure_parser.add_argument("source", type=str) + configure_parser.add_argument("dest", type=str) + configure_parser.add_argument("tag_delimiter", type=str) + configure_parser.add_argument("param_dict", type=str) return arg_parser -def parse_arguments() -> str: # -> TelemetryMonitorArgs: - # """Parse the command line arguments and return an instance - # of TelemetryMonitorArgs populated with the CLI inputs +def parse_arguments() -> str: + """Parse the command line arguments - # :returns: `TelemetryMonitorArgs` instance populated with command line arguments - #""" + :returns: the parsed command line arguments + """ parser = get_parser() parsed_args = parser.parse_args() - parsed_args.func(parsed_args) - #print(parsed_args) - # return TelemetryMonitorArgs( - # parsed_args.exp_dir, - # parsed_args.frequency, - # parsed_args.cooldown, - # parsed_args.loglevel, - # ) - #parsed_args.func(parsed_args) return parsed_args - - if __name__ == "__main__": - """Prepare the telemetry monitor process using command line arguments. - - Sample usage: - python -m smartsim._core.entrypoints.telemetrymonitor -exp_dir - -frequency 30 -cooldown 90 -loglevel INFO - The experiment id is generated during experiment startup - and can be found in the manifest.json in /.smartsim/telemetry - """ + """Run file operations move, remove, symlink, copy, and configure using command line arguments.""" os.environ["PYTHONUNBUFFERED"] = "1" - args = parse_arguments() #JPNOTE get from here - pos args, first one in, the rest will get fed into the rest of the functions - #sys args? - some number of strings that come after - #configure_logger(logger, args.log_level, args.exp_dir) - - print(args) - - print("IN MAIN HERE") - import json - - - - # if arg.copy - # my_dictionary = json.loads(args.my_dict) - - # args = parser.parse_args() - - - # def command1(args): - # print("command1: %s" % args.name) - - # def command2(args): - # print("comamnd2: %s" % args.frequency) - - # if __name__ == '__main__': - # main() - - #ns = parser.parse_args(args) - - # if args[1] == 'remove': - - # to_remove = args[2] - # remove_op(to_remove) - - # if args[1] == 'move': - # to_move = args [2] - # entity_path = args[3] - # move_op(to_move, entity_path) - - # if args[1] == 'symlink': - - # symlink_op() - - # if args[1] == 'copy': - # copy_op() - - # if args[1] == 'configure': - # configure_op() - - - - - - - - - - # sys.exit(1) # do I need? + args = parse_arguments() diff --git a/smartsim/_core/generation/commandgenerator.py b/smartsim/_core/generation/commandgenerator.py deleted file mode 100644 index 2bf2acf7e..000000000 --- a/smartsim/_core/generation/commandgenerator.py +++ /dev/null @@ -1,274 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2024, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import typing as t -from os import path - - -# ## - -# # module -- named function things? -# # open file locally, -# # open up module and run that module - -# - each of these going - -# one single entry point file -# with a main function with an arg parser (takes in at least one arg (the op), all the other args for that function) - -# move_op, arg1, arg2 - -# - no smartsim imports - -# core/entrypoints/file_operations.py - - - - - - - - - -def move_op(to_move: str, dest_file: str): - """Write a python script to moves a file - - :param to_move: Path to a source file to be copied - :param dest_file: Path to a file to copy the contents from the source file into - :return: string of python code to perform operation that can be executed - """ - return f"import shutil;shutil.move('{to_move}','{dest_file}')" - - -def delete_op(to_delete: str): - """Write a python script that deletes a file when executed. - - :param to_delete: Path to the file to be deleted - :return: string of python code to perform operation that can be executed - """ - - if path.isfile(to_delete): - return rf""" -import os -os.remove('{to_delete}') - """ - else: - raise FileNotFoundError(f"{to_delete} is not a valid path") - - -def copy_op(to_copy: str, entity_path: str): - """ - Write a python script to copy the entity files and directories attached - to this entity into an entity directory - - :param to_copy: Path to directory, or path to file to copy into an entity directory - :param entity_path: Path to a directory of an entity - :return: string of python code to perform operation that can be executed - """ - - if not path.exists(entity_path): - raise FileNotFoundError(f"{entity_path} is not a valid path") - elif not path.exists(to_copy): - raise FileNotFoundError(f"{to_copy} is not a valid path") - - else: - - return rf"""import shutil -from distutils import dir_util -from os import path -dst_path = path.join('{entity_path}', path.basename('{to_copy}')) - - -if path.isdir('{to_copy}'): - dir_util.copy_tree('{to_copy}', '{entity_path}') -else: - shutil.copyfile('{to_copy}', dst_path) -""" - - - - -def symlink_op(source: str, dest: str): - """ - Write a python script to make the to_link path a symlink pointing to the entity path. - - :param to_link: path to link - :param entity_path: - :return: string of python code to perform operation that can be executed - """ - # return rf"""import os - import os - - - os.symlink(source, dest) - #""" - - - # dest_path = path.join('{entity_path}', path.basename('{to_link}')) - # os.symlink('{to_link}', dest_path) - - - - - - - - - -# TODO: needs a destination path -- is there a dest file? - -# TODO: make sure im not missing anything -# TODO: application writer - should be captured in the entry point - -def configure_op( - to_configure: str, dest: t.Optional[str], params: t.Dict[str, str], tag_delimiter: str = ';' -): - - - """Write a python script to set, search and replace the tagged parameters for the configure operation - within tagged files attached to an entity. - - :param to configure: - :param params: A dict of parameter names and values set for the file - :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" - :return: string of python code to perform operation that can be executed - """ - - import re - import collections - - def _get_prev_value(tagged_line: str) -> str: - split_tag = tagged_line.split(tag_delimiter) - return split_tag[1] - - def _is_ensemble_spec( - tagged_line: str, model_params: dict - ) -> bool: - split_tag = tagged_line.split(tag_delimiter) - prev_val = split_tag[1] - if prev_val in model_params.keys(): - return True - return False - - edited = [] - used_params = {} - # params = params - - tag_delimiter = tag_delimiter - - regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) - - # Set the lines to iterate over - with open(to_configure,'r+', encoding='utf-8') as file_stream: - lines = file_stream.readlines() - - unused_tags = collections.defaultdict(list) - - # read lines in file - for i, line in enumerate(lines, 1): - while search := re.search(regex, line): - tagged_line = search.group(0) - previous_value = _get_prev_value(tagged_line) - if _is_ensemble_spec(tagged_line, params): - new_val = str(params[previous_value]) - line = re.sub(regex, new_val, line, 1) - used_params[previous_value] = new_val - - # if a tag_delimiter is found but is not in this model's configurations - # put in placeholder value - else: - tag_delimiter_ = tagged_line.split(tag_delimiter)[1] - unused_tags[tag_delimiter_].append(i) - line = re.sub(regex, previous_value, line) - break - edited.append(line) - - lines = edited - - # write to file - if dest: - file_stream = dest - - with open(to_configure, "w+", encoding="utf-8") as file_stream: - for line in lines: - file_stream.write(line) - - -# return rf"""import re -# import collections - -# def _get_prev_value(tagged_line: str) -> str: -# split_tag = tagged_line.split(tag_delimiter) -# return split_tag[1] - -# def _is_ensemble_spec( -# tagged_line: str, model_params: dict -# ) -> bool: -# split_tag = tagged_line.split(tag_delimiter) -# prev_val = split_tag[1] -# if prev_val in model_params.keys(): -# return True -# return False - -# edited = [] -# used_params = {{}} -# params = {params} - -# tag_delimiter = '{tag_delimiter}' - -# regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) - -# # Set the lines to iterate over -# with open('{to_configure}','r+', encoding='utf-8') as file_stream: -# lines = file_stream.readlines() - -# unused_tags = collections.defaultdict(list) - -# # read lines in file -# for i, line in enumerate(lines, 1): -# while search := re.search(regex, line): -# tagged_line = search.group(0) -# previous_value = _get_prev_value(tagged_line) -# if _is_ensemble_spec(tagged_line, params): -# new_val = str(params[previous_value]) -# line = re.sub(regex, new_val, line, 1) -# used_params[previous_value] = new_val - -# # if a tag_delimiter is found but is not in this model's configurations -# # put in placeholder value -# else: -# tag_delimiter_ = tagged_line.split(tag_delimiter)[1] -# unused_tags[tag_delimiter_].append(i) -# line = re.sub(regex, previous_value, line) -# break -# edited.append(line) - -# lines = edited - -# with open('{to_configure}', "w+", encoding="utf-8") as file_stream: -# for line in lines: -# file_stream.write(line) -# """ diff --git a/tests/test_command_generation.py b/tests/test_file_operations.py similarity index 67% rename from tests/test_command_generation.py rename to tests/test_file_operations.py index 1daece861..c2f3c4e4e 100644 --- a/tests/test_command_generation.py +++ b/tests/test_file_operations.py @@ -24,18 +24,19 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import base64 import filecmp import os import pathlib - -import typing as t +import pickle from distutils import dir_util from glob import glob from os import path as osp import pytest -from smartsim._core.entrypoints import command_generator -from smartsim._core.entrypoints.command_generator import get_parser + +from smartsim._core.entrypoints import file_operations +from smartsim._core.entrypoints.file_operations import get_parser test_path = os.path.dirname(os.path.abspath(__file__)) test_output_root = os.path.join(test_path, "tests", "test_output") @@ -45,32 +46,32 @@ def test_symlink_files(): """ Test operation to symlink files """ - + # Set source directory and file source = pathlib.Path(test_output_root) / "sym_source" os.mkdir(source) source_file = pathlib.Path(source) / "sym_source.txt" with open(source_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("") - # entity_path to be the dest dir - entity_path = os.path.join(test_output_root, "entity_name") + # Set path to be the destination directory + entity_path = os.path.join(test_output_root, "entity_name") parser = get_parser() - cmd = f"symlink {source_file} {entity_path}" + cmd = f"symlink {source_file} {entity_path}" args = cmd.split() ns = parser.parse_args(args) - command_generator.symlink(ns) + file_operations.symlink(ns) - link = pathlib.Path(test_output_root) / "entity_name" # Assert the two files are the same file + link = pathlib.Path(test_output_root) / "entity_name" assert link.is_symlink() assert os.readlink(link) == str(source_file) # Clean up the test directory os.unlink(link) os.remove(pathlib.Path(source) / "sym_source.txt") - os.rmdir(pathlib.Path(test_output_root) / "sym_source") + os.rmdir(pathlib.Path(test_output_root) / "sym_source") def test_symlink_dir(): @@ -85,11 +86,11 @@ def test_symlink_dir(): entity_path = os.path.join(test_output_root, "entity_name") parser = get_parser() - cmd = f"symlink {source} {entity_path}" + cmd = f"symlink {source} {entity_path}" args = cmd.split() ns = parser.parse_args(args) - command_generator.symlink(ns) + file_operations.symlink(ns) link = pathlib.Path(test_output_root) / "entity_name" # Assert the two files are the same file @@ -98,7 +99,7 @@ def test_symlink_dir(): # Clean up the test directory os.unlink(link) - os.rmdir(pathlib.Path(test_output_root) / "sym_source") + os.rmdir(pathlib.Path(test_output_root) / "sym_source") def test_copy_op_file(): @@ -119,14 +120,13 @@ def test_copy_op_file(): with open(dest_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("") - parser = get_parser() - cmd = f"copy {source_file} {dest_file}" + cmd = f"copy {source_file} {dest_file}" args = cmd.split() ns = parser.parse_args(args) # Execute copy - command_generator.copy(ns) + file_operations.copy(ns) # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -137,8 +137,7 @@ def test_copy_op_file(): def test_copy_op_dirs(): - """Test the oeprations that copies an entire directory tree source to a new location destination - """ + """Test the oeprations that copies an entire directory tree source to a new location destination""" to_copy = os.path.join(test_output_root, "to_copy") os.mkdir(to_copy) @@ -157,12 +156,12 @@ def test_copy_op_dirs(): os.mkdir(entity_path) parser = get_parser() - cmd = f"copy {to_copy} {entity_path}" + cmd = f"copy {to_copy} {entity_path}" args = cmd.split() ns = parser.parse_args(args) # Execute copy - command_generator.copy(ns) + file_operations.copy(ns) # Clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -174,8 +173,7 @@ def test_copy_op_dirs(): def test_copy_op_bad_source_file(): - """Test that a FileNotFoundError is raised when there is a bad source file - """ + """Test that a FileNotFoundError is raised when there is a bad source file""" to_copy = os.path.join(test_output_root, "to_copy") os.mkdir(to_copy) @@ -184,24 +182,23 @@ def test_copy_op_bad_source_file(): bad_path = "/not/a/real/path" # Execute copy - + parser = get_parser() - cmd = f"copy {bad_path} {entity_path}" + cmd = f"copy {bad_path} {entity_path}" args = cmd.split() ns = parser.parse_args(args) with pytest.raises(FileNotFoundError) as ex: - command_generator.copy(ns) + file_operations.copy(ns) assert f"File or Directory {bad_path} not found" in ex.value.args[0] - + # clean up os.rmdir(pathlib.Path(test_output_root) / "to_copy") os.rmdir(pathlib.Path(test_output_root) / "entity_name") def test_copy_op_bad_dest_path(): - """Test that a FileNotFoundError is raised when there is a bad destination file. - """ + """Test that a FileNotFoundError is raised when there is a bad destination file.""" to_copy = os.path.join(test_output_root, "to_copy") os.mkdir(to_copy) @@ -215,12 +212,12 @@ def test_copy_op_bad_dest_path(): bad_path = "/not/a/real/path" parser = get_parser() - cmd = f"copy {source_file} {bad_path}" + cmd = f"copy {source_file} {bad_path}" args = cmd.split() ns = parser.parse_args(args) with pytest.raises(FileNotFoundError) as ex: - command_generator.copy(ns) + file_operations.copy(ns) assert f"File or Directory {bad_path} not found" in ex.value.args[0] # clean up @@ -237,11 +234,11 @@ def test_move_op(): dest_dir = os.path.join(test_output_root, "to_here") os.mkdir(dest_dir) - #dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") + # dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") dest_file = pathlib.Path(dest_dir) / "to_here.txt" with open(dest_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write(" ") - + source_file = pathlib.Path(source_dir) / "app_move.txt" with open(source_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("dummy") @@ -249,13 +246,13 @@ def test_move_op(): assert osp.exists(source_file) with open(source_file, "r", encoding="utf-8") as dummy_file: assert dummy_file.read() == "dummy" - + parser = get_parser() - cmd = f"move {source_file} {dest_file}" + cmd = f"move {source_file} {dest_file}" args = cmd.split() ns = parser.parse_args(args) - command_generator.move(ns) + file_operations.move(ns) # Assert that the move was successful assert not osp.exists(source_file) @@ -268,6 +265,7 @@ def test_move_op(): os.remove(dest_file) os.rmdir(dest_dir) + def test_remove_op(): """Test the operation to delete a file""" @@ -281,11 +279,11 @@ def test_remove_op(): assert dummy_file.read() == "dummy" parser = get_parser() - cmd = f"remove {to_del}" + cmd = f"remove {to_del}" args = cmd.split() ns = parser.parse_args(args) - command_generator.remove(ns) + file_operations.remove(ns) # Assert file has been deleted assert not osp.exists(to_del) @@ -295,38 +293,46 @@ def test_remove_op_bad_path(): """Test that FileNotFoundError is raised when a bad path is given to the soperation to delete a file""" - test_output_root = os.path.join(test_path, "tests", "test_output") to_del = pathlib.Path(test_output_root) / "not_real.txt" parser = get_parser() - cmd = f"remove {to_del}" + cmd = f"remove {to_del}" args = cmd.split() ns = parser.parse_args(args) with pytest.raises(FileNotFoundError) as ex: - command_generator.remove(ns) + file_operations.remove(ns) assert f"File or Directory {to_del} not found" in ex.value.args[0] -def test_configure_op(test_dir, fileutils): - """Test configure param operations with a tag parameter given""" - - # the param dict for configure operations - # param_dict = { - # "5": 10, # MOM_input - # "FIRST": "SECOND", # example_input.i - # "17": 20, # in.airebo - # "65": "70", # in.atm - # "placeholder": "group leftupper region", # in.crack - # "1200": "120", # input.nml - # } - #"{\"error\":\"Wrong account\"}" - param_dict = '{"5":10,"FIRST":"SECOND","17":20,"65":"70","placeholder":\"group leftupper region\","1200":"120"}' +@pytest.mark.parametrize( + ["param_dict", "error_type"], + [ + pytest.param( + { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + }, + "None", + id="correct dict", + ), + pytest.param( + ["list", "of", "values"], + "TypeError", + id="incorrect dict", + ), + pytest.param({}, "ValueError", id="empty dict"), + ], +) +def test_configure_op(test_dir, fileutils, param_dict, error_type): + """Test configure operation with correct parameter dictionary, empty dicitonary, and an incorrect type""" + tag = ";" - # json loads --> needs to be ' with double quotes inside ' - # but for the arg parse -- needs to have double quotes around everything - # retreive tagged files conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) # retrieve files to compare after test correct_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "correct/")) @@ -338,81 +344,52 @@ def test_configure_op(test_dir, fileutils): tagged_files = sorted(glob(test_dir + "/*")) correct_files = sorted(glob(correct_path + "*")) + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) - parser = get_parser() - cmd = f" configure {tagged_files[0]} {tagged_files[0]} {tag} {param_dict}" - args = cmd.split() - ns = parser.parse_args(args) - print(ns) - - #command_generator.configure(ns) - #Run configure op on test files + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict) + + # Run configure op on test files for tagged_file in tagged_files: - #cmd = f" configure {tagged_file} {tagged_file} {param_dict} {tag}" parser = get_parser() - cmd = f" configure {tagged_file} {tagged_file} {tag} {param_dict}" + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" args = cmd.split() ns = parser.parse_args(args) - command_generator.configure(ns) - #tagged_file, dest=tagged_file, param_dict=param_dict,tag_delimiter=tag - #subprocess.run([sys.executable, "-c", cmd]) - - # check that files and correct files are the same - for written, correct in zip(tagged_files, correct_files): - assert filecmp.cmp(written, correct) - - -def test_configure_op_no_tag(test_dir, fileutils): - """Test configure param operations with no tag parameter given""" - - # the param dict for configure operations - # param_dict = { - # "5": 10, # MOM_input - # "FIRST": "SECOND", # example_input.i - # "17": 20, # in.airebo - # "65": "70", # in.atm - # "placeholder": "group leftupper region", # in.crack - # "1200": "120", # input.nml - # } - param_dict = '{"5":10,"FIRST":"SECOND","17":20,"65":"70","placeholder":"group_leftupper_region","1200":"120"}' - # retreive tagged files - conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) - # retrieve files to compare after test - correct_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "correct/")) - # copy files to test directory - dir_util.copy_tree(conf_path, test_dir) - assert osp.isdir(test_dir) - - tagged_files = sorted(glob(test_dir + "/*")) - correct_files = sorted(glob(correct_path + "*")) - - # Run configure op on test files - for tagged_file in tagged_files: - cmd = command_generator.configure(tagged_file,dest_path=None, param_dict=param_dict) - # subprocess.run([sys.executable, "-c", cmd]) + if error_type == "ValueError": + with pytest.raises(ValueError) as ex: + file_operations.configure(ns) + assert "param dictionary is empty" in ex.value.args[0] + elif error_type == "TypeError": + with pytest.raises(TypeError) as ex: + file_operations.configure(ns) + assert "param dict is not a valid dictionary" in ex.value.args[0] + else: + file_operations.configure(ns) - # check that files and correct files are the same - for written, correct in zip(tagged_files, correct_files): - assert filecmp.cmp(written, correct) + if error_type == "None": + for written, correct in zip(tagged_files, correct_files): + assert filecmp.cmp(written, correct) def test_parser_move(): - """Test that the parser succeeds when receiving expected args""" + """Test that the parser succeeds when receiving expected args for the move operation""" parser = get_parser() src_path = "/absolute/file/src/path" dest_path = "/absolute/file/dest/path" - cmd = f"move {src_path} {dest_path}" + cmd = f"move {src_path} {dest_path}" args = cmd.split() ns = parser.parse_args(args) assert ns.source == src_path assert ns.dest == dest_path + def test_parser_remove(): - """Test that the parser succeeds when receiving expected args""" + """Test that the parser succeeds when receiving expected args for the remove operation""" parser = get_parser() file_path = "/absolute/file/path" @@ -420,11 +397,12 @@ def test_parser_remove(): args = cmd.split() ns = parser.parse_args(args) - + assert ns.to_remove == file_path + def test_parser_symlink(): - """Test that the parser succeeds when receiving expected args""" + """Test that the parser succeeds when receiving expected args for the symlink operation""" parser = get_parser() src_path = "/absolute/file/src/path" @@ -438,14 +416,15 @@ def test_parser_symlink(): assert ns.source == src_path assert ns.dest == dest_path + def test_parser_copy(): - """Test that the parser succeeds when receiving expected args""" + """Test that the parser succeeds when receiving expected args for the copy operation""" parser = get_parser() src_path = "/absolute/file/src/path" dest_path = "/absolute/file/dest/path" - cmd = f"copy {src_path} {dest_path}" + cmd = f"copy {src_path} {dest_path}" args = cmd.split() ns = parser.parse_args(args) @@ -455,56 +434,30 @@ def test_parser_copy(): def test_parser_configure_parse(): - """Test that the parser succeeds when receiving expected args""" + """Test that the parser succeeds when receiving expected args for the configure operation""" parser = get_parser() src_path = "/absolute/file/src/path" dest_path = "/absolute/file/dest/path" tag_delimiter = ";" - params = '{"5":10}' - #params = '{"5":10,"FIRST":"SECOND","17":20,"65":"70","placeholder":"group_leftupper_region","1200":"120"}' + param_dict = { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + } - cmd = f"configure {src_path} {dest_path} {tag_delimiter} {params}" + pickled_dict = pickle.dumps(param_dict) + encoded_dict = base64.b64encode(pickled_dict) + + cmd = f"configure {src_path} {dest_path} {tag_delimiter} {encoded_dict}" args = cmd.split() ns = parser.parse_args(args) assert ns.source == src_path assert ns.dest == dest_path assert ns.tag_delimiter == tag_delimiter - assert ns.param_dict == params - - - - -# TODO: should I add -ALL_ARGS = {"-exp_dir", "-frequency"} - -@pytest.mark.parametrize( - ["cmd", "missing"], - [ - pytest.param("", {"-exp_dir", "-frequency"}, id="no args"), - pytest.param("-exp_dir /foo/bar", {"-frequency"}, id="no freq"), - pytest.param("-frequency 123", {"-exp_dir"}, id="no dir"), - ], -) - -def test_parser_reqd_args(capsys, cmd, missing): - """Test that the parser reports any missing required arguments""" - parser = get_parser() - - args = cmd.split() - - captured = capsys.readouterr() # throw away existing output - with pytest.raises(SystemExit) as ex: - ns = parser.parse_args(args) - - captured = capsys.readouterr() - assert "the following arguments are required" in captured.err - err_desc = captured.err.split("the following arguments are required:")[-1] - for arg in missing: - assert arg in err_desc - - expected = ALL_ARGS - missing - for exp in expected: - assert exp not in err_desc \ No newline at end of file + assert ns.param_dict == str(encoded_dict) From 35e1c05404bd2481288e648b4b13313ac7c724b0 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 11 Jul 2024 16:45:16 -0500 Subject: [PATCH 06/17] lint errors --- smartsim/_core/entrypoints/file_operations.py | 125 ++++++++++-------- .../_core/entrypoints/telemetrymonitor.py | 3 +- 2 files changed, 69 insertions(+), 59 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 0ddd9f8e3..22e98f6cb 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -31,7 +31,8 @@ import pickle import re import shutil -from distutils import dir_util +from ast import literal_eval +from distutils import dir_util # pylint: disable=deprecated-module from pathlib import Path @@ -51,94 +52,103 @@ def _check_path(file_path: str) -> Path: raise FileNotFoundError(f"File or Directory {file_path} not found") -def move(args) -> None: +def move(parsed_args) -> None: """Move a file Sample usage: - python _core/entrypoints/file_operations.py move /absolute/file/src/path /absolute/file/dest/path + python _core/entrypoints/file_operations.py move + /absolute/file/src/path /absolute/file/dest/path - :param to_move: Path to a source file to be copied - :param dst_path: Path to a file to copy the contents from the source file into + source path: Path to a source file to be copied + dest path: Path to a file to copy the contents from the source file into """ - _check_path(args.source) - _check_path(args.dest) - shutil.move(args.source, args.dest) + _check_path(parsed_args.source) + _check_path(parsed_args.dest) + shutil.move(parsed_args.source, parsed_args.dest) -def remove(args) -> None: +def remove(parsed_args) -> None: """Write a python script that removes a file when executed. Sample usage: - python _core/entrypoints/file_operations.py remove /absolute/file/path # must be absolute path + python _core/entrypoints/file_operations.py remove + /absolute/file/path - :param to_remove: Path to the file to be deleted + file path: Path to the file to be deleted """ - _check_path(args.to_remove) - os.remove(args.to_remove) + _check_path(parsed_args.to_remove) + os.remove(parsed_args.to_remove) -def copy(args): +def copy(parsed_args): """ Write a python script to copy the entity files and directories attached to this entity into an entity directory Sample usage: - python _core/entrypoints/file_operations.py copy /absolute/file/src/path /absolute/file/dest/path + python _core/entrypoints/file_operations.py copy + /absolute/file/src/path /absolute/file/dest/path - :param source_path: Path to directory, or path to file to copy into an entity directory - :param dest_path: Path to destination directory or path to destination file to copy + source path: Path to directory, or path to file to copy into an entity directory + dest path: Path to destination directory or path to destination file to copy """ - _check_path(args.source) - _check_path(args.dest) + _check_path(parsed_args.source) + _check_path(parsed_args.dest) - if os.path.isdir(args.source): - dir_util.copy_tree(args.source, args.dest) + if os.path.isdir(parsed_args.source): + dir_util.copy_tree(parsed_args.source, parsed_args.dest) else: - shutil.copyfile(args.source, args.dest) + shutil.copyfile(parsed_args.source, parsed_args.dest) -def symlink(args): +def symlink(parsed_args): """ Create a symbolic link pointing to the exisiting source file named link Sample usage: - python _core/entrypoints/file_operations.py symlink /absolute/file/src/path /absolute/file/dest/path + python _core/entrypoints/file_operations.py symlink + /absolute/file/src/path /absolute/file/dest/path - :param src_file: the exisiting source path - :param dst_file: target name where the symlink will be created. + source path: the exisiting source path + dest path: target name where the symlink will be created. """ - _check_path(args.source) + _check_path(parsed_args.source) - os.symlink(args.source, args.dest) + os.symlink(parsed_args.source, parsed_args.dest) -def configure(args): - """Write a python script to set, search and replace the tagged parameters for the configure operation - within tagged files attached to an entity. +def configure(parsed_args): + """Write a python script to set, search and replace the tagged parameters for the + configure operation within tagged files attached to an entity. - User-formatted files can be attached using the `configure` argument. These files will be modified - during ``Model`` generation to replace tagged sections in the user-formatted files with - values from the `params` initializer argument used during ``Model`` creation: + User-formatted files can be attached using the `configure` argument. These files + will be modified during ``Application`` generation to replace tagged sections in the + user-formatted files with values from the `params` initializer argument used during + ``Application`` creation: Sample usage: - python _core/entrypoints/file_operations.py configure /absolute/file/src/path /absolute/file/dest/path tag_deliminator param_dict + python _core/entrypoints/file_operations.py configure + /absolute/file/src/path /absolute/file/dest/path tag_deliminator param_dict + + source path: The tagged files the search and replace operations to be + performed upon + dest path: Optional destination for configured files to be written to + tag_delimiter: tag for the configure operation to search for, defaults to + semi-colon e.g. ";" + param_dict: A dict of parameter names and values set for the file - :param to_configure: The tagged files the search and replace operations to be performed upon - :param dest: Optional destination for configured files to be written to - :param param_dict: A dict of parameter names and values set for the file - :tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" """ - _check_path(args.source) - if args.dest: - _check_path(args.dest) + _check_path(parsed_args.source) + if parsed_args.dest: + _check_path(parsed_args.dest) tag_delimiter = ";" - if args.tag_delimiter: - tag_delimiter = args.tag_delimiter + if parsed_args.tag_delimiter: + tag_delimiter = parsed_args.tag_delimiter - decoded_dict = base64.b64decode(eval(args.param_dict)) + decoded_dict = base64.b64decode(literal_eval(parsed_args.param_dict)) param_dict = pickle.loads(decoded_dict) if not param_dict: @@ -150,28 +160,28 @@ def _get_prev_value(tagged_line: str) -> str: split_tag = tagged_line.split(tag_delimiter) return split_tag[1] - def _is_ensemble_spec(tagged_line: str, model_params: dict) -> bool: + def _is_ensemble_spec(tagged_line: str, application_params: dict) -> bool: split_tag = tagged_line.split(tag_delimiter) prev_val = split_tag[1] - if prev_val in model_params.keys(): + if prev_val in application_params.keys(): return True return False edited = [] used_params = {} - # Set the tag for the modelwriter to search for within + # Set the tag for the application writer to search for within # tagged files attached to an entity. regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) # Set the lines to iterate over - with open(args.source, "r+", encoding="utf-8") as file_stream: + with open(parsed_args.source, "r+", encoding="utf-8") as file_stream: lines = file_stream.readlines() unused_tags = collections.defaultdict(list) # Replace the tagged parameters within the file attached to this - # model. The tag defaults to ";" + # application. The tag defaults to ";" for i, line in enumerate(lines, 1): while search := re.search(regex, line): tagged_line = search.group(0) @@ -181,8 +191,8 @@ def _is_ensemble_spec(tagged_line: str, model_params: dict) -> bool: line = re.sub(regex, new_val, line, 1) used_params[previous_value] = new_val - # if a tag_delimiter is found but is not in this model's configurations - # put in placeholder value + # if a tag_delimiter is found but is not in this application's + # configurations put in placeholder value else: tag_delimiter_ = tagged_line.split(tag_delimiter)[1] unused_tags[tag_delimiter_].append(i) @@ -193,14 +203,13 @@ def _is_ensemble_spec(tagged_line: str, model_params: dict) -> bool: lines = edited # write configured file to destination specified. Default is an overwrite - if args.dest: - file_stream = args.dest + if parsed_args.dest: + file_stream = parsed_args.dest - with open(args.source, "w+", encoding="utf-8") as file_stream: + with open(parsed_args.source, "w+", encoding="utf-8") as file_stream: for line in lines: file_stream.write(line) - def get_parser() -> argparse.ArgumentParser: """Instantiate a parser to process command line arguments @@ -255,7 +264,9 @@ def parse_arguments() -> str: if __name__ == "__main__": - """Run file operations move, remove, symlink, copy, and configure using command line arguments.""" + """Run file operations move, remove, symlink, copy, and configure + using command line arguments. + """ os.environ["PYTHONUNBUFFERED"] = "1" args = parse_arguments() diff --git a/smartsim/_core/entrypoints/telemetrymonitor.py b/smartsim/_core/entrypoints/telemetrymonitor.py index 95cf8b4d7..5ed1a0c91 100644 --- a/smartsim/_core/entrypoints/telemetrymonitor.py +++ b/smartsim/_core/entrypoints/telemetrymonitor.py @@ -148,8 +148,7 @@ def configure_logger(logger_: logging.Logger, log_level_: int, exp_dir: str) -> """ os.environ["PYTHONUNBUFFERED"] = "1" - args = parse_arguments() #JPNOTE get from here - pos args, first one in, the rest will get fed into the rest of the functions - #sys args? - some number of strings that come after + args = parse_arguments() configure_logger(logger, args.log_level, args.exp_dir) telemetry_monitor = TelemetryMonitor(args) From bfee56abbbf20aaf796b023d6058ebd7f649129a Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 11 Jul 2024 16:48:44 -0500 Subject: [PATCH 07/17] removed print statement --- tests/test_telemetry_monitor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_telemetry_monitor.py b/tests/test_telemetry_monitor.py index cf2db9696..c1bfe2719 100644 --- a/tests/test_telemetry_monitor.py +++ b/tests/test_telemetry_monitor.py @@ -149,7 +149,6 @@ def test_parser(): args = cmd.split() ns = parser.parse_args(args) - print(ns) assert ns.exp_dir == test_dir assert ns.frequency == test_freq From 2eb4221ae088fbcf8b664b6a2ccb791a14d167ef Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 12 Jul 2024 14:06:41 -0500 Subject: [PATCH 08/17] fixed mypy errors --- smartsim/_core/entrypoints/file_operations.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 22e98f6cb..782ffcac8 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -33,10 +33,9 @@ import shutil from ast import literal_eval from distutils import dir_util # pylint: disable=deprecated-module -from pathlib import Path -def _check_path(file_path: str) -> Path: +def _check_path(file_path: str) -> str: """Given a user provided path-like str, find the actual path to the directory or file and create a full path. @@ -52,7 +51,7 @@ def _check_path(file_path: str) -> Path: raise FileNotFoundError(f"File or Directory {file_path} not found") -def move(parsed_args) -> None: +def move(parsed_args: argparse.Namespace) -> None: """Move a file Sample usage: @@ -62,12 +61,13 @@ def move(parsed_args) -> None: source path: Path to a source file to be copied dest path: Path to a file to copy the contents from the source file into """ + print(type(parsed_args)) _check_path(parsed_args.source) _check_path(parsed_args.dest) shutil.move(parsed_args.source, parsed_args.dest) -def remove(parsed_args) -> None: +def remove(parsed_args: argparse.Namespace) -> None: """Write a python script that removes a file when executed. Sample usage: @@ -80,7 +80,7 @@ def remove(parsed_args) -> None: os.remove(parsed_args.to_remove) -def copy(parsed_args): +def copy(parsed_args: argparse.Namespace) -> None: """ Write a python script to copy the entity files and directories attached to this entity into an entity directory @@ -101,7 +101,7 @@ def copy(parsed_args): shutil.copyfile(parsed_args.source, parsed_args.dest) -def symlink(parsed_args): +def symlink(parsed_args: argparse.Namespace) -> None: """ Create a symbolic link pointing to the exisiting source file named link @@ -118,7 +118,7 @@ def symlink(parsed_args): os.symlink(parsed_args.source, parsed_args.dest) -def configure(parsed_args): +def configure(parsed_args: argparse.Namespace) -> None: """Write a python script to set, search and replace the tagged parameters for the configure operation within tagged files attached to an entity. @@ -160,7 +160,7 @@ def _get_prev_value(tagged_line: str) -> str: split_tag = tagged_line.split(tag_delimiter) return split_tag[1] - def _is_ensemble_spec(tagged_line: str, application_params: dict) -> bool: + def _is_ensemble_spec(tagged_line: str, application_params: dict[str,str]) -> bool: split_tag = tagged_line.split(tag_delimiter) prev_val = split_tag[1] if prev_val in application_params.keys(): @@ -252,7 +252,7 @@ def get_parser() -> argparse.ArgumentParser: return arg_parser -def parse_arguments() -> str: +def parse_arguments() -> argparse.Namespace: """Parse the command line arguments :returns: the parsed command line arguments From cd7f1d22675850632b8733d820a2710c21f3f762 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 12 Jul 2024 14:46:22 -0500 Subject: [PATCH 09/17] black fix --- smartsim/_core/entrypoints/file_operations.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 782ffcac8..95721497b 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -160,7 +160,7 @@ def _get_prev_value(tagged_line: str) -> str: split_tag = tagged_line.split(tag_delimiter) return split_tag[1] - def _is_ensemble_spec(tagged_line: str, application_params: dict[str,str]) -> bool: + def _is_ensemble_spec(tagged_line: str, application_params: dict[str, str]) -> bool: split_tag = tagged_line.split(tag_delimiter) prev_val = split_tag[1] if prev_val in application_params.keys(): @@ -210,6 +210,7 @@ def _is_ensemble_spec(tagged_line: str, application_params: dict[str,str]) -> bo for line in lines: file_stream.write(line) + def get_parser() -> argparse.ArgumentParser: """Instantiate a parser to process command line arguments @@ -264,7 +265,7 @@ def parse_arguments() -> argparse.Namespace: if __name__ == "__main__": - """Run file operations move, remove, symlink, copy, and configure + """Run file operations move, remove, symlink, copy, and configure using command line arguments. """ os.environ["PYTHONUNBUFFERED"] = "1" From 8bcd259e2a9416443671a2cad5b0fe17a8827cb6 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 12 Jul 2024 15:16:57 -0500 Subject: [PATCH 10/17] test_output root -> test_dir; cleaning --- smartsim/_core/entrypoints/file_operations.py | 1 - tests/test_file_operations.py | 79 +++++++++---------- 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 95721497b..ec4c6b4f9 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -61,7 +61,6 @@ def move(parsed_args: argparse.Namespace) -> None: source path: Path to a source file to be copied dest path: Path to a file to copy the contents from the source file into """ - print(type(parsed_args)) _check_path(parsed_args.source) _check_path(parsed_args.dest) shutil.move(parsed_args.source, parsed_args.dest) diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index c2f3c4e4e..4f7adbe56 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -38,23 +38,20 @@ from smartsim._core.entrypoints import file_operations from smartsim._core.entrypoints.file_operations import get_parser -test_path = os.path.dirname(os.path.abspath(__file__)) -test_output_root = os.path.join(test_path, "tests", "test_output") - -def test_symlink_files(): +def test_symlink_files(test_dir): """ Test operation to symlink files """ # Set source directory and file - source = pathlib.Path(test_output_root) / "sym_source" + source = pathlib.Path(test_dir) / "sym_source" os.mkdir(source) source_file = pathlib.Path(source) / "sym_source.txt" with open(source_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("") # Set path to be the destination directory - entity_path = os.path.join(test_output_root, "entity_name") + entity_path = os.path.join(test_dir, "entity_name") parser = get_parser() cmd = f"symlink {source_file} {entity_path}" @@ -64,26 +61,26 @@ def test_symlink_files(): file_operations.symlink(ns) # Assert the two files are the same file - link = pathlib.Path(test_output_root) / "entity_name" + link = pathlib.Path(test_dir) / "entity_name" assert link.is_symlink() assert os.readlink(link) == str(source_file) # Clean up the test directory os.unlink(link) os.remove(pathlib.Path(source) / "sym_source.txt") - os.rmdir(pathlib.Path(test_output_root) / "sym_source") + os.rmdir(pathlib.Path(test_dir) / "sym_source") -def test_symlink_dir(): +def test_symlink_dir(test_dir): """ Test operation to symlink directories """ - source = pathlib.Path(test_output_root) / "sym_source" + source = pathlib.Path(test_dir) / "sym_source" os.mkdir(source) # entity_path to be the dest dir - entity_path = os.path.join(test_output_root, "entity_name") + entity_path = os.path.join(test_dir, "entity_name") parser = get_parser() cmd = f"symlink {source} {entity_path}" @@ -92,31 +89,31 @@ def test_symlink_dir(): file_operations.symlink(ns) - link = pathlib.Path(test_output_root) / "entity_name" + link = pathlib.Path(test_dir) / "entity_name" # Assert the two files are the same file assert link.is_symlink() assert os.readlink(link) == str(source) # Clean up the test directory os.unlink(link) - os.rmdir(pathlib.Path(test_output_root) / "sym_source") + os.rmdir(pathlib.Path(test_dir) / "sym_source") -def test_copy_op_file(): +def test_copy_op_file(test_dir): """Test the operation to copy the content of the source file to the destination path with an empty file of the same name already in the directory""" - to_copy = os.path.join(test_output_root, "to_copy") + to_copy = os.path.join(test_dir, "to_copy") os.mkdir(to_copy) source_file = pathlib.Path(to_copy) / "copy_file.txt" with open(source_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("dummy") - entity_path = os.path.join(test_output_root, "entity_name") + entity_path = os.path.join(test_dir, "entity_name") os.mkdir(entity_path) - dest_file = os.path.join(test_output_root, "entity_name", "copy_file.txt") + dest_file = os.path.join(test_dir, "entity_name", "copy_file.txt") with open(dest_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("") @@ -130,16 +127,16 @@ def test_copy_op_file(): # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") - os.rmdir(pathlib.Path(test_output_root) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "to_copy") os.remove(pathlib.Path(entity_path) / "copy_file.txt") - os.rmdir(pathlib.Path(test_output_root) / "entity_name") + os.rmdir(pathlib.Path(test_dir) / "entity_name") -def test_copy_op_dirs(): +def test_copy_op_dirs(test_dir): """Test the oeprations that copies an entire directory tree source to a new location destination""" - to_copy = os.path.join(test_output_root, "to_copy") + to_copy = os.path.join(test_dir, "to_copy") os.mkdir(to_copy) # write some test files in the dir @@ -152,7 +149,7 @@ def test_copy_op_dirs(): dummy_file.write("dummy2") # entity_path to be the dest dir - entity_path = os.path.join(test_output_root, "entity_name") + entity_path = os.path.join(test_dir, "entity_name") os.mkdir(entity_path) parser = get_parser() @@ -166,18 +163,18 @@ def test_copy_op_dirs(): # Clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") os.remove(pathlib.Path(to_copy) / "copy_file_2.txt") - os.rmdir(pathlib.Path(test_output_root) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "to_copy") os.remove(pathlib.Path(entity_path) / "copy_file.txt") os.remove(pathlib.Path(entity_path) / "copy_file_2.txt") - os.rmdir(pathlib.Path(test_output_root) / "entity_name") + os.rmdir(pathlib.Path(test_dir) / "entity_name") -def test_copy_op_bad_source_file(): +def test_copy_op_bad_source_file(test_dir): """Test that a FileNotFoundError is raised when there is a bad source file""" - to_copy = os.path.join(test_output_root, "to_copy") + to_copy = os.path.join(test_dir, "to_copy") os.mkdir(to_copy) - entity_path = os.path.join(test_output_root, "entity_name") + entity_path = os.path.join(test_dir, "entity_name") os.mkdir(entity_path) bad_path = "/not/a/real/path" @@ -193,20 +190,20 @@ def test_copy_op_bad_source_file(): assert f"File or Directory {bad_path} not found" in ex.value.args[0] # clean up - os.rmdir(pathlib.Path(test_output_root) / "to_copy") - os.rmdir(pathlib.Path(test_output_root) / "entity_name") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") -def test_copy_op_bad_dest_path(): +def test_copy_op_bad_dest_path(test_dir): """Test that a FileNotFoundError is raised when there is a bad destination file.""" - to_copy = os.path.join(test_output_root, "to_copy") + to_copy = os.path.join(test_dir, "to_copy") os.mkdir(to_copy) source_file = pathlib.Path(to_copy) / "copy_file.txt" with open(source_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write("dummy1") - entity_path = os.path.join(test_output_root, "entity_name") + entity_path = os.path.join(test_dir, "entity_name") os.mkdir(entity_path) bad_path = "/not/a/real/path" @@ -222,16 +219,16 @@ def test_copy_op_bad_dest_path(): # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") - os.rmdir(pathlib.Path(test_output_root) / "to_copy") - os.rmdir(pathlib.Path(test_output_root) / "entity_name") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") -def test_move_op(): +def test_move_op(test_dir): """Test the operation to move a file""" - source_dir = os.path.join(test_output_root, "from_here") + source_dir = os.path.join(test_dir, "from_here") os.mkdir(source_dir) - dest_dir = os.path.join(test_output_root, "to_here") + dest_dir = os.path.join(test_dir, "to_here") os.mkdir(dest_dir) # dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") @@ -266,11 +263,11 @@ def test_move_op(): os.rmdir(dest_dir) -def test_remove_op(): +def test_remove_op(test_dir): """Test the operation to delete a file""" # Make a test file with dummy text - to_del = pathlib.Path(test_output_root) / "app_del.txt" + to_del = pathlib.Path(test_dir) / "app_del.txt" with open(to_del, "w+", encoding="utf-8") as dummy_file: dummy_file.write("dummy") @@ -289,11 +286,11 @@ def test_remove_op(): assert not osp.exists(to_del) -def test_remove_op_bad_path(): +def test_remove_op_bad_path(test_dir): """Test that FileNotFoundError is raised when a bad path is given to the soperation to delete a file""" - to_del = pathlib.Path(test_output_root) / "not_real.txt" + to_del = pathlib.Path(test_dir) / "not_real.txt" parser = get_parser() cmd = f"remove {to_del}" From 240d4539fac245010bc060add7d2668d1a845ba2 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 15 Jul 2024 18:42:04 -0500 Subject: [PATCH 11/17] matt and amanda review comments --- smartsim/_core/entrypoints/file_operations.py | 91 +++++++------------ tests/test_file_operations.py | 47 ++++++++-- 2 files changed, 74 insertions(+), 64 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index ec4c6b4f9..a93413206 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -35,65 +35,50 @@ from distutils import dir_util # pylint: disable=deprecated-module -def _check_path(file_path: str) -> str: - """Given a user provided path-like str, find the actual path to - the directory or file and create a full path. - - :param file_path: path to a specific file or directory - :raises FileNotFoundError: if file or directory does not exist - :return: full path to file or directory - """ - full_path = os.path.abspath(file_path) - if os.path.isfile(full_path): - return full_path - if os.path.isdir(full_path): - return full_path - raise FileNotFoundError(f"File or Directory {file_path} not found") - - def move(parsed_args: argparse.Namespace) -> None: - """Move a file + """Move a source file or directory to another location. If dest is an + existing directory or a symlink to a directory, then the srouce will + be moved inside that directory. The destination path in that directory + must not already exist. If dest is an existing file, it will be overwritten. Sample usage: python _core/entrypoints/file_operations.py move - /absolute/file/src/path /absolute/file/dest/path + /absolute/file/source/path /absolute/file/dest/path - source path: Path to a source file to be copied - dest path: Path to a file to copy the contents from the source file into + /absolute/file/source/path: File or directory to be moved + /absolute/file/dest/path: Path to a file or directory location """ - _check_path(parsed_args.source) - _check_path(parsed_args.dest) shutil.move(parsed_args.source, parsed_args.dest) def remove(parsed_args: argparse.Namespace) -> None: - """Write a python script that removes a file when executed. + """Remove a file or directory. Sample usage: python _core/entrypoints/file_operations.py remove /absolute/file/path - file path: Path to the file to be deleted + /absolute/file/path: Path to the file or directory to be deleted """ - _check_path(parsed_args.to_remove) - os.remove(parsed_args.to_remove) + if os.path.isdir(parsed_args.to_remove): + os.rmdir(parsed_args.to_remove) + else: + os.remove(parsed_args.to_remove) def copy(parsed_args: argparse.Namespace) -> None: - """ - Write a python script to copy the entity files and directories attached - to this entity into an entity directory + """Copy the contents from the source file into the dest file. + If source is a directory, copy the entire directory tree source to dest. Sample usage: python _core/entrypoints/file_operations.py copy - /absolute/file/src/path /absolute/file/dest/path + /absolute/file/source/path /absolute/file/dest/path - source path: Path to directory, or path to file to copy into an entity directory - dest path: Path to destination directory or path to destination file to copy + /absolute/file/source/path: Path to directory, or path to file to + copy to a new location + /absolute/file/dest/path: Path to destination directory or path to + destination file """ - _check_path(parsed_args.source) - _check_path(parsed_args.dest) - if os.path.isdir(parsed_args.source): dir_util.copy_tree(parsed_args.source, parsed_args.dest) else: @@ -103,46 +88,40 @@ def copy(parsed_args: argparse.Namespace) -> None: def symlink(parsed_args: argparse.Namespace) -> None: """ Create a symbolic link pointing to the exisiting source file - named link + named link. Sample usage: python _core/entrypoints/file_operations.py symlink - /absolute/file/src/path /absolute/file/dest/path + /absolute/file/source/path /absolute/file/dest/path - source path: the exisiting source path - dest path: target name where the symlink will be created. + /absolute/file/source/path: the exisiting source path + /absolute/file/dest/path: target name where the symlink will be created. """ - _check_path(parsed_args.source) - os.symlink(parsed_args.source, parsed_args.dest) def configure(parsed_args: argparse.Namespace) -> None: - """Write a python script to set, search and replace the tagged parameters for the - configure operation within tagged files attached to an entity. + """Write a python script to set, search and replace the tagged parameters + for the configure operation within tagged files attached to an entity. - User-formatted files can be attached using the `configure` argument. These files - will be modified during ``Application`` generation to replace tagged sections in the - user-formatted files with values from the `params` initializer argument used during - ``Application`` creation: + User-formatted files can be attached using the `configure` argument. + These files will be modified during ``Application`` generation to replace + tagged sections in the user-formatted files with values from the `params` + initializer argument used during ``Application`` creation: Sample usage: python _core/entrypoints/file_operations.py configure - /absolute/file/src/path /absolute/file/dest/path tag_deliminator param_dict + /absolute/file/source/pat /absolute/file/dest/path tag_deliminator param_dict - source path: The tagged files the search and replace operations to be - performed upon - dest path: Optional destination for configured files to be written to + /absolute/file/source/path: The tagged files the search and replace operations + to be performed upon + /absolute/file/dest/path: Optional destination for configured files to be + written to. tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" param_dict: A dict of parameter names and values set for the file """ - - _check_path(parsed_args.source) - if parsed_args.dest: - _check_path(parsed_args.dest) - tag_delimiter = ";" if parsed_args.tag_delimiter: tag_delimiter = parsed_args.tag_delimiter diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 4f7adbe56..53160a4d9 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -46,9 +46,9 @@ def test_symlink_files(test_dir): # Set source directory and file source = pathlib.Path(test_dir) / "sym_source" os.mkdir(source) - source_file = pathlib.Path(source) / "sym_source.txt" + source_file = source / "sym_source.txt" with open(source_file, "w+", encoding="utf-8") as dummy_file: - dummy_file.write("") + dummy_file.write("dummy") # Set path to be the destination directory entity_path = os.path.join(test_dir, "entity_name") @@ -125,7 +125,11 @@ def test_copy_op_file(test_dir): # Execute copy file_operations.copy(ns) - # clean up + # Assert files were copied over + with open(dest_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + # Clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") os.rmdir(pathlib.Path(test_dir) / "to_copy") @@ -160,6 +164,15 @@ def test_copy_op_dirs(test_dir): # Execute copy file_operations.copy(ns) + # Assert dirs were copied over + entity_files_1 = pathlib.Path(entity_path) / "copy_file.txt" + with open(entity_files_1, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy1" + + entity_files_2 = pathlib.Path(entity_path) / "copy_file_2.txt" + with open(entity_files_2, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy2" + # Clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") os.remove(pathlib.Path(to_copy) / "copy_file_2.txt") @@ -187,9 +200,9 @@ def test_copy_op_bad_source_file(test_dir): with pytest.raises(FileNotFoundError) as ex: file_operations.copy(ns) - assert f"File or Directory {bad_path} not found" in ex.value.args[0] + assert f"No such file or directory" in ex.value.args - # clean up + # Clean up os.rmdir(pathlib.Path(test_dir) / "to_copy") os.rmdir(pathlib.Path(test_dir) / "entity_name") @@ -215,7 +228,7 @@ def test_copy_op_bad_dest_path(test_dir): with pytest.raises(FileNotFoundError) as ex: file_operations.copy(ns) - assert f"File or Directory {bad_path} not found" in ex.value.args[0] + assert f"No such file or directory" in ex.value.args # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -263,7 +276,7 @@ def test_move_op(test_dir): os.rmdir(dest_dir) -def test_remove_op(test_dir): +def test_remove_op_file(test_dir): """Test the operation to delete a file""" # Make a test file with dummy text @@ -286,6 +299,24 @@ def test_remove_op(test_dir): assert not osp.exists(to_del) +def test_remove_op_dir(test_dir): + """Test the operation to delete a directory""" + + # Make a test file with dummy text + to_del = pathlib.Path(test_dir) / "dir_del" + os.mkdir(to_del) + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.remove(ns) + + # Assert directory has been deleted + assert not osp.exists(to_del) + + def test_remove_op_bad_path(test_dir): """Test that FileNotFoundError is raised when a bad path is given to the soperation to delete a file""" @@ -299,7 +330,7 @@ def test_remove_op_bad_path(test_dir): with pytest.raises(FileNotFoundError) as ex: file_operations.remove(ns) - assert f"File or Directory {to_del} not found" in ex.value.args[0] + assert f"No such file or directory" in ex.value.args @pytest.mark.parametrize( From 9f6d50ff6067390da9496d8b6dbb7a56bcd80b07 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 16 Jul 2024 17:19:54 -0500 Subject: [PATCH 12/17] absolute path check, cleaning --- smartsim/_core/entrypoints/file_operations.py | 34 ++++- tests/test_file_operations.py | 125 +++++++++++++++++- 2 files changed, 151 insertions(+), 8 deletions(-) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index a93413206..8c428d6ae 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -48,6 +48,12 @@ def move(parsed_args: argparse.Namespace) -> None: /absolute/file/source/path: File or directory to be moved /absolute/file/dest/path: Path to a file or directory location """ + if not os.path.isabs(parsed_args.source): + raise ValueError(f"path {parsed_args.source} must be absolute") + + if not os.path.isabs(parsed_args.dest): + raise ValueError(f"path {parsed_args.dest} must be absolute") + shutil.move(parsed_args.source, parsed_args.dest) @@ -60,6 +66,9 @@ def remove(parsed_args: argparse.Namespace) -> None: /absolute/file/path: Path to the file or directory to be deleted """ + if not os.path.isabs(parsed_args.to_remove): + raise ValueError(f"path {parsed_args.to_remove} must be absolute") + if os.path.isdir(parsed_args.to_remove): os.rmdir(parsed_args.to_remove) else: @@ -79,6 +88,12 @@ def copy(parsed_args: argparse.Namespace) -> None: /absolute/file/dest/path: Path to destination directory or path to destination file """ + if not os.path.isabs(parsed_args.source): + raise ValueError(f"path {parsed_args.source} must be absolute") + + if not os.path.isabs(parsed_args.dest): + raise ValueError(f"path {parsed_args.dest} must be absolute") + if os.path.isdir(parsed_args.source): dir_util.copy_tree(parsed_args.source, parsed_args.dest) else: @@ -97,12 +112,18 @@ def symlink(parsed_args: argparse.Namespace) -> None: /absolute/file/source/path: the exisiting source path /absolute/file/dest/path: target name where the symlink will be created. """ + if not os.path.isabs(parsed_args.source): + raise ValueError(f"path {parsed_args.source} must be absolute") + + if not os.path.isabs(parsed_args.dest): + raise ValueError(f"path {parsed_args.dest} must be absolute") + os.symlink(parsed_args.source, parsed_args.dest) def configure(parsed_args: argparse.Namespace) -> None: - """Write a python script to set, search and replace the tagged parameters - for the configure operation within tagged files attached to an entity. + """Set, search and replace the tagged parameters for the + configure operation within tagged files attached to an entity. User-formatted files can be attached using the `configure` argument. These files will be modified during ``Application`` generation to replace @@ -115,13 +136,19 @@ def configure(parsed_args: argparse.Namespace) -> None: /absolute/file/source/path: The tagged files the search and replace operations to be performed upon - /absolute/file/dest/path: Optional destination for configured files to be + /absolute/file/dest/path: The destination for configured files to be written to. tag_delimiter: tag for the configure operation to search for, defaults to semi-colon e.g. ";" param_dict: A dict of parameter names and values set for the file """ + if not os.path.isabs(parsed_args.source): + raise ValueError(f"path {parsed_args.source} must be absolute") + + if not os.path.isabs(parsed_args.dest): + raise ValueError(f"path {parsed_args.dest} must be absolute") + tag_delimiter = ";" if parsed_args.tag_delimiter: tag_delimiter = parsed_args.tag_delimiter @@ -206,6 +233,7 @@ def get_parser() -> argparse.ArgumentParser: # Subparser for remove op remove_parser = subparsers.add_parser("remove") + remove_parser.set_defaults(func=remove) remove_parser.add_argument("to_remove", type=str) # Subparser for copy op diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 53160a4d9..36624fd26 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -99,6 +99,30 @@ def test_symlink_dir(test_dir): os.rmdir(pathlib.Path(test_dir) / "sym_source") +def test_symlink_not_absolute(test_dir): + """Test that ValueError is raised when a relative path + is given to the symlink operation + """ + # Set source directory and file + source = pathlib.Path(test_dir) / "sym_source" + os.mkdir(source) + source_file = source / "sym_source.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + # Set path to be the destination directory + entity_path = ".." + + parser = get_parser() + cmd = f"symlink {source_file} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(ValueError) as ex: + file_operations.symlink(ns) + assert f"path {entity_path} must be absolute" in ex.value.args + + def test_copy_op_file(test_dir): """Test the operation to copy the content of the source file to the destination path with an empty file of the same name already in the directory""" @@ -200,7 +224,7 @@ def test_copy_op_bad_source_file(test_dir): with pytest.raises(FileNotFoundError) as ex: file_operations.copy(ns) - assert f"No such file or directory" in ex.value.args + assert "No such file or directory" in ex.value.args # Clean up os.rmdir(pathlib.Path(test_dir) / "to_copy") @@ -228,7 +252,35 @@ def test_copy_op_bad_dest_path(test_dir): with pytest.raises(FileNotFoundError) as ex: file_operations.copy(ns) - assert f"No such file or directory" in ex.value.args + assert "No such file or directory" in ex.value.args + + # clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_copy_not_absolute(test_dir): + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + bad_path = ".." + + parser = get_parser() + cmd = f"copy {source_file} {bad_path}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(ValueError) as ex: + file_operations.copy(ns) + assert f"path {bad_path} must be absolute" in ex.value.args # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -244,7 +296,6 @@ def test_move_op(test_dir): dest_dir = os.path.join(test_dir, "to_here") os.mkdir(dest_dir) - # dest_file = os.path.join(test_output_root, "to_here", "to_here.txt") dest_file = pathlib.Path(dest_dir) / "to_here.txt" with open(dest_file, "w+", encoding="utf-8") as dummy_file: dummy_file.write(" ") @@ -276,6 +327,31 @@ def test_move_op(test_dir): os.rmdir(dest_dir) +def test_move_not_absolute(test_dir): + """Test that a ValueError is raised when a relative + path is given to the move operation""" + + source_dir = os.path.join(test_dir, "from_here") + os.mkdir(source_dir) + dest_dir = os.path.join(test_dir, "to_here") + os.mkdir(dest_dir) + + dest_file = ".." + + source_file = pathlib.Path(source_dir) / "app_move.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + parser = get_parser() + cmd = f"move {source_file} {dest_file}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(ValueError) as ex: + file_operations.move(ns) + assert f"path {dest_file} must be absolute" in ex.value.args + + def test_remove_op_file(test_dir): """Test the operation to delete a file""" @@ -319,7 +395,7 @@ def test_remove_op_dir(test_dir): def test_remove_op_bad_path(test_dir): """Test that FileNotFoundError is raised when a bad path is given to the - soperation to delete a file""" + operation to delete a file""" to_del = pathlib.Path(test_dir) / "not_real.txt" @@ -330,7 +406,23 @@ def test_remove_op_bad_path(test_dir): with pytest.raises(FileNotFoundError) as ex: file_operations.remove(ns) - assert f"No such file or directory" in ex.value.args + assert "No such file or directory" in ex.value.args + + +def test_remove_op_no_absolute(): + """Test that ValueError is raised when a relative path + is given to the operation to delete a file""" + + to_del = ".." + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(ValueError) as ex: + file_operations.remove(ns) + assert f"path {to_del} must be absolute" in ex.value.args @pytest.mark.parametrize( @@ -401,6 +493,29 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): assert filecmp.cmp(written, correct) +def test_configure_not_absolute(): + """Test that ValueError is raised when tagged files + given to configure op are not absolute paths + """ + + tagged_file = ".." + tag = ";" + param_dict = {"5": 10} + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict) + parser = get_parser() + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(ValueError) as ex: + file_operations.move(ns) + assert f"path {tagged_file} must be absolute" in ex.value.args + + def test_parser_move(): """Test that the parser succeeds when receiving expected args for the move operation""" parser = get_parser() From e1b54e6796ed77b3a810568b2cdd61e10218417d Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Wed, 17 Jul 2024 18:10:59 -0500 Subject: [PATCH 13/17] invalid tag test, configure clean, value error clean --- smartsim/_core/entrypoints/file_operations.py | 202 ++++++++---------- .../tagged_tests/correct/invalidtag.txt | 3 + .../tagged_tests/marked/invalidtag.txt | 3 + tests/test_file_operations.py | 107 +++++++--- 4 files changed, 170 insertions(+), 145 deletions(-) create mode 100644 tests/test_configs/tagged_tests/correct/invalidtag.txt create mode 100644 tests/test_configs/tagged_tests/marked/invalidtag.txt diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 8c428d6ae..0e8352e35 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -27,12 +27,29 @@ import argparse import base64 import collections +import functools import os +import pathlib import pickle import re import shutil -from ast import literal_eval -from distutils import dir_util # pylint: disable=deprecated-module + +from typing import Callable + +from ...log import get_logger + +logger = get_logger(__name__) + +"""Run file operations move, remove, symlink, copy, and configure +using command line arguments. +""" + + +def _abspath(s: str) -> pathlib.Path: + p = pathlib.Path(s) + if not p.is_absolute(): + raise ValueError(f"path `{p}` must be absolute") + return p def move(parsed_args: argparse.Namespace) -> None: @@ -42,18 +59,14 @@ def move(parsed_args: argparse.Namespace) -> None: must not already exist. If dest is an existing file, it will be overwritten. Sample usage: - python _core/entrypoints/file_operations.py move - /absolute/file/source/path /absolute/file/dest/path + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + move /absolute/file/source/path /absolute/file/dest/path - /absolute/file/source/path: File or directory to be moved - /absolute/file/dest/path: Path to a file or directory location + /absolute/file/source/path: File or directory to be moved + /absolute/file/dest/path: Path to a file or directory location """ - if not os.path.isabs(parsed_args.source): - raise ValueError(f"path {parsed_args.source} must be absolute") - - if not os.path.isabs(parsed_args.dest): - raise ValueError(f"path {parsed_args.dest} must be absolute") - shutil.move(parsed_args.source, parsed_args.dest) @@ -61,14 +74,13 @@ def remove(parsed_args: argparse.Namespace) -> None: """Remove a file or directory. Sample usage: - python _core/entrypoints/file_operations.py remove - /absolute/file/path + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + remove /absolute/file/path - /absolute/file/path: Path to the file or directory to be deleted + /absolute/file/path: Path to the file or directory to be deleted """ - if not os.path.isabs(parsed_args.to_remove): - raise ValueError(f"path {parsed_args.to_remove} must be absolute") - if os.path.isdir(parsed_args.to_remove): os.rmdir(parsed_args.to_remove) else: @@ -80,22 +92,18 @@ def copy(parsed_args: argparse.Namespace) -> None: If source is a directory, copy the entire directory tree source to dest. Sample usage: - python _core/entrypoints/file_operations.py copy - /absolute/file/source/path /absolute/file/dest/path - - /absolute/file/source/path: Path to directory, or path to file to - copy to a new location - /absolute/file/dest/path: Path to destination directory or path to - destination file + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + copy /absolute/file/source/path /absolute/file/dest/path + + /absolute/file/source/path: Path to directory, or path to file to + copy to a new location + /absolute/file/dest/path: Path to destination directory or path to + destination file """ - if not os.path.isabs(parsed_args.source): - raise ValueError(f"path {parsed_args.source} must be absolute") - - if not os.path.isabs(parsed_args.dest): - raise ValueError(f"path {parsed_args.dest} must be absolute") - if os.path.isdir(parsed_args.source): - dir_util.copy_tree(parsed_args.source, parsed_args.dest) + shutil.copytree(parsed_args.source, parsed_args.dest, dirs_exist_ok=True) else: shutil.copyfile(parsed_args.source, parsed_args.dest) @@ -106,18 +114,14 @@ def symlink(parsed_args: argparse.Namespace) -> None: named link. Sample usage: - python _core/entrypoints/file_operations.py symlink - /absolute/file/source/path /absolute/file/dest/path + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + symlink /absolute/file/source/path /absolute/file/dest/path - /absolute/file/source/path: the exisiting source path - /absolute/file/dest/path: target name where the symlink will be created. + /absolute/file/source/path: the exisiting source path + /absolute/file/dest/path: target name where the symlink will be created. """ - if not os.path.isabs(parsed_args.source): - raise ValueError(f"path {parsed_args.source} must be absolute") - - if not os.path.isabs(parsed_args.dest): - raise ValueError(f"path {parsed_args.dest} must be absolute") - os.symlink(parsed_args.source, parsed_args.dest) @@ -131,29 +135,24 @@ def configure(parsed_args: argparse.Namespace) -> None: initializer argument used during ``Application`` creation: Sample usage: - python _core/entrypoints/file_operations.py configure - /absolute/file/source/pat /absolute/file/dest/path tag_deliminator param_dict - - /absolute/file/source/path: The tagged files the search and replace operations - to be performed upon - /absolute/file/dest/path: The destination for configured files to be - written to. - tag_delimiter: tag for the configure operation to search for, defaults to - semi-colon e.g. ";" - param_dict: A dict of parameter names and values set for the file + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + configure /absolute/file/source/pat /absolute/file/dest/path \ + tag_deliminator param_dict + + /absolute/file/source/path: The tagged files the search and replace operations + to be performed upon + /absolute/file/dest/path: The destination for configured files to be + written to. + tag_delimiter: tag for the configure operation to search for, defaults to + semi-colon e.g. ";" + param_dict: A dict of parameter names and values set for the file """ - if not os.path.isabs(parsed_args.source): - raise ValueError(f"path {parsed_args.source} must be absolute") + tag_delimiter = parsed_args.tag_delimiter - if not os.path.isabs(parsed_args.dest): - raise ValueError(f"path {parsed_args.dest} must be absolute") - - tag_delimiter = ";" - if parsed_args.tag_delimiter: - tag_delimiter = parsed_args.tag_delimiter - - decoded_dict = base64.b64decode(literal_eval(parsed_args.param_dict)) + decoded_dict = base64.b64decode(parsed_args.param_dict) param_dict = pickle.loads(decoded_dict) if not param_dict: @@ -165,19 +164,17 @@ def _get_prev_value(tagged_line: str) -> str: split_tag = tagged_line.split(tag_delimiter) return split_tag[1] + def make_substitution(tag_name: str, replacement: str) -> Callable[[str], str]: + return lambda s: s.replace( + f"{tag_delimiter}{tag_name}{tag_delimiter}", str(replacement) + ) + def _is_ensemble_spec(tagged_line: str, application_params: dict[str, str]) -> bool: - split_tag = tagged_line.split(tag_delimiter) - prev_val = split_tag[1] - if prev_val in application_params.keys(): - return True - return False + prev_val = _get_prev_value(tagged_line) + return prev_val in application_params edited = [] - used_params = {} - - # Set the tag for the application writer to search for within - # tagged files attached to an entity. - regex = "".join(("(", tag_delimiter, ".+", tag_delimiter, ")")) + # used_params = {} # Set the lines to iterate over with open(parsed_args.source, "r+", encoding="utf-8") as file_stream: @@ -185,35 +182,19 @@ def _is_ensemble_spec(tagged_line: str, application_params: dict[str, str]) -> b unused_tags = collections.defaultdict(list) - # Replace the tagged parameters within the file attached to this - # application. The tag defaults to ";" - for i, line in enumerate(lines, 1): - while search := re.search(regex, line): - tagged_line = search.group(0) - previous_value = _get_prev_value(tagged_line) - if _is_ensemble_spec(tagged_line, param_dict): - new_val = str(param_dict[previous_value]) - line = re.sub(regex, new_val, line, 1) - used_params[previous_value] = new_val - - # if a tag_delimiter is found but is not in this application's - # configurations put in placeholder value - else: - tag_delimiter_ = tagged_line.split(tag_delimiter)[1] - unused_tags[tag_delimiter_].append(i) - line = re.sub(regex, previous_value, line) - break - edited.append(line) + substitutions = tuple(make_substitution(k, v) for k, v in param_dict.items()) + replace_tags_in = lambda s: functools.reduce(lambda a, fn: fn(a), substitutions, s) - lines = edited + edited = [replace_tags_in(line) for line in lines] - # write configured file to destination specified. Default is an overwrite - if parsed_args.dest: - file_stream = parsed_args.dest + for tag, value in unused_tags.items(): + missing_tag_message = f"Unused tag {tag} on line(s): {str(value)}" + logger.warning(missing_tag_message) + lines = edited - with open(parsed_args.source, "w+", encoding="utf-8") as file_stream: - for line in lines: - file_stream.write(line) + # write configured file to destination specified + with open(parsed_args.dest, "w+", encoding="utf-8") as file_stream: + file_stream.writelines(lines) def get_parser() -> argparse.ArgumentParser: @@ -228,32 +209,32 @@ def get_parser() -> argparse.ArgumentParser: # Subparser for move op move_parser = subparsers.add_parser("move") move_parser.set_defaults(func=move) - move_parser.add_argument("source") - move_parser.add_argument("dest") + move_parser.add_argument("source", type=_abspath) + move_parser.add_argument("dest", type=_abspath) # Subparser for remove op remove_parser = subparsers.add_parser("remove") remove_parser.set_defaults(func=remove) - remove_parser.add_argument("to_remove", type=str) + remove_parser.add_argument("to_remove", type=_abspath) # Subparser for copy op copy_parser = subparsers.add_parser("copy") copy_parser.set_defaults(func=copy) - copy_parser.add_argument("source", type=str) - copy_parser.add_argument("dest", type=str) + copy_parser.add_argument("source", type=_abspath) + copy_parser.add_argument("dest", type=_abspath) # Subparser for symlink op symlink_parser = subparsers.add_parser("symlink") symlink_parser.set_defaults(func=symlink) - symlink_parser.add_argument("source", type=str) - symlink_parser.add_argument("dest", type=str) + symlink_parser.add_argument("source", type=_abspath) + symlink_parser.add_argument("dest", type=_abspath) # Subparser for configure op configure_parser = subparsers.add_parser("configure") configure_parser.set_defaults(func=configure) - configure_parser.add_argument("source", type=str) - configure_parser.add_argument("dest", type=str) - configure_parser.add_argument("tag_delimiter", type=str) + configure_parser.add_argument("source", type=_abspath) + configure_parser.add_argument("dest", type=_abspath) + configure_parser.add_argument("tag_delimiter", type=str, default=";") configure_parser.add_argument("param_dict", type=str) return arg_parser @@ -266,14 +247,11 @@ def parse_arguments() -> argparse.Namespace: """ parser = get_parser() parsed_args = parser.parse_args() - parsed_args.func(parsed_args) return parsed_args if __name__ == "__main__": - """Run file operations move, remove, symlink, copy, and configure - using command line arguments. - """ os.environ["PYTHONUNBUFFERED"] = "1" args = parse_arguments() + args.func(args) diff --git a/tests/test_configs/tagged_tests/correct/invalidtag.txt b/tests/test_configs/tagged_tests/correct/invalidtag.txt new file mode 100644 index 000000000..cb640c64f --- /dev/null +++ b/tests/test_configs/tagged_tests/correct/invalidtag.txt @@ -0,0 +1,3 @@ +some text before +some params are valid and others are ;INVALID; but we mostly encounter valid params +some text after \ No newline at end of file diff --git a/tests/test_configs/tagged_tests/marked/invalidtag.txt b/tests/test_configs/tagged_tests/marked/invalidtag.txt new file mode 100644 index 000000000..4d339a49a --- /dev/null +++ b/tests/test_configs/tagged_tests/marked/invalidtag.txt @@ -0,0 +1,3 @@ +some text before +some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params +some text after \ No newline at end of file diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 36624fd26..2e662df16 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -24,20 +24,24 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import argparse import base64 import filecmp import os import pathlib import pickle -from distutils import dir_util + from glob import glob from os import path as osp +import shutil import pytest from smartsim._core.entrypoints import file_operations from smartsim._core.entrypoints.file_operations import get_parser +pytestmark = pytest.mark.group_a + def test_symlink_files(test_dir): """ @@ -116,11 +120,12 @@ def test_symlink_not_absolute(test_dir): parser = get_parser() cmd = f"symlink {source_file} {entity_path}" args = cmd.split() - ns = parser.parse_args(args) - with pytest.raises(ValueError) as ex: - file_operations.symlink(ns) - assert f"path {entity_path} must be absolute" in ex.value.args + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message def test_copy_op_file(test_dir): @@ -276,11 +281,12 @@ def test_copy_not_absolute(test_dir): parser = get_parser() cmd = f"copy {source_file} {bad_path}" args = cmd.split() - ns = parser.parse_args(args) - with pytest.raises(ValueError) as ex: - file_operations.copy(ns) - assert f"path {bad_path} must be absolute" in ex.value.args + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message # clean up os.remove(pathlib.Path(to_copy) / "copy_file.txt") @@ -345,11 +351,12 @@ def test_move_not_absolute(test_dir): parser = get_parser() cmd = f"move {source_file} {dest_file}" args = cmd.split() - ns = parser.parse_args(args) - with pytest.raises(ValueError) as ex: - file_operations.move(ns) - assert f"path {dest_file} must be absolute" in ex.value.args + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message def test_remove_op_file(test_dir): @@ -409,7 +416,7 @@ def test_remove_op_bad_path(test_dir): assert "No such file or directory" in ex.value.args -def test_remove_op_no_absolute(): +def test_remove_op_not_absolute(): """Test that ValueError is raised when a relative path is given to the operation to delete a file""" @@ -418,11 +425,12 @@ def test_remove_op_no_absolute(): parser = get_parser() cmd = f"remove {to_del}" args = cmd.split() - ns = parser.parse_args(args) - with pytest.raises(ValueError) as ex: - file_operations.remove(ns) - assert f"path {to_del} must be absolute" in ex.value.args + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message @pytest.mark.parametrize( @@ -436,6 +444,7 @@ def test_remove_op_no_absolute(): "65": "70", "placeholder": "group leftupper region", "1200": "120", + "VALID": "valid", }, "None", id="correct dict", @@ -457,8 +466,16 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): # retrieve files to compare after test correct_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "correct/")) + conf_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "marked/") + ) + # retrieve files to compare after test + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + # copy files to test directory - dir_util.copy_tree(conf_path, test_dir) + shutil.copytree(conf_path, test_dir, dirs_exist_ok=True) assert osp.isdir(test_dir) tagged_files = sorted(glob(test_dir + "/*")) @@ -468,7 +485,7 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): pickled_dict = pickle.dumps(param_dict) # Encode the pickled dictionary with Base64 - encoded_dict = base64.b64encode(pickled_dict) + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") # Run configure op on test files for tagged_file in tagged_files: @@ -493,6 +510,29 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): assert filecmp.cmp(written, correct) +def test_configure_invalid_tags(fileutils): + + # = /home/users/putko/scratch/SmartSim/tests/test_configs/tagged_tests/invalidtag.txt + tagged_file = fileutils.get_test_conf_path( + osp.join("tagged_tests", "invalidtag.txt") + ) + + tag = ";" + param_dict = {"VALID": "valid"} + + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + parser = get_parser() + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.configure(ns) + + def test_configure_not_absolute(): """Test that ValueError is raised when tagged files given to configure op are not absolute paths @@ -509,19 +549,20 @@ def test_configure_not_absolute(): parser = get_parser() cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" args = cmd.split() - ns = parser.parse_args(args) - with pytest.raises(ValueError) as ex: - file_operations.move(ns) - assert f"path {tagged_file} must be absolute" in ex.value.args + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message def test_parser_move(): """Test that the parser succeeds when receiving expected args for the move operation""" parser = get_parser() - src_path = "/absolute/file/src/path" - dest_path = "/absolute/file/dest/path" + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") cmd = f"move {src_path} {dest_path}" args = cmd.split() @@ -535,7 +576,7 @@ def test_parser_remove(): """Test that the parser succeeds when receiving expected args for the remove operation""" parser = get_parser() - file_path = "/absolute/file/path" + file_path = pathlib.Path("/absolute/file/path") cmd = f"remove {file_path}" args = cmd.split() @@ -548,8 +589,8 @@ def test_parser_symlink(): """Test that the parser succeeds when receiving expected args for the symlink operation""" parser = get_parser() - src_path = "/absolute/file/src/path" - dest_path = "/absolute/file/dest/path" + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") cmd = f"symlink {src_path} {dest_path}" args = cmd.split() @@ -564,8 +605,8 @@ def test_parser_copy(): """Test that the parser succeeds when receiving expected args for the copy operation""" parser = get_parser() - src_path = "/absolute/file/src/path" - dest_path = "/absolute/file/dest/path" + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") cmd = f"copy {src_path} {dest_path}" @@ -580,8 +621,8 @@ def test_parser_configure_parse(): """Test that the parser succeeds when receiving expected args for the configure operation""" parser = get_parser() - src_path = "/absolute/file/src/path" - dest_path = "/absolute/file/dest/path" + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") tag_delimiter = ";" param_dict = { From 517de846aafadbf279aecd90f01086ceccbdbcc0 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 23 Jul 2024 16:43:17 -0500 Subject: [PATCH 14/17] resolving last of PR comments --- smartsim/_core/entrypoints/file_operations.py | 81 ++-- .../easy}/correct/invalidtag.txt | 2 +- .../easy/marked/invalidtag.txt | 3 + .../tagged_tests/correct/MOM_input | 363 ------------------ .../tagged_tests/correct/example_input.i | 118 ------ .../tagged_tests/correct/in.airebo | 22 -- .../test_configs/tagged_tests/correct/in.atm | 31 -- .../tagged_tests/correct/in.crack | 78 ---- .../tagged_tests/correct/in.ellipse.gayberne | 66 ---- .../tagged_tests/correct/input-file.inp | 245 ------------ .../tagged_tests/correct/input.nml | 22 -- .../tagged_tests/correct/simple-H20.xml | 86 ----- .../tagged_tests/marked/MOM_input | 363 ------------------ .../tagged_tests/marked/example_input.i | 118 ------ .../tagged_tests/marked/in.airebo | 22 -- tests/test_configs/tagged_tests/marked/in.atm | 31 -- .../test_configs/tagged_tests/marked/in.crack | 78 ---- .../tagged_tests/marked/in.ellipse.gayberne | 66 ---- .../tagged_tests/marked/input-file.inp | 245 ------------ .../tagged_tests/marked/input.nml | 22 -- .../tagged_tests/marked/invalidtag.txt | 3 - .../tagged_tests/marked/simple-H20.xml | 86 ----- tests/test_file_operations.py | 58 ++- 23 files changed, 92 insertions(+), 2117 deletions(-) rename tests/test_configs/{tagged_tests => generator_files/easy}/correct/invalidtag.txt (86%) create mode 100644 tests/test_configs/generator_files/easy/marked/invalidtag.txt delete mode 100644 tests/test_configs/tagged_tests/correct/MOM_input delete mode 100644 tests/test_configs/tagged_tests/correct/example_input.i delete mode 100644 tests/test_configs/tagged_tests/correct/in.airebo delete mode 100644 tests/test_configs/tagged_tests/correct/in.atm delete mode 100644 tests/test_configs/tagged_tests/correct/in.crack delete mode 100644 tests/test_configs/tagged_tests/correct/in.ellipse.gayberne delete mode 100644 tests/test_configs/tagged_tests/correct/input-file.inp delete mode 100644 tests/test_configs/tagged_tests/correct/input.nml delete mode 100644 tests/test_configs/tagged_tests/correct/simple-H20.xml delete mode 100644 tests/test_configs/tagged_tests/marked/MOM_input delete mode 100644 tests/test_configs/tagged_tests/marked/example_input.i delete mode 100644 tests/test_configs/tagged_tests/marked/in.airebo delete mode 100644 tests/test_configs/tagged_tests/marked/in.atm delete mode 100644 tests/test_configs/tagged_tests/marked/in.crack delete mode 100644 tests/test_configs/tagged_tests/marked/in.ellipse.gayberne delete mode 100644 tests/test_configs/tagged_tests/marked/input-file.inp delete mode 100644 tests/test_configs/tagged_tests/marked/input.nml delete mode 100644 tests/test_configs/tagged_tests/marked/invalidtag.txt delete mode 100644 tests/test_configs/tagged_tests/marked/simple-H20.xml diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 0e8352e35..9ab23f299 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -26,14 +26,12 @@ import argparse import base64 -import collections import functools import os import pathlib import pickle -import re import shutil - +import typing as t from typing import Callable from ...log import get_logger @@ -45,11 +43,29 @@ """ -def _abspath(s: str) -> pathlib.Path: - p = pathlib.Path(s) - if not p.is_absolute(): - raise ValueError(f"path `{p}` must be absolute") - return p +def _abspath(input_path: str) -> pathlib.Path: + """Helper function to check that paths are absolute""" + path = pathlib.Path(input_path) + if not path.is_absolute(): + raise ValueError(f"path `{path}` must be absolute") + return path + + +def _make_substitution( + tag_name: str, replacement: str | int | float, tag_delimiter: str +) -> Callable[[str], str]: + """Helper function to replace tags""" + return lambda s: s.replace( + f"{tag_delimiter}{tag_name}{tag_delimiter}", str(replacement) + ) + + +def _replace_tags_in( + item: str, + substitutions: t.Sequence[Callable[[str], str]], +) -> str: + """Helper function to derive the lines in which to make the substitutions""" + return functools.reduce(lambda a, fn: fn(a), substitutions, item) def move(parsed_args: argparse.Namespace) -> None: @@ -94,16 +110,26 @@ def copy(parsed_args: argparse.Namespace) -> None: Sample usage: .. highlight:: bash .. code-block:: bash - python -m smartsim._core.entrypoints.file_operations \ - copy /absolute/file/source/path /absolute/file/dest/path + python -m smartsim._core.entrypoints.file_operations copy \ + /absolute/file/source/path /absolute/file/dest/path \ + --dirs_exist_ok /absolute/file/source/path: Path to directory, or path to file to copy to a new location /absolute/file/dest/path: Path to destination directory or path to destination file + --dirs_exist_ok: if the flag is included, the copying operation will + continue if the destination directory and files alrady exist, + and will be overwritten by corresponding files. If the flag is + not includedm and the destination file already exists, a + FileExistsError will be raised """ if os.path.isdir(parsed_args.source): - shutil.copytree(parsed_args.source, parsed_args.dest, dirs_exist_ok=True) + shutil.copytree( + parsed_args.source, + parsed_args.dest, + dirs_exist_ok=parsed_args.dirs_exist_ok, + ) else: shutil.copyfile(parsed_args.source, parsed_args.dest) @@ -160,37 +186,13 @@ def configure(parsed_args: argparse.Namespace) -> None: if not isinstance(param_dict, dict): raise TypeError("param dict is not a valid dictionary") - def _get_prev_value(tagged_line: str) -> str: - split_tag = tagged_line.split(tag_delimiter) - return split_tag[1] - - def make_substitution(tag_name: str, replacement: str) -> Callable[[str], str]: - return lambda s: s.replace( - f"{tag_delimiter}{tag_name}{tag_delimiter}", str(replacement) - ) - - def _is_ensemble_spec(tagged_line: str, application_params: dict[str, str]) -> bool: - prev_val = _get_prev_value(tagged_line) - return prev_val in application_params - - edited = [] - # used_params = {} + substitutions = tuple( + _make_substitution(k, v, tag_delimiter) for k, v in param_dict.items() + ) # Set the lines to iterate over with open(parsed_args.source, "r+", encoding="utf-8") as file_stream: - lines = file_stream.readlines() - - unused_tags = collections.defaultdict(list) - - substitutions = tuple(make_substitution(k, v) for k, v in param_dict.items()) - replace_tags_in = lambda s: functools.reduce(lambda a, fn: fn(a), substitutions, s) - - edited = [replace_tags_in(line) for line in lines] - - for tag, value in unused_tags.items(): - missing_tag_message = f"Unused tag {tag} on line(s): {str(value)}" - logger.warning(missing_tag_message) - lines = edited + lines = [_replace_tags_in(line, substitutions) for line in file_stream] # write configured file to destination specified with open(parsed_args.dest, "w+", encoding="utf-8") as file_stream: @@ -222,6 +224,7 @@ def get_parser() -> argparse.ArgumentParser: copy_parser.set_defaults(func=copy) copy_parser.add_argument("source", type=_abspath) copy_parser.add_argument("dest", type=_abspath) + copy_parser.add_argument("--dirs_exist_ok", action="store_true") # Subparser for symlink op symlink_parser = subparsers.add_parser("symlink") diff --git a/tests/test_configs/tagged_tests/correct/invalidtag.txt b/tests/test_configs/generator_files/easy/correct/invalidtag.txt similarity index 86% rename from tests/test_configs/tagged_tests/correct/invalidtag.txt rename to tests/test_configs/generator_files/easy/correct/invalidtag.txt index cb640c64f..2165ae8d1 100644 --- a/tests/test_configs/tagged_tests/correct/invalidtag.txt +++ b/tests/test_configs/generator_files/easy/correct/invalidtag.txt @@ -1,3 +1,3 @@ some text before some params are valid and others are ;INVALID; but we mostly encounter valid params -some text after \ No newline at end of file +some text after diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt new file mode 100644 index 000000000..2165ae8d1 --- /dev/null +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -0,0 +1,3 @@ +some text before +some params are valid and others are ;INVALID; but we mostly encounter valid params +some text after diff --git a/tests/test_configs/tagged_tests/correct/MOM_input b/tests/test_configs/tagged_tests/correct/MOM_input deleted file mode 100644 index 1e9002333..000000000 --- a/tests/test_configs/tagged_tests/correct/MOM_input +++ /dev/null @@ -1,363 +0,0 @@ -! This file was written by the model and records the non-default parameters used at run-time. - -! === module MOM === - -! === module MOM_unit_scaling === -! Parameters for doing unit scaling of variables. -ENABLE_THERMODYNAMICS = False ! [Boolean] default = True - ! If true, Temperature and salinity are used as state - ! variables. -ADIABATIC = True ! [Boolean] default = False - ! There are no diapycnal mass fluxes if ADIABATIC is - ! true. This assumes that KD = KDML = 0.0 and that - ! there is no buoyancy forcing, but makes the model - ! faster by eliminating subroutine calls. -THICKNESSDIFFUSE = True ! [Boolean] default = False - ! If true, interface heights are diffused with a - ! coefficient of KHTH. -DT = 1200.0 ! [s] - ! The (baroclinic) dynamics time step. The time-step that - ! is actually used will be an integer fraction of the - ! forcing time-step (DT_FORCING in ocean-only mode or the - ! coupling timestep in coupled mode.) -DTBT_RESET_PERIOD = -1.0 ! [s] default = 1200.0 - ! The period between recalculations of DTBT (if DTBT <= 0). - ! If DTBT_RESET_PERIOD is negative, DTBT is set based - ! only on information available at initialization. If 0, - ! DTBT will be set every dynamics time step. The default - ! is set by DT_THERM. This is only used if SPLIT is true. - -! === module MOM_domains === -REENTRANT_X = False ! [Boolean] default = True - ! If true, the domain is zonally reentrant. -NIGLOBAL = 80 ! - ! The total number of thickness grid points in the - ! x-direction in the physical domain. With STATIC_MEMORY_ - ! this is set in MOM_memory.h at compile time. -NJGLOBAL = 40 ! - ! The total number of thickness grid points in the - ! y-direction in the physical domain. With STATIC_MEMORY_ - ! this is set in MOM_memory.h at compile time. - -! === module MOM_hor_index === -! Sets the horizontal array index types. - -! === module MOM_verticalGrid === -! Parameters providing information about the vertical grid. -NK = 10 ! [nondim] - ! The number of model layers. - -! === module MOM_fixed_initialization === -INPUTDIR = "INPUT" ! default = "." - ! The directory in which input files are found. - -! === module MOM_grid_init === -GRID_CONFIG = "spherical" ! - ! A character string that determines the method for - ! defining the horizontal grid. Current options are: - ! mosaic - read the grid from a mosaic (supergrid) - ! file set by GRID_FILE. - ! cartesian - use a (flat) Cartesian grid. - ! spherical - use a simple spherical grid. - ! mercator - use a Mercator spherical grid. -SOUTHLAT = 30.0 ! [degrees] - ! The southern latitude of the domain. -LENLAT = 20.0 ! [degrees] - ! The latitudinal length of the domain. -LENLON = 40.0 ! [degrees] - ! The longitudinal length of the domain. -TOPO_CONFIG = "spoon" ! - ! This specifies how bathymetry is specified: - ! file - read bathymetric information from the file - ! specified by (TOPO_FILE). - ! flat - flat bottom set to MAXIMUM_DEPTH. - ! bowl - an analytically specified bowl-shaped basin - ! ranging between MAXIMUM_DEPTH and MINIMUM_DEPTH. - ! spoon - a similar shape to 'bowl', but with an vertical - ! wall at the southern face. - ! halfpipe - a zonally uniform channel with a half-sine - ! profile in the meridional direction. - ! benchmark - use the benchmark test case topography. - ! Neverland - use the Neverland test case topography. - ! DOME - use a slope and channel configuration for the - ! DOME sill-overflow test case. - ! ISOMIP - use a slope and channel configuration for the - ! ISOMIP test case. - ! DOME2D - use a shelf and slope configuration for the - ! DOME2D gravity current/overflow test case. - ! Kelvin - flat but with rotated land mask. - ! seamount - Gaussian bump for spontaneous motion test case. - ! dumbbell - Sloshing channel with reservoirs on both ends. - ! shelfwave - exponential slope for shelfwave test case. - ! Phillips - ACC-like idealized topography used in the Phillips config. - ! dense - Denmark Strait-like dense water formation and overflow. - ! USER - call a user modified routine. -MINIMUM_DEPTH = 1.0 ! [m] default = 0.0 - ! The minimum depth of the ocean. -MAXIMUM_DEPTH = 2000.0 ! [m] - ! The maximum depth of the ocean. - -! === module MOM_open_boundary === -! Controls where open boundaries are located, what kind of boundary condition to impose, and what data to apply, if any. - -! === module MOM_tracer_registry === - -! === module MOM_restart === - -! === module MOM_tracer_flow_control === -USE_REGIONAL_DYES = True ! [Boolean] default = False - ! If true, use the regional_dyes tracer package. - -! === module regional_dyes === -NUM_DYE_TRACERS = 3 ! default = 0 - ! The number of dye tracers in this run. Each tracer - ! should have a separate region. -DYE_SOURCE_MINLON = 3*10.0 ! [not defined] - ! This is the starting longitude at which we start injecting dyes. -DYE_SOURCE_MAXLON = 3*12.0 ! [not defined] - ! This is the ending longitude at which we finish injecting dyes. -DYE_SOURCE_MINLAT = 33.0, 39.0, 45.0 ! [not defined] - ! This is the starting latitude at which we start injecting dyes. -DYE_SOURCE_MAXLAT = 35.0, 41.0, 47.0 ! [not defined] - ! This is the ending latitude at which we finish injecting dyes. -DYE_SOURCE_MINDEPTH = 3*0.0 ! [m] - ! This is the minumum depth at which we inject dyes. -DYE_SOURCE_MAXDEPTH = 3*2000.0 ! [m] - ! This is the maximum depth at which we inject dyes. - -! === module MOM_coord_initialization === -COORD_CONFIG = "gprime" ! - ! This specifies how layers are to be defined: - ! ALE or none - used to avoid defining layers in ALE mode - ! file - read coordinate information from the file - ! specified by (COORD_FILE). - ! BFB - Custom coords for buoyancy-forced basin case - ! based on SST_S, T_BOT and DRHO_DT. - ! linear - linear based on interfaces not layers - ! layer_ref - linear based on layer densities - ! ts_ref - use reference temperature and salinity - ! ts_range - use range of temperature and salinity - ! (T_REF and S_REF) to determine surface density - ! and GINT calculate internal densities. - ! gprime - use reference density (RHO_0) for surface - ! density and GINT calculate internal densities. - ! ts_profile - use temperature and salinity profiles - ! (read from COORD_FILE) to set layer densities. - ! USER - call a user modified routine. -GFS = 0.98 ! [m s-2] default = 9.8 - ! The reduced gravity at the free surface. -GINT = 0.004 ! [m s-2] - ! The reduced gravity across internal interfaces. - -! === module MOM_grid === -! Parameters providing information about the lateral grid. - -! === module MOM_state_initialization === -THICKNESS_CONFIG = "uniform" ! - ! A string that determines how the initial layer - ! thicknesses are specified for a new run: - ! file - read interface heights from the file specified - ! thickness_file - read thicknesses from the file specified - ! by (THICKNESS_FILE). - ! coord - determined by ALE coordinate. - ! uniform - uniform thickness layers evenly distributed - ! between the surface and MAXIMUM_DEPTH. - ! list - read a list of positive interface depths. - ! DOME - use a slope and channel configuration for the - ! DOME sill-overflow test case. - ! ISOMIP - use a configuration for the - ! ISOMIP test case. - ! benchmark - use the benchmark test case thicknesses. - ! Neverland - use the Neverland test case thicknesses. - ! search - search a density profile for the interface - ! densities. This is not yet implemented. - ! circle_obcs - the circle_obcs test case is used. - ! DOME2D - 2D version of DOME initialization. - ! adjustment2d - 2D lock exchange thickness ICs. - ! sloshing - sloshing gravity thickness ICs. - ! seamount - no motion test with seamount ICs. - ! dumbbell - sloshing channel ICs. - ! soliton - Equatorial Rossby soliton. - ! rossby_front - a mixed layer front in thermal wind balance. - ! USER - call a user modified routine. - -! === module MOM_diag_mediator === - -! === module MOM_MEKE === - -! === module MOM_lateral_mixing_coeffs === - -! === module MOM_set_visc === -LINEAR_DRAG = True ! [Boolean] default = False - ! If LINEAR_DRAG and BOTTOMDRAGLAW are defined the drag - ! law is cdrag*DRAG_BG_VEL*u. -HBBL = 10.0 ! [m] - ! The thickness of a bottom boundary layer with a - ! viscosity of KVBBL if BOTTOMDRAGLAW is not defined, or - ! the thickness over which near-bottom velocities are - ! averaged for the drag law if BOTTOMDRAGLAW is defined - ! but LINEAR_DRAG is not. -DRAG_BG_VEL = 0.1 ! [m s-1] default = 0.0 - ! DRAG_BG_VEL is either the assumed bottom velocity (with - ! LINEAR_DRAG) or an unresolved velocity that is - ! combined with the resolved velocity to estimate the - ! velocity magnitude. DRAG_BG_VEL is only used when - ! BOTTOMDRAGLAW is defined. -BBL_THICK_MIN = 0.1 ! [m] default = 0.0 - ! The minimum bottom boundary layer thickness that can be - ! used with BOTTOMDRAGLAW. This might be - ! Kv / (cdrag * drag_bg_vel) to give Kv as the minimum - ! near-bottom viscosity. -KV = 1.0E-04 ! [m2 s-1] - ! The background kinematic viscosity in the interior. - ! The molecular value, ~1e-6 m2 s-1, may be used. - -! === module MOM_continuity === - -! === module MOM_continuity_PPM === -ETA_TOLERANCE = 1.0E-12 ! [m] default = 2.5E-10 - ! The tolerance for the differences between the - ! barotropic and baroclinic estimates of the sea surface - ! height due to the fluxes through each face. The total - ! tolerance for SSH is 4 times this value. The default - ! is 0.5*NK*ANGSTROM, and this should not be set less x - ! than about 10^-15*MAXIMUM_DEPTH. - -! === module MOM_CoriolisAdv === -BOUND_CORIOLIS = True ! [Boolean] default = False - ! If true, the Coriolis terms at u-points are bounded by - ! the four estimates of (f+rv)v from the four neighboring - ! v-points, and similarly at v-points. This option would - ! have no effect on the SADOURNY Coriolis scheme if it - ! were possible to use centered difference thickness fluxes. - -! === module MOM_PressureForce === - -! === module MOM_PressureForce_AFV === - -! === module MOM_hor_visc === -SMAGORINSKY_AH = True ! [Boolean] default = False - ! If true, use a biharmonic Smagorinsky nonlinear eddy - ! viscosity. -SMAG_BI_CONST = 0.06 ! [nondim] default = 0.0 - ! The nondimensional biharmonic Smagorinsky constant, - ! typically 0.015 - 0.06. - -! === module MOM_vert_friction === -DIRECT_STRESS = True ! [Boolean] default = False - ! If true, the wind stress is distributed over the - ! topmost HMIX_STRESS of fluid (like in HYCOM), and KVML - ! may be set to a very small value. -HARMONIC_VISC = True ! [Boolean] default = False - ! If true, use the harmonic mean thicknesses for - ! calculating the vertical viscosity. -HMIX_FIXED = 20.0 ! [m] - ! The prescribed depth over which the near-surface - ! viscosity and diffusivity are elevated when the bulk - ! mixed layer is not used. -KVML = 0.01 ! [m2 s-1] default = 1.0E-04 - ! The kinematic viscosity in the mixed layer. A typical - ! value is ~1e-2 m2 s-1. KVML is not used if - ! BULKMIXEDLAYER is true. The default is set by KV. -MAXVEL = 6.0 ! [m s-1] default = 3.0E+08 - ! The maximum velocity allowed before the velocity - ! components are truncated. - -! === module MOM_barotropic === -BOUND_BT_CORRECTION = True ! [Boolean] default = False - ! If true, the corrective pseudo mass-fluxes into the - ! barotropic solver are limited to values that require - ! less than maxCFL_BT_cont to be accommodated. -BT_PROJECT_VELOCITY = True ! [Boolean] default = False - ! If true, step the barotropic velocity first and project - ! out the velocity tendancy by 1+BEBT when calculating the - ! transport. The default (false) is to use a predictor - ! continuity step to find the pressure field, and then - ! to do a corrector continuity step using a weighted - ! average of the old and new velocities, with weights - ! of (1-BEBT) and BEBT. -BEBT = 0.2 ! [nondim] default = 0.1 - ! BEBT determines whether the barotropic time stepping - ! uses the forward-backward time-stepping scheme or a - ! backward Euler scheme. BEBT is valid in the range from - ! 0 (for a forward-backward treatment of nonrotating - ! gravity waves) to 1 (for a backward Euler treatment). - ! In practice, BEBT must be greater than about 0.05. -DTBT = -0.9 ! [s or nondim] default = -0.98 - ! The barotropic time step, in s. DTBT is only used with - ! the split explicit time stepping. To set the time step - ! automatically based the maximum stable value use 0, or - ! a negative value gives the fraction of the stable value. - ! Setting DTBT to 0 is the same as setting it to -0.98. - ! The value of DTBT that will actually be used is an - ! integer fraction of DT, rounding down. - -! === module MOM_thickness_diffuse === -KHTH = 0. ! [m2 s-1] default = 0.0 - ! The background horizontal thickness diffusivity. -KHTH_USE_FGNV_STREAMFUNCTION = False ! [Boolean] default = False - ! If true, use the streamfunction formulation of - ! Ferrari et al., 2010, which effectively emphasizes - ! graver vertical modes by smoothing in the vertical. - -! === module MOM_mixed_layer_restrat === - -! === module MOM_diag_to_Z === - -! === module MOM_diabatic_driver === -! The following parameters are used for diabatic processes. - -! === module MOM_tracer_advect === - -! === module MOM_tracer_hor_diff === - -! === module MOM_neutral_diffusion === -! This module implements neutral diffusion of tracers - -! === module MOM_sum_output === -DATE_STAMPED_STDOUT = False ! [Boolean] default = True - ! If true, use dates (not times) in messages to stdout -READ_DEPTH_LIST = True ! [Boolean] default = False - ! Read the depth list from a file if it exists or - ! create that file otherwise. -DEPTH_LIST_MIN_INC = 1.0E-06 ! [m] default = 1.0E-10 - ! The minimum increment between the depths of the - ! entries in the depth-list file. - -! === module MOM_surface_forcing === -VARIABLE_WINDS = False ! [Boolean] default = True - ! If true, the winds vary in time after the initialization. -VARIABLE_BUOYFORCE = False ! [Boolean] default = True - ! If true, the buoyancy forcing varies in time after the - ! initialization of the model. -BUOY_CONFIG = "zero" ! - ! The character string that indicates how buoyancy forcing - ! is specified. Valid options include (file), (zero), - ! (linear), (USER), (BFB) and (NONE). -WIND_CONFIG = "2gyre" ! - ! The character string that indicates how wind forcing - ! is specified. Valid options include (file), (2gyre), - ! (1gyre), (gyres), (zero), and (USER). - -! === module MOM_restart === - -! === module MOM_main (MOM_driver) === -DT_FORCING = 2400.0 ! [s] default = 1200.0 - ! The time step for changing forcing, coupling with other - ! components, or potentially writing certain diagnostics. - ! The default value is given by DT. -DAYMAX = 1460.0 ! [days] - ! The final time of the whole simulation, in units of - ! TIMEUNIT seconds. This also sets the potential end - ! time of the present run segment if the end time is - ! not set via ocean_solo_nml in input.nml. -RESTART_CONTROL = 3 ! default = 1 - ! An integer whose bits encode which restart files are - ! written. Add 2 (bit 1) for a time-stamped file, and odd - ! (bit 0) for a non-time-stamped file. A non-time-stamped - ! restart file is saved at the end of the run segment - ! for any non-negative value. - -! === module MOM_write_cputime === - -! === module MOM_file_parser === diff --git a/tests/test_configs/tagged_tests/correct/example_input.i b/tests/test_configs/tagged_tests/correct/example_input.i deleted file mode 100644 index 099485e11..000000000 --- a/tests/test_configs/tagged_tests/correct/example_input.i +++ /dev/null @@ -1,118 +0,0 @@ -[Mesh] - file = reactor.e - # Let's assign human friendly names to the blocks on the fly - block_id = '1 2' - block_name = 'fuel deflector' - - boundary_id = '4 5' - boundary_name = 'bottom top' -[] - -[Variables] - [./diffused] - order = SECOND - family = LAGRANGE - initial_condition = 0.5 # shortcut/convenience for setting constant initial condition - [../] - - [./convected] - order = FIRST - family = LAGRANGE - initial_condition = 0.0 # shortcut/convenience for setting constant initial condition - [../] -[] - -[Kernels] - # This Kernel consumes a real-gradient material property from the active material - [./convection] - type = ExampleConvection - variable = convected - [../] - - [./diff_convected] - type = Diffusion - variable = convected - [../] - - [./example_diff] - # This Kernel uses "diffusivity" from the active material - type = ExampleDiffusion - variable = diffused - [../] - - [./time_deriv_diffused] - type = TimeDerivative - variable = diffused - [../] - - [./time_deriv_convected] - type = TimeDerivative - variable = convected - [../] -[] - -[BCs] - [./bottom_diffused] - type = DirichletBC - variable = diffused - boundary = 'bottom' - value = 0 - [../] - - [./top_diffused] - type = DirichletBC - variable = diffused - boundary = 'top' - value = 5 - [../] - - [./bottom_convected] - type = DirichletBC - variable = convected - boundary = 'bottom' - value = 0 - [../] - - [./top_convected] - type = NeumannBC - variable = convected - boundary = 'top' - value = 1 - [../] -[] - -[Materials] - [./example] - type = ExampleMaterial - block = 'fuel' - diffusion_gradient = 'diffused' - - # Approximate Parabolic Diffusivity - independent_vals = '0 0.25 0.5 0.75 1.0' - dependent_vals = '1e-2 5e-3 1e-3 5e-3 1e-2' - [../] - - [./example1] - type = ExampleMaterial - block = 'deflector' - diffusion_gradient = 'diffused' - - # Constant Diffusivity - independent_vals = '0 1.0' - dependent_vals = '1e-1 1e-1' - [../] -[] - -[Executioner] - type = Transient - solve_type = 'PJFNK' - petsc_options_iname = '-pc_type -pc_hypre_type' - petsc_options_value = 'hypre boomeramg' - dt = 0.1 - num_steps = 10 -[] - -[Outputs] - execute_on = 'timestep_end' - exodus = true -[] diff --git a/tests/test_configs/tagged_tests/correct/in.airebo b/tests/test_configs/tagged_tests/correct/in.airebo deleted file mode 100644 index 56ff000c3..000000000 --- a/tests/test_configs/tagged_tests/correct/in.airebo +++ /dev/null @@ -1,22 +0,0 @@ -# AIREBO polyethelene benchmark - -units metal -atom_style atomic - -read_data data.airebo - -replicate 20 16 2 - -neighbor 0.5 bin -neigh_modify delay 5 every 1 - -pair_style airebo 3.0 1 1 -pair_coeff * * CH.airebo C H - -velocity all create 300.0 761341 - -fix 1 all nve -timestep 0.0005 - -thermo 10 -run 100 diff --git a/tests/test_configs/tagged_tests/correct/in.atm b/tests/test_configs/tagged_tests/correct/in.atm deleted file mode 100644 index 055c1986b..000000000 --- a/tests/test_configs/tagged_tests/correct/in.atm +++ /dev/null @@ -1,31 +0,0 @@ -# Axilrod-Teller-Muto potential example - -variable x index 1 -variable y index 1 -variable z index 1 - -variable xx equal 10*$x -variable yy equal 10*$y -variable zz equal 10*$z - -units lj -atom_style atomic - -lattice fcc 0.70 -region box block 0 ${xx} 0 ${yy} 0 ${zz} -create_box 1 box -create_atoms 1 box - -pair_style hybrid/overlay lj/cut 4.5 atm 4.5 2.5 -pair_coeff * * lj/cut 1.0 1.0 -pair_coeff * * atm * 0.072 - -mass * 1.0 -velocity all create 1.033 12345678 loop geom - -fix 1 all nvt temp 1.033 1.033 0.05 - -timestep 0.002 -thermo 5 - -run 25 diff --git a/tests/test_configs/tagged_tests/correct/in.crack b/tests/test_configs/tagged_tests/correct/in.crack deleted file mode 100644 index 150b84fc9..000000000 --- a/tests/test_configs/tagged_tests/correct/in.crack +++ /dev/null @@ -1,78 +0,0 @@ -# 2d LJ crack simulation - -dimension 2 -boundary s s p - -atom_style atomic -neighbor 0.3 bin -neigh_modify delay 5 - -# create geometry - -lattice hex 0.93 -region box block 0 100 0 40 -0.25 0.25 -create_box 5 box -create_atoms 1 box - -mass 1 1.0 -mass 2 1.0 -mass 3 1.0 -mass 4 1.0 -mass 5 1.0 - -# LJ potentials - -pair_style lj/cut 2.5 -pair_coeff * * 1.0 1.0 2.5 - -# define groups - -region 1 block INF INF INF 1.25 INF INF -group lower region 1 -region 2 block INF INF 38.75 INF INF INF -group upper region 2 -group boundary union lower upper -group mobile subtract all boundary -group leftupper region - -region leftupper block INF 20 20 INF INF INF -region leftlower block INF 20 INF 20 INF INF -group leftupper region leftupper -group leftlower region leftlower - -set group leftupper type 2 -set group leftlower type 3 -set group lower type 4 -set group upper type 5 - -# initial velocities - -compute new mobile temp -velocity mobile create 0.01 887723 temp new -velocity upper set 0.0 0.3 0.0 -velocity mobile ramp vy 0.0 0.3 y 1.25 38.75 sum yes - -# fixes - -fix 1 all nve -fix 2 boundary setforce NULL 0.0 0.0 - -# run - -timestep 0.003 -thermo 200 -thermo_modify temp new - -neigh_modify exclude type 2 3 - -#dump 1 all atom 500 dump.crack - -#dump 2 all image 250 image.*.jpg type type & -# zoom 1.6 adiam 1.5 -#dump_modify 2 pad 4 - -#dump 3 all movie 250 movie.mpg type type & -# zoom 1.6 adiam 1.5 -#dump_modify 3 pad 4 - -run 5000 diff --git a/tests/test_configs/tagged_tests/correct/in.ellipse.gayberne b/tests/test_configs/tagged_tests/correct/in.ellipse.gayberne deleted file mode 100644 index fe783ac6d..000000000 --- a/tests/test_configs/tagged_tests/correct/in.ellipse.gayberne +++ /dev/null @@ -1,66 +0,0 @@ -# GayBerne ellipsoids in LJ background fluid - -units lj -atom_style ellipsoid -dimension 2 - -lattice sq 0.02 -region box block 0 20 0 20 -0.5 0.5 -create_box 2 box -create_atoms 1 box - -set group all type/fraction 2 0.1 95392 -set type 1 mass 1.0 -set type 2 mass 1.5 -set type 1 shape 1 1 1 -set type 2 shape 3 1 1 -set group all quat/random 18238 - -compute rot all temp/asphere -group spheroid type 1 -variable dof equal count(spheroid)+2 -compute_modify rot extra/dof ${dof} - -velocity all create 2.4 87287 loop geom - -pair_style gayberne 1.0 3.0 1.0 4.0 -pair_coeff 1 1 3.0 1.0 1 1 1 1 1 1 2.5 -pair_coeff 1 2 3.0 1.0 1 1 1 0 0 0 -pair_coeff 2 2 1.0 1.0 1 1 0.2 0 0 0 - -neighbor 0.8 bin - -thermo_style custom step c_rot epair etotal press vol -thermo 100 - -timestep 0.002 - -compute q all property/atom quatw quati quatj quatk - -#dump 1 all custom 100 dump.ellipse.gayberne & -# id type x y z c_q[1] c_q[2] c_q[3] c_q[4] - -#dump 2 all image 100 image.*.jpg type type & -# zoom 1.6 center d 0.5 0.5 0.5 -#dump_modify 2 pad 4 adiam 1 1.0 adiam 2 2.0 - -#dump 3 all movie 100 movie.mpg type type & -# zoom 1.6 center d 0.5 0.5 0.5 -#dump_modify 3 pad 4 adiam 1 1.0 adiam 2 2.0 - -fix 1 all npt/asphere temp 2.0 2.0 0.1 iso 0.0 1.0 1.0 & - mtk no pchain 0 tchain 1 -fix 2 all enforce2d - -compute_modify 1_temp extra/dof ${dof} - -# equilibrate to shrink box around dilute system - -run 2000 - -# run dynamics on dense system - -unfix 1 -fix 1 all nve/asphere - -run 2000 diff --git a/tests/test_configs/tagged_tests/correct/input-file.inp b/tests/test_configs/tagged_tests/correct/input-file.inp deleted file mode 100644 index f135ecec1..000000000 --- a/tests/test_configs/tagged_tests/correct/input-file.inp +++ /dev/null @@ -1,245 +0,0 @@ -&FORCE_EVAL - METHOD Quickstep - &DFT - &QS - METHOD PM6 - EPS_DEFAULT 1.0E-10 - &SE - ANALYTICAL_GRADIENTS T - &COULOMB - CUTOFF [angstrom] 12.0 - RC_RANGE [bohr] 1.0 - &END - &EXCHANGE - CUTOFF [angstrom] 4.9325 - RC_TAPER [angstrom] 12.0 - RC_RANGE [bohr] 1.0 - &END - &END - &END QS - &SCF - &PRINT - &RESTART OFF - &END - &RESTART_HISTORY OFF - &END - &END - SCF_GUESS MOPAC - EPS_SCF 1.0E-5 - MAX_SCF 45 - &OT - PRECONDITIONER FULL_SINGLE_INVERSE - MINIMIZER DIIS - N_DIIS 9 - &END - &OUTER_SCF - MAX_SCF 8 - EPS_SCF 1.0E-5 - &END - &END SCF - &XC - &XC_FUNCTIONAL PBE - &END XC_FUNCTIONAL - &END XC - &END DFT - &SUBSYS - &CELL - PERIODIC NONE - ABC 16.0000 16.0000000 16.0000 - &END CELL - &COLVAR - &REACTION_PATH - LAMBDA 5.0 - STEP_SIZE 0.01 - RANGE -2.0 37.0 - VARIABLE T - FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T - &MAP - RANGE 2 12 - RANGE 2 12 - GRID_SPACING 0.5 - GRID_SPACING 0.5 - &END - &COLVAR - &DISTANCE - ATOMS 26 32 - &END - &END - FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T - &COLVAR - &DISTANCE - ATOMS 26 30 - &END - &END - &END - &END - &COLVAR - &DISTANCE_FROM_PATH - LAMBDA 15.0 - STEP_SIZE 0.01 - RANGE -2.0 37.0 - VARIABLE T - FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T - &MAP - RANGE 2 12 - RANGE 2 12 - GRID_SPACING 0.5 - GRID_SPACING 0.5 - &END - &COLVAR - &DISTANCE - ATOMS 26 32 - &END - &END - FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T - &COLVAR - &DISTANCE - ATOMS 26 30 - &END - &END - &END - &END - &COORD - O -4.4247387624 -0.5309572750 0.5697230862 - H -5.0183448733 -2.6944585590 1.3203736786 - C -3.6109170457 -1.3741775205 0.8680427212 - O -4.0237717839 -2.6214903530 1.2703514467 - C -2.1306370189 -1.2791406640 0.8535019283 - C -1.3120456446 -2.3978600881 1.1275197410 - H -1.7701653707 -3.3666124305 1.3548030036 - C 0.0688545253 -2.2437189267 1.1156801180 - H 0.7574075334 -3.0761272058 1.3486991675 - N 0.6738559410 -1.0371313768 0.8361136511 - C -0.1389885304 0.0574156107 0.5692682480 - C -1.5454296273 -0.0457404425 0.5761779393 - H -2.1704627231 0.8321461706 0.3801763635 - C 0.5913111673 1.2918018616 0.2805097546 - C -0.0299604321 2.4779270180 -0.1639273497 - H -1.1228367654 2.5372055901 -0.2444243606 - C 0.7663086277 3.5666879578 -0.5045075502 - C 0.1920272309 4.8496216484 -0.9668879613 - O -1.1836917368 4.8023669936 -0.9803620784 - O 0.7656701641 5.8524002130 -1.3220490954 - C 2.1762122923 3.4640311228 -0.4070263504 - H 2.8012745440 4.3073721391 -0.7188446821 - C 2.7340753851 2.2923512744 0.0769505212 - H 3.8188537022 2.1726101379 0.1961014861 - N 1.9703552650 1.1968447154 0.4447247525 - Ru 2.6808139034 -0.6268135255 1.0491515077 - N 3.2563830464 -2.5532275954 1.5027135434 - C 3.4490476757 -3.6669628938 1.8370067213 - S 3.7025486662 -5.2038677765 2.2394078258 - N 2.1955375549 -0.3650270770 3.0173819208 - C 1.6049936707 0.3098036981 3.7855301635 - S 0.8289774632 1.2452756100 4.8086474146 - N 3.3853640961 -0.9960228944 -0.8720138955 - C 2.6610011074 -1.5613188486 -1.8986641089 - H 1.5915143243 -1.7170221188 -1.6810559568 - C 3.2453301524 -1.9340252113 -3.1029040105 - H 2.6589980673 -2.4156892719 -3.8913029786 - C 4.6278759505 -1.7104040677 -3.2859858733 - C 5.3108087788 -2.0657932775 -4.5535542391 - O 6.4687902929 -1.8960108056 -4.8530811803 - H 10.0999394091 1.8836409660 2.4997876783 - O 4.4387949377 -2.6355643612 -5.4502144227 - C 5.3859781105 -1.1418894308 -2.2656271240 - H 6.4625500813 -0.9854422725 -2.3976463282 - C 4.7478203106 -0.7953695233 -1.0558453820 - C 5.4376024832 -0.2248315007 0.1002998824 - C 6.7974932647 0.1530791814 0.1052796969 - H 7.4226310419 0.0133029862 -0.7846209826 - C 7.3399526057 0.7040767770 1.2625106413 - C 8.7635758427 1.1048269295 1.2588255996 - O 9.5655997786 1.0448892034 0.3558301760 - O 9.1376353505 1.6165130959 2.4790228433 - C 6.5282331717 0.8715673047 2.4115537092 - H 6.9561129325 1.2952014265 3.3275244791 - C 5.2007233179 0.4774776840 2.3595740688 - H 4.5207036878 0.5633558687 3.2285882081 - N 4.6296913260 -0.0725021202 1.2253520131 - H 4.8874702054 -2.8845712381 -6.3088218514 - H -1.5910323092 5.6705894857 -1.2601791678 - &END COORD - &END SUBSYS -&END FORCE_EVAL -&GLOBAL - PROJECT N3_rp_dp_md - RUN_TYPE MD - PRINT_LEVEL LOW -&END GLOBAL -&MOTION - &FREE_ENERGY - &METADYN - DO_HILLS T - LAGRANGE - NT_HILLS 10 - WW 1.0e-4 - TEMPERATURE 100. - TEMP_TOL 50 - &METAVAR - COLVAR 1 - LAMBDA 0.5 - MASS 10.0 - SCALE 0.03 - &END - &METAVAR - COLVAR 2 - LAMBDA 4.5 - MASS 2.0 - SCALE 0.03 - &END - &PRINT - &COLVAR - COMMON_ITERATION_LEVELS 3 - &EACH - MD 1 - &END - &END - &HILLS - COMMON_ITERATION_LEVELS 3 - &EACH - MD 1 - &END - &END - &END - &END METADYN - &END FREE_ENERGY - &MD - ENSEMBLE NVT - STEPS 10 - TIMESTEP 1 - TEMPERATURE 300.0 - TEMP_TOL 50 - &THERMOSTAT - TYPE NOSE - &NOSE - LENGTH 6 - YOSHIDA 3 - TIMECON 1500 - MTS 2 - &END NOSE - &END THERMOSTAT - &PRINT - &ENERGY - &EACH - MD 10 - &END - ADD_LAST NUMERIC - &END - &END PRINT - &END MD - &PRINT - &TRAJECTORY - &EACH - MD 10 - &END - &END - &RESTART - &EACH - MD 100 - &END - &END - &RESTART_HISTORY OFF - &END - &END -&END MOTION diff --git a/tests/test_configs/tagged_tests/correct/input.nml b/tests/test_configs/tagged_tests/correct/input.nml deleted file mode 100644 index 12b131307..000000000 --- a/tests/test_configs/tagged_tests/correct/input.nml +++ /dev/null @@ -1,22 +0,0 @@ - &MOM_input_nml - output_directory = './', - input_filename = 'n' - restart_input_dir = 'INPUT/', - restart_output_dir = 'RESTART/', - parameter_filename = 'MOM_input', - 'MOM_override' / -&fms_nml - domains_stack_size = 552960 - / - - &diag_manager_nml - / - - &ocean_solo_nml - months = 120 - days = 0 - date_init = 1,1,1,0,0,0 - hours = 0 - minutes = 0 - seconds = 0 - calendar = 'noleap' / diff --git a/tests/test_configs/tagged_tests/correct/simple-H20.xml b/tests/test_configs/tagged_tests/correct/simple-H20.xml deleted file mode 100644 index 120843be5..000000000 --- a/tests/test_configs/tagged_tests/correct/simple-H20.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - Simple Example of moleculear H2O - - - - - - -1 - - 2.9151687332e-01 -6.5123272502e-01 -1.2188463918e-01 - 5.8423636048e-01 4.2730406357e-01 -4.5964306231e-03 - 3.5228575807e-01 -3.5027014639e-01 5.2644808295e-01 - -5.1686250912e-01 -1.6648002292e+00 6.5837023441e-01 - - - - -1 - - 3.1443445436e-01 6.5068682609e-01 -4.0983449009e-02 - -3.8686061749e-01 -9.3744432997e-02 -6.0456005388e-01 - 2.4978241724e-02 -3.2862514649e-02 -7.2266047173e-01 - -4.0352404772e-01 1.1927734805e+00 5.5610824921e-01 - - - - - - 6 - 4 - 8 - - - 1 - 1 - 1 - - - 0.0000000000e+00 0.0000000000e+00 0.0000000000e+00 - 0.0000000000e+00 -1.4308249289e+00 1.1078707576e+00 - 0.0000000000e+00 1.4308249289e+00 1.1078707576e+00 - - - O H H - - - - - - - - - - - - - - - - - - - - 1 - 64 - 1 - 5 - 100 - 1 - 0.3 - no - - - - 128 - no - 100 - 0.005 - 10 - 200 - yes - - - diff --git a/tests/test_configs/tagged_tests/marked/MOM_input b/tests/test_configs/tagged_tests/marked/MOM_input deleted file mode 100644 index 27b9336b6..000000000 --- a/tests/test_configs/tagged_tests/marked/MOM_input +++ /dev/null @@ -1,363 +0,0 @@ -! This file was written by the model and records the non-default parameters used at run-time. - -! === module MOM === - -! === module MOM_unit_scaling === -! Parameters for doing unit scaling of variables. -ENABLE_THERMODYNAMICS = False ! [Boolean] default = True - ! If true, Temperature and salinity are used as state - ! variables. -ADIABATIC = True ! [Boolean] default = False - ! There are no diapycnal mass fluxes if ADIABATIC is - ! true. This assumes that KD = KDML = 0.0 and that - ! there is no buoyancy forcing, but makes the model - ! faster by eliminating subroutine calls. -THICKNESSDIFFUSE = True ! [Boolean] default = False - ! If true, interface heights are diffused with a - ! coefficient of KHTH. -DT = 1200.0 ! [s] - ! The (baroclinic) dynamics time step. The time-step that - ! is actually used will be an integer fraction of the - ! forcing time-step (DT_FORCING in ocean-only mode or the - ! coupling timestep in coupled mode.) -DTBT_RESET_PERIOD = -1.0 ! [s] default = 1200.0 - ! The period between recalculations of DTBT (if DTBT <= 0). - ! If DTBT_RESET_PERIOD is negative, DTBT is set based - ! only on information available at initialization. If 0, - ! DTBT will be set every dynamics time step. The default - ! is set by DT_THERM. This is only used if SPLIT is true. - -! === module MOM_domains === -REENTRANT_X = False ! [Boolean] default = True - ! If true, the domain is zonally reentrant. -NIGLOBAL = 80 ! - ! The total number of thickness grid points in the - ! x-direction in the physical domain. With STATIC_MEMORY_ - ! this is set in MOM_memory.h at compile time. -NJGLOBAL = 40 ! - ! The total number of thickness grid points in the - ! y-direction in the physical domain. With STATIC_MEMORY_ - ! this is set in MOM_memory.h at compile time. - -! === module MOM_hor_index === -! Sets the horizontal array index types. - -! === module MOM_verticalGrid === -! Parameters providing information about the vertical grid. -NK = ;5; ! [nondim] - ! The number of model layers. - -! === module MOM_fixed_initialization === -INPUTDIR = "INPUT" ! default = "." - ! The directory in which input files are found. - -! === module MOM_grid_init === -GRID_CONFIG = "spherical" ! - ! A character string that determines the method for - ! defining the horizontal grid. Current options are: - ! mosaic - read the grid from a mosaic (supergrid) - ! file set by GRID_FILE. - ! cartesian - use a (flat) Cartesian grid. - ! spherical - use a simple spherical grid. - ! mercator - use a Mercator spherical grid. -SOUTHLAT = 30.0 ! [degrees] - ! The southern latitude of the domain. -LENLAT = 20.0 ! [degrees] - ! The latitudinal length of the domain. -LENLON = 40.0 ! [degrees] - ! The longitudinal length of the domain. -TOPO_CONFIG = "spoon" ! - ! This specifies how bathymetry is specified: - ! file - read bathymetric information from the file - ! specified by (TOPO_FILE). - ! flat - flat bottom set to MAXIMUM_DEPTH. - ! bowl - an analytically specified bowl-shaped basin - ! ranging between MAXIMUM_DEPTH and MINIMUM_DEPTH. - ! spoon - a similar shape to 'bowl', but with an vertical - ! wall at the southern face. - ! halfpipe - a zonally uniform channel with a half-sine - ! profile in the meridional direction. - ! benchmark - use the benchmark test case topography. - ! Neverland - use the Neverland test case topography. - ! DOME - use a slope and channel configuration for the - ! DOME sill-overflow test case. - ! ISOMIP - use a slope and channel configuration for the - ! ISOMIP test case. - ! DOME2D - use a shelf and slope configuration for the - ! DOME2D gravity current/overflow test case. - ! Kelvin - flat but with rotated land mask. - ! seamount - Gaussian bump for spontaneous motion test case. - ! dumbbell - Sloshing channel with reservoirs on both ends. - ! shelfwave - exponential slope for shelfwave test case. - ! Phillips - ACC-like idealized topography used in the Phillips config. - ! dense - Denmark Strait-like dense water formation and overflow. - ! USER - call a user modified routine. -MINIMUM_DEPTH = 1.0 ! [m] default = 0.0 - ! The minimum depth of the ocean. -MAXIMUM_DEPTH = 2000.0 ! [m] - ! The maximum depth of the ocean. - -! === module MOM_open_boundary === -! Controls where open boundaries are located, what kind of boundary condition to impose, and what data to apply, if any. - -! === module MOM_tracer_registry === - -! === module MOM_restart === - -! === module MOM_tracer_flow_control === -USE_REGIONAL_DYES = True ! [Boolean] default = False - ! If true, use the regional_dyes tracer package. - -! === module regional_dyes === -NUM_DYE_TRACERS = 3 ! default = 0 - ! The number of dye tracers in this run. Each tracer - ! should have a separate region. -DYE_SOURCE_MINLON = 3*10.0 ! [not defined] - ! This is the starting longitude at which we start injecting dyes. -DYE_SOURCE_MAXLON = 3*12.0 ! [not defined] - ! This is the ending longitude at which we finish injecting dyes. -DYE_SOURCE_MINLAT = 33.0, 39.0, 45.0 ! [not defined] - ! This is the starting latitude at which we start injecting dyes. -DYE_SOURCE_MAXLAT = 35.0, 41.0, 47.0 ! [not defined] - ! This is the ending latitude at which we finish injecting dyes. -DYE_SOURCE_MINDEPTH = 3*0.0 ! [m] - ! This is the minumum depth at which we inject dyes. -DYE_SOURCE_MAXDEPTH = 3*2000.0 ! [m] - ! This is the maximum depth at which we inject dyes. - -! === module MOM_coord_initialization === -COORD_CONFIG = "gprime" ! - ! This specifies how layers are to be defined: - ! ALE or none - used to avoid defining layers in ALE mode - ! file - read coordinate information from the file - ! specified by (COORD_FILE). - ! BFB - Custom coords for buoyancy-forced basin case - ! based on SST_S, T_BOT and DRHO_DT. - ! linear - linear based on interfaces not layers - ! layer_ref - linear based on layer densities - ! ts_ref - use reference temperature and salinity - ! ts_range - use range of temperature and salinity - ! (T_REF and S_REF) to determine surface density - ! and GINT calculate internal densities. - ! gprime - use reference density (RHO_0) for surface - ! density and GINT calculate internal densities. - ! ts_profile - use temperature and salinity profiles - ! (read from COORD_FILE) to set layer densities. - ! USER - call a user modified routine. -GFS = 0.98 ! [m s-2] default = 9.8 - ! The reduced gravity at the free surface. -GINT = 0.004 ! [m s-2] - ! The reduced gravity across internal interfaces. - -! === module MOM_grid === -! Parameters providing information about the lateral grid. - -! === module MOM_state_initialization === -THICKNESS_CONFIG = "uniform" ! - ! A string that determines how the initial layer - ! thicknesses are specified for a new run: - ! file - read interface heights from the file specified - ! thickness_file - read thicknesses from the file specified - ! by (THICKNESS_FILE). - ! coord - determined by ALE coordinate. - ! uniform - uniform thickness layers evenly distributed - ! between the surface and MAXIMUM_DEPTH. - ! list - read a list of positive interface depths. - ! DOME - use a slope and channel configuration for the - ! DOME sill-overflow test case. - ! ISOMIP - use a configuration for the - ! ISOMIP test case. - ! benchmark - use the benchmark test case thicknesses. - ! Neverland - use the Neverland test case thicknesses. - ! search - search a density profile for the interface - ! densities. This is not yet implemented. - ! circle_obcs - the circle_obcs test case is used. - ! DOME2D - 2D version of DOME initialization. - ! adjustment2d - 2D lock exchange thickness ICs. - ! sloshing - sloshing gravity thickness ICs. - ! seamount - no motion test with seamount ICs. - ! dumbbell - sloshing channel ICs. - ! soliton - Equatorial Rossby soliton. - ! rossby_front - a mixed layer front in thermal wind balance. - ! USER - call a user modified routine. - -! === module MOM_diag_mediator === - -! === module MOM_MEKE === - -! === module MOM_lateral_mixing_coeffs === - -! === module MOM_set_visc === -LINEAR_DRAG = True ! [Boolean] default = False - ! If LINEAR_DRAG and BOTTOMDRAGLAW are defined the drag - ! law is cdrag*DRAG_BG_VEL*u. -HBBL = 10.0 ! [m] - ! The thickness of a bottom boundary layer with a - ! viscosity of KVBBL if BOTTOMDRAGLAW is not defined, or - ! the thickness over which near-bottom velocities are - ! averaged for the drag law if BOTTOMDRAGLAW is defined - ! but LINEAR_DRAG is not. -DRAG_BG_VEL = 0.1 ! [m s-1] default = 0.0 - ! DRAG_BG_VEL is either the assumed bottom velocity (with - ! LINEAR_DRAG) or an unresolved velocity that is - ! combined with the resolved velocity to estimate the - ! velocity magnitude. DRAG_BG_VEL is only used when - ! BOTTOMDRAGLAW is defined. -BBL_THICK_MIN = 0.1 ! [m] default = 0.0 - ! The minimum bottom boundary layer thickness that can be - ! used with BOTTOMDRAGLAW. This might be - ! Kv / (cdrag * drag_bg_vel) to give Kv as the minimum - ! near-bottom viscosity. -KV = 1.0E-04 ! [m2 s-1] - ! The background kinematic viscosity in the interior. - ! The molecular value, ~1e-6 m2 s-1, may be used. - -! === module MOM_continuity === - -! === module MOM_continuity_PPM === -ETA_TOLERANCE = 1.0E-12 ! [m] default = 2.5E-10 - ! The tolerance for the differences between the - ! barotropic and baroclinic estimates of the sea surface - ! height due to the fluxes through each face. The total - ! tolerance for SSH is 4 times this value. The default - ! is 0.5*NK*ANGSTROM, and this should not be set less x - ! than about 10^-15*MAXIMUM_DEPTH. - -! === module MOM_CoriolisAdv === -BOUND_CORIOLIS = True ! [Boolean] default = False - ! If true, the Coriolis terms at u-points are bounded by - ! the four estimates of (f+rv)v from the four neighboring - ! v-points, and similarly at v-points. This option would - ! have no effect on the SADOURNY Coriolis scheme if it - ! were possible to use centered difference thickness fluxes. - -! === module MOM_PressureForce === - -! === module MOM_PressureForce_AFV === - -! === module MOM_hor_visc === -SMAGORINSKY_AH = True ! [Boolean] default = False - ! If true, use a biharmonic Smagorinsky nonlinear eddy - ! viscosity. -SMAG_BI_CONST = 0.06 ! [nondim] default = 0.0 - ! The nondimensional biharmonic Smagorinsky constant, - ! typically 0.015 - 0.06. - -! === module MOM_vert_friction === -DIRECT_STRESS = True ! [Boolean] default = False - ! If true, the wind stress is distributed over the - ! topmost HMIX_STRESS of fluid (like in HYCOM), and KVML - ! may be set to a very small value. -HARMONIC_VISC = True ! [Boolean] default = False - ! If true, use the harmonic mean thicknesses for - ! calculating the vertical viscosity. -HMIX_FIXED = 20.0 ! [m] - ! The prescribed depth over which the near-surface - ! viscosity and diffusivity are elevated when the bulk - ! mixed layer is not used. -KVML = 0.01 ! [m2 s-1] default = 1.0E-04 - ! The kinematic viscosity in the mixed layer. A typical - ! value is ~1e-2 m2 s-1. KVML is not used if - ! BULKMIXEDLAYER is true. The default is set by KV. -MAXVEL = 6.0 ! [m s-1] default = 3.0E+08 - ! The maximum velocity allowed before the velocity - ! components are truncated. - -! === module MOM_barotropic === -BOUND_BT_CORRECTION = True ! [Boolean] default = False - ! If true, the corrective pseudo mass-fluxes into the - ! barotropic solver are limited to values that require - ! less than maxCFL_BT_cont to be accommodated. -BT_PROJECT_VELOCITY = True ! [Boolean] default = False - ! If true, step the barotropic velocity first and project - ! out the velocity tendancy by 1+BEBT when calculating the - ! transport. The default (false) is to use a predictor - ! continuity step to find the pressure field, and then - ! to do a corrector continuity step using a weighted - ! average of the old and new velocities, with weights - ! of (1-BEBT) and BEBT. -BEBT = 0.2 ! [nondim] default = 0.1 - ! BEBT determines whether the barotropic time stepping - ! uses the forward-backward time-stepping scheme or a - ! backward Euler scheme. BEBT is valid in the range from - ! 0 (for a forward-backward treatment of nonrotating - ! gravity waves) to 1 (for a backward Euler treatment). - ! In practice, BEBT must be greater than about 0.05. -DTBT = -0.9 ! [s or nondim] default = -0.98 - ! The barotropic time step, in s. DTBT is only used with - ! the split explicit time stepping. To set the time step - ! automatically based the maximum stable value use 0, or - ! a negative value gives the fraction of the stable value. - ! Setting DTBT to 0 is the same as setting it to -0.98. - ! The value of DTBT that will actually be used is an - ! integer fraction of DT, rounding down. - -! === module MOM_thickness_diffuse === -KHTH = 0. ! [m2 s-1] default = 0.0 - ! The background horizontal thickness diffusivity. -KHTH_USE_FGNV_STREAMFUNCTION = False ! [Boolean] default = False - ! If true, use the streamfunction formulation of - ! Ferrari et al., 2010, which effectively emphasizes - ! graver vertical modes by smoothing in the vertical. - -! === module MOM_mixed_layer_restrat === - -! === module MOM_diag_to_Z === - -! === module MOM_diabatic_driver === -! The following parameters are used for diabatic processes. - -! === module MOM_tracer_advect === - -! === module MOM_tracer_hor_diff === - -! === module MOM_neutral_diffusion === -! This module implements neutral diffusion of tracers - -! === module MOM_sum_output === -DATE_STAMPED_STDOUT = False ! [Boolean] default = True - ! If true, use dates (not times) in messages to stdout -READ_DEPTH_LIST = True ! [Boolean] default = False - ! Read the depth list from a file if it exists or - ! create that file otherwise. -DEPTH_LIST_MIN_INC = 1.0E-06 ! [m] default = 1.0E-10 - ! The minimum increment between the depths of the - ! entries in the depth-list file. - -! === module MOM_surface_forcing === -VARIABLE_WINDS = False ! [Boolean] default = True - ! If true, the winds vary in time after the initialization. -VARIABLE_BUOYFORCE = False ! [Boolean] default = True - ! If true, the buoyancy forcing varies in time after the - ! initialization of the model. -BUOY_CONFIG = "zero" ! - ! The character string that indicates how buoyancy forcing - ! is specified. Valid options include (file), (zero), - ! (linear), (USER), (BFB) and (NONE). -WIND_CONFIG = "2gyre" ! - ! The character string that indicates how wind forcing - ! is specified. Valid options include (file), (2gyre), - ! (1gyre), (gyres), (zero), and (USER). - -! === module MOM_restart === - -! === module MOM_main (MOM_driver) === -DT_FORCING = 2400.0 ! [s] default = 1200.0 - ! The time step for changing forcing, coupling with other - ! components, or potentially writing certain diagnostics. - ! The default value is given by DT. -DAYMAX = 1460.0 ! [days] - ! The final time of the whole simulation, in units of - ! TIMEUNIT seconds. This also sets the potential end - ! time of the present run segment if the end time is - ! not set via ocean_solo_nml in input.nml. -RESTART_CONTROL = 3 ! default = 1 - ! An integer whose bits encode which restart files are - ! written. Add 2 (bit 1) for a time-stamped file, and odd - ! (bit 0) for a non-time-stamped file. A non-time-stamped - ! restart file is saved at the end of the run segment - ! for any non-negative value. - -! === module MOM_write_cputime === - -! === module MOM_file_parser === diff --git a/tests/test_configs/tagged_tests/marked/example_input.i b/tests/test_configs/tagged_tests/marked/example_input.i deleted file mode 100644 index 97157a2f3..000000000 --- a/tests/test_configs/tagged_tests/marked/example_input.i +++ /dev/null @@ -1,118 +0,0 @@ -[Mesh] - file = reactor.e - # Let's assign human friendly names to the blocks on the fly - block_id = '1 2' - block_name = 'fuel deflector' - - boundary_id = '4 5' - boundary_name = 'bottom top' -[] - -[Variables] - [./diffused] - order = ;FIRST; - family = LAGRANGE - initial_condition = 0.5 # shortcut/convenience for setting constant initial condition - [../] - - [./convected] - order = FIRST - family = LAGRANGE - initial_condition = 0.0 # shortcut/convenience for setting constant initial condition - [../] -[] - -[Kernels] - # This Kernel consumes a real-gradient material property from the active material - [./convection] - type = ExampleConvection - variable = convected - [../] - - [./diff_convected] - type = Diffusion - variable = convected - [../] - - [./example_diff] - # This Kernel uses "diffusivity" from the active material - type = ExampleDiffusion - variable = diffused - [../] - - [./time_deriv_diffused] - type = TimeDerivative - variable = diffused - [../] - - [./time_deriv_convected] - type = TimeDerivative - variable = convected - [../] -[] - -[BCs] - [./bottom_diffused] - type = DirichletBC - variable = diffused - boundary = 'bottom' - value = 0 - [../] - - [./top_diffused] - type = DirichletBC - variable = diffused - boundary = 'top' - value = 5 - [../] - - [./bottom_convected] - type = DirichletBC - variable = convected - boundary = 'bottom' - value = 0 - [../] - - [./top_convected] - type = NeumannBC - variable = convected - boundary = 'top' - value = 1 - [../] -[] - -[Materials] - [./example] - type = ExampleMaterial - block = 'fuel' - diffusion_gradient = 'diffused' - - # Approximate Parabolic Diffusivity - independent_vals = '0 0.25 0.5 0.75 1.0' - dependent_vals = '1e-2 5e-3 1e-3 5e-3 1e-2' - [../] - - [./example1] - type = ExampleMaterial - block = 'deflector' - diffusion_gradient = 'diffused' - - # Constant Diffusivity - independent_vals = '0 1.0' - dependent_vals = '1e-1 1e-1' - [../] -[] - -[Executioner] - type = Transient - solve_type = 'PJFNK' - petsc_options_iname = '-pc_type -pc_hypre_type' - petsc_options_value = 'hypre boomeramg' - dt = 0.1 - num_steps = 10 -[] - -[Outputs] - execute_on = 'timestep_end' - exodus = true -[] diff --git a/tests/test_configs/tagged_tests/marked/in.airebo b/tests/test_configs/tagged_tests/marked/in.airebo deleted file mode 100644 index 7523a8e77..000000000 --- a/tests/test_configs/tagged_tests/marked/in.airebo +++ /dev/null @@ -1,22 +0,0 @@ -# AIREBO polyethelene benchmark - -units metal -atom_style atomic - -read_data data.airebo - -replicate ;17; 16 2 - -neighbor 0.5 bin -neigh_modify delay 5 every 1 - -pair_style airebo 3.0 1 1 -pair_coeff * * CH.airebo C H - -velocity all create 300.0 761341 - -fix 1 all nve -timestep 0.0005 - -thermo 10 -run 100 diff --git a/tests/test_configs/tagged_tests/marked/in.atm b/tests/test_configs/tagged_tests/marked/in.atm deleted file mode 100644 index c162fa7ee..000000000 --- a/tests/test_configs/tagged_tests/marked/in.atm +++ /dev/null @@ -1,31 +0,0 @@ -# Axilrod-Teller-Muto potential example - -variable x index 1 -variable y index 1 -variable z index 1 - -variable xx equal 10*$x -variable yy equal 10*$y -variable zz equal 10*$z - -units lj -atom_style atomic - -lattice fcc 0.;65; -region box block 0 ${xx} 0 ${yy} 0 ${zz} -create_box 1 box -create_atoms 1 box - -pair_style hybrid/overlay lj/cut 4.5 atm 4.5 2.5 -pair_coeff * * lj/cut 1.0 1.0 -pair_coeff * * atm * 0.072 - -mass * 1.0 -velocity all create 1.033 12345678 loop geom - -fix 1 all nvt temp 1.033 1.033 0.05 - -timestep 0.002 -thermo 5 - -run 25 diff --git a/tests/test_configs/tagged_tests/marked/in.crack b/tests/test_configs/tagged_tests/marked/in.crack deleted file mode 100644 index e9e70240d..000000000 --- a/tests/test_configs/tagged_tests/marked/in.crack +++ /dev/null @@ -1,78 +0,0 @@ -# 2d LJ crack simulation - -dimension 2 -boundary s s p - -atom_style atomic -neighbor 0.3 bin -neigh_modify delay 5 - -# create geometry - -lattice hex 0.93 -region box block 0 100 0 40 -0.25 0.25 -create_box 5 box -create_atoms 1 box - -mass 1 1.0 -mass 2 1.0 -mass 3 1.0 -mass 4 1.0 -mass 5 1.0 - -# LJ potentials - -pair_style lj/cut 2.5 -pair_coeff * * 1.0 1.0 2.5 - -# define groups - -region 1 block INF INF INF 1.25 INF INF -group lower region 1 -region 2 block INF INF 38.75 INF INF INF -group upper region 2 -group boundary union lower upper -group mobile subtract all boundary -;placeholder; - -region leftupper block INF 20 20 INF INF INF -region leftlower block INF 20 INF 20 INF INF -group leftupper region leftupper -group leftlower region leftlower - -set group leftupper type 2 -set group leftlower type 3 -set group lower type 4 -set group upper type 5 - -# initial velocities - -compute new mobile temp -velocity mobile create 0.01 887723 temp new -velocity upper set 0.0 0.3 0.0 -velocity mobile ramp vy 0.0 0.3 y 1.25 38.75 sum yes - -# fixes - -fix 1 all nve -fix 2 boundary setforce NULL 0.0 0.0 - -# run - -timestep 0.003 -thermo 200 -thermo_modify temp new - -neigh_modify exclude type 2 3 - -#dump 1 all atom 500 dump.crack - -#dump 2 all image 250 image.*.jpg type type & -# zoom 1.6 adiam 1.5 -#dump_modify 2 pad 4 - -#dump 3 all movie 250 movie.mpg type type & -# zoom 1.6 adiam 1.5 -#dump_modify 3 pad 4 - -run 5000 diff --git a/tests/test_configs/tagged_tests/marked/in.ellipse.gayberne b/tests/test_configs/tagged_tests/marked/in.ellipse.gayberne deleted file mode 100644 index fe783ac6d..000000000 --- a/tests/test_configs/tagged_tests/marked/in.ellipse.gayberne +++ /dev/null @@ -1,66 +0,0 @@ -# GayBerne ellipsoids in LJ background fluid - -units lj -atom_style ellipsoid -dimension 2 - -lattice sq 0.02 -region box block 0 20 0 20 -0.5 0.5 -create_box 2 box -create_atoms 1 box - -set group all type/fraction 2 0.1 95392 -set type 1 mass 1.0 -set type 2 mass 1.5 -set type 1 shape 1 1 1 -set type 2 shape 3 1 1 -set group all quat/random 18238 - -compute rot all temp/asphere -group spheroid type 1 -variable dof equal count(spheroid)+2 -compute_modify rot extra/dof ${dof} - -velocity all create 2.4 87287 loop geom - -pair_style gayberne 1.0 3.0 1.0 4.0 -pair_coeff 1 1 3.0 1.0 1 1 1 1 1 1 2.5 -pair_coeff 1 2 3.0 1.0 1 1 1 0 0 0 -pair_coeff 2 2 1.0 1.0 1 1 0.2 0 0 0 - -neighbor 0.8 bin - -thermo_style custom step c_rot epair etotal press vol -thermo 100 - -timestep 0.002 - -compute q all property/atom quatw quati quatj quatk - -#dump 1 all custom 100 dump.ellipse.gayberne & -# id type x y z c_q[1] c_q[2] c_q[3] c_q[4] - -#dump 2 all image 100 image.*.jpg type type & -# zoom 1.6 center d 0.5 0.5 0.5 -#dump_modify 2 pad 4 adiam 1 1.0 adiam 2 2.0 - -#dump 3 all movie 100 movie.mpg type type & -# zoom 1.6 center d 0.5 0.5 0.5 -#dump_modify 3 pad 4 adiam 1 1.0 adiam 2 2.0 - -fix 1 all npt/asphere temp 2.0 2.0 0.1 iso 0.0 1.0 1.0 & - mtk no pchain 0 tchain 1 -fix 2 all enforce2d - -compute_modify 1_temp extra/dof ${dof} - -# equilibrate to shrink box around dilute system - -run 2000 - -# run dynamics on dense system - -unfix 1 -fix 1 all nve/asphere - -run 2000 diff --git a/tests/test_configs/tagged_tests/marked/input-file.inp b/tests/test_configs/tagged_tests/marked/input-file.inp deleted file mode 100644 index f135ecec1..000000000 --- a/tests/test_configs/tagged_tests/marked/input-file.inp +++ /dev/null @@ -1,245 +0,0 @@ -&FORCE_EVAL - METHOD Quickstep - &DFT - &QS - METHOD PM6 - EPS_DEFAULT 1.0E-10 - &SE - ANALYTICAL_GRADIENTS T - &COULOMB - CUTOFF [angstrom] 12.0 - RC_RANGE [bohr] 1.0 - &END - &EXCHANGE - CUTOFF [angstrom] 4.9325 - RC_TAPER [angstrom] 12.0 - RC_RANGE [bohr] 1.0 - &END - &END - &END QS - &SCF - &PRINT - &RESTART OFF - &END - &RESTART_HISTORY OFF - &END - &END - SCF_GUESS MOPAC - EPS_SCF 1.0E-5 - MAX_SCF 45 - &OT - PRECONDITIONER FULL_SINGLE_INVERSE - MINIMIZER DIIS - N_DIIS 9 - &END - &OUTER_SCF - MAX_SCF 8 - EPS_SCF 1.0E-5 - &END - &END SCF - &XC - &XC_FUNCTIONAL PBE - &END XC_FUNCTIONAL - &END XC - &END DFT - &SUBSYS - &CELL - PERIODIC NONE - ABC 16.0000 16.0000000 16.0000 - &END CELL - &COLVAR - &REACTION_PATH - LAMBDA 5.0 - STEP_SIZE 0.01 - RANGE -2.0 37.0 - VARIABLE T - FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T - &MAP - RANGE 2 12 - RANGE 2 12 - GRID_SPACING 0.5 - GRID_SPACING 0.5 - &END - &COLVAR - &DISTANCE - ATOMS 26 32 - &END - &END - FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T - &COLVAR - &DISTANCE - ATOMS 26 30 - &END - &END - &END - &END - &COLVAR - &DISTANCE_FROM_PATH - LAMBDA 15.0 - STEP_SIZE 0.01 - RANGE -2.0 37.0 - VARIABLE T - FUNCTION 6.70208+3.8*sin(-0.131993*T+7.1788)+0.0678055*T - &MAP - RANGE 2 12 - RANGE 2 12 - GRID_SPACING 0.5 - GRID_SPACING 0.5 - &END - &COLVAR - &DISTANCE - ATOMS 26 32 - &END - &END - FUNCTION 7.34534+3.8*sin(-0.100683*T+5.23466)+0.0728908*T - &COLVAR - &DISTANCE - ATOMS 26 30 - &END - &END - &END - &END - &COORD - O -4.4247387624 -0.5309572750 0.5697230862 - H -5.0183448733 -2.6944585590 1.3203736786 - C -3.6109170457 -1.3741775205 0.8680427212 - O -4.0237717839 -2.6214903530 1.2703514467 - C -2.1306370189 -1.2791406640 0.8535019283 - C -1.3120456446 -2.3978600881 1.1275197410 - H -1.7701653707 -3.3666124305 1.3548030036 - C 0.0688545253 -2.2437189267 1.1156801180 - H 0.7574075334 -3.0761272058 1.3486991675 - N 0.6738559410 -1.0371313768 0.8361136511 - C -0.1389885304 0.0574156107 0.5692682480 - C -1.5454296273 -0.0457404425 0.5761779393 - H -2.1704627231 0.8321461706 0.3801763635 - C 0.5913111673 1.2918018616 0.2805097546 - C -0.0299604321 2.4779270180 -0.1639273497 - H -1.1228367654 2.5372055901 -0.2444243606 - C 0.7663086277 3.5666879578 -0.5045075502 - C 0.1920272309 4.8496216484 -0.9668879613 - O -1.1836917368 4.8023669936 -0.9803620784 - O 0.7656701641 5.8524002130 -1.3220490954 - C 2.1762122923 3.4640311228 -0.4070263504 - H 2.8012745440 4.3073721391 -0.7188446821 - C 2.7340753851 2.2923512744 0.0769505212 - H 3.8188537022 2.1726101379 0.1961014861 - N 1.9703552650 1.1968447154 0.4447247525 - Ru 2.6808139034 -0.6268135255 1.0491515077 - N 3.2563830464 -2.5532275954 1.5027135434 - C 3.4490476757 -3.6669628938 1.8370067213 - S 3.7025486662 -5.2038677765 2.2394078258 - N 2.1955375549 -0.3650270770 3.0173819208 - C 1.6049936707 0.3098036981 3.7855301635 - S 0.8289774632 1.2452756100 4.8086474146 - N 3.3853640961 -0.9960228944 -0.8720138955 - C 2.6610011074 -1.5613188486 -1.8986641089 - H 1.5915143243 -1.7170221188 -1.6810559568 - C 3.2453301524 -1.9340252113 -3.1029040105 - H 2.6589980673 -2.4156892719 -3.8913029786 - C 4.6278759505 -1.7104040677 -3.2859858733 - C 5.3108087788 -2.0657932775 -4.5535542391 - O 6.4687902929 -1.8960108056 -4.8530811803 - H 10.0999394091 1.8836409660 2.4997876783 - O 4.4387949377 -2.6355643612 -5.4502144227 - C 5.3859781105 -1.1418894308 -2.2656271240 - H 6.4625500813 -0.9854422725 -2.3976463282 - C 4.7478203106 -0.7953695233 -1.0558453820 - C 5.4376024832 -0.2248315007 0.1002998824 - C 6.7974932647 0.1530791814 0.1052796969 - H 7.4226310419 0.0133029862 -0.7846209826 - C 7.3399526057 0.7040767770 1.2625106413 - C 8.7635758427 1.1048269295 1.2588255996 - O 9.5655997786 1.0448892034 0.3558301760 - O 9.1376353505 1.6165130959 2.4790228433 - C 6.5282331717 0.8715673047 2.4115537092 - H 6.9561129325 1.2952014265 3.3275244791 - C 5.2007233179 0.4774776840 2.3595740688 - H 4.5207036878 0.5633558687 3.2285882081 - N 4.6296913260 -0.0725021202 1.2253520131 - H 4.8874702054 -2.8845712381 -6.3088218514 - H -1.5910323092 5.6705894857 -1.2601791678 - &END COORD - &END SUBSYS -&END FORCE_EVAL -&GLOBAL - PROJECT N3_rp_dp_md - RUN_TYPE MD - PRINT_LEVEL LOW -&END GLOBAL -&MOTION - &FREE_ENERGY - &METADYN - DO_HILLS T - LAGRANGE - NT_HILLS 10 - WW 1.0e-4 - TEMPERATURE 100. - TEMP_TOL 50 - &METAVAR - COLVAR 1 - LAMBDA 0.5 - MASS 10.0 - SCALE 0.03 - &END - &METAVAR - COLVAR 2 - LAMBDA 4.5 - MASS 2.0 - SCALE 0.03 - &END - &PRINT - &COLVAR - COMMON_ITERATION_LEVELS 3 - &EACH - MD 1 - &END - &END - &HILLS - COMMON_ITERATION_LEVELS 3 - &EACH - MD 1 - &END - &END - &END - &END METADYN - &END FREE_ENERGY - &MD - ENSEMBLE NVT - STEPS 10 - TIMESTEP 1 - TEMPERATURE 300.0 - TEMP_TOL 50 - &THERMOSTAT - TYPE NOSE - &NOSE - LENGTH 6 - YOSHIDA 3 - TIMECON 1500 - MTS 2 - &END NOSE - &END THERMOSTAT - &PRINT - &ENERGY - &EACH - MD 10 - &END - ADD_LAST NUMERIC - &END - &END PRINT - &END MD - &PRINT - &TRAJECTORY - &EACH - MD 10 - &END - &END - &RESTART - &EACH - MD 100 - &END - &END - &RESTART_HISTORY OFF - &END - &END -&END MOTION diff --git a/tests/test_configs/tagged_tests/marked/input.nml b/tests/test_configs/tagged_tests/marked/input.nml deleted file mode 100644 index 4ab606fd8..000000000 --- a/tests/test_configs/tagged_tests/marked/input.nml +++ /dev/null @@ -1,22 +0,0 @@ - &MOM_input_nml - output_directory = './', - input_filename = 'n' - restart_input_dir = 'INPUT/', - restart_output_dir = 'RESTART/', - parameter_filename = 'MOM_input', - 'MOM_override' / -&fms_nml - domains_stack_size = 552960 - / - - &diag_manager_nml - / - - &ocean_solo_nml - months = ;1200; - days = 0 - date_init = 1,1,1,0,0,0 - hours = 0 - minutes = 0 - seconds = 0 - calendar = 'noleap' / diff --git a/tests/test_configs/tagged_tests/marked/invalidtag.txt b/tests/test_configs/tagged_tests/marked/invalidtag.txt deleted file mode 100644 index 4d339a49a..000000000 --- a/tests/test_configs/tagged_tests/marked/invalidtag.txt +++ /dev/null @@ -1,3 +0,0 @@ -some text before -some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params -some text after \ No newline at end of file diff --git a/tests/test_configs/tagged_tests/marked/simple-H20.xml b/tests/test_configs/tagged_tests/marked/simple-H20.xml deleted file mode 100644 index 120843be5..000000000 --- a/tests/test_configs/tagged_tests/marked/simple-H20.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - Simple Example of moleculear H2O - - - - - - -1 - - 2.9151687332e-01 -6.5123272502e-01 -1.2188463918e-01 - 5.8423636048e-01 4.2730406357e-01 -4.5964306231e-03 - 3.5228575807e-01 -3.5027014639e-01 5.2644808295e-01 - -5.1686250912e-01 -1.6648002292e+00 6.5837023441e-01 - - - - -1 - - 3.1443445436e-01 6.5068682609e-01 -4.0983449009e-02 - -3.8686061749e-01 -9.3744432997e-02 -6.0456005388e-01 - 2.4978241724e-02 -3.2862514649e-02 -7.2266047173e-01 - -4.0352404772e-01 1.1927734805e+00 5.5610824921e-01 - - - - - - 6 - 4 - 8 - - - 1 - 1 - 1 - - - 0.0000000000e+00 0.0000000000e+00 0.0000000000e+00 - 0.0000000000e+00 -1.4308249289e+00 1.1078707576e+00 - 0.0000000000e+00 1.4308249289e+00 1.1078707576e+00 - - - O H H - - - - - - - - - - - - - - - - - - - - 1 - 64 - 1 - 5 - 100 - 1 - 0.3 - no - - - - 128 - no - 100 - 0.005 - 10 - 200 - yes - - - diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 2e662df16..6f516a569 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -30,10 +30,9 @@ import os import pathlib import pickle - +import shutil from glob import glob from os import path as osp -import shutil import pytest @@ -167,7 +166,8 @@ def test_copy_op_file(test_dir): def test_copy_op_dirs(test_dir): - """Test the oeprations that copies an entire directory tree source to a new location destination""" + """Test the operation that copies an entire directory tree source to a new location destination + that already exists""" to_copy = os.path.join(test_dir, "to_copy") os.mkdir(to_copy) @@ -186,7 +186,7 @@ def test_copy_op_dirs(test_dir): os.mkdir(entity_path) parser = get_parser() - cmd = f"copy {to_copy} {entity_path}" + cmd = f"copy {to_copy} {entity_path} --dirs_exist_ok" args = cmd.split() ns = parser.parse_args(args) @@ -211,6 +211,45 @@ def test_copy_op_dirs(test_dir): os.rmdir(pathlib.Path(test_dir) / "entity_name") +def test_copy_op_dirs_file_exists_error(test_dir): + """Test that a FileExistsError is raised when copying a directory tree source to a new location destination + when the destination already exists, and the flag --dirs_exist_ok is not included + """ + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + # write some test files in the dir + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + + source_file_2 = pathlib.Path(to_copy) / "copy_file_2.txt" + with open(source_file_2, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy2") + + # entity_path to be the dest dir + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + parser = get_parser() + # command does not include the --dirs_exist_ok flag + cmd = f"copy {to_copy} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + # Execute copy + with pytest.raises(FileExistsError) as ex: + file_operations.copy(ns) + assert f"File exists" in ex.value.args + + # Clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.remove(pathlib.Path(to_copy) / "copy_file_2.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + def test_copy_op_bad_source_file(test_dir): """Test that a FileNotFoundError is raised when there is a bad source file""" @@ -462,10 +501,6 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): tag = ";" - conf_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "marked/")) - # retrieve files to compare after test - correct_path = fileutils.get_test_conf_path(osp.join("tagged_tests", "correct/")) - conf_path = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "marked/") ) @@ -479,7 +514,7 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): assert osp.isdir(test_dir) tagged_files = sorted(glob(test_dir + "/*")) - correct_files = sorted(glob(correct_path + "*")) + correct_files = sorted(glob(correct_path + "/*")) # Pickle the dictionary pickled_dict = pickle.dumps(param_dict) @@ -511,10 +546,9 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): def test_configure_invalid_tags(fileutils): - - # = /home/users/putko/scratch/SmartSim/tests/test_configs/tagged_tests/invalidtag.txt + """Test configure operation with an invalid tag""" tagged_file = fileutils.get_test_conf_path( - osp.join("tagged_tests", "invalidtag.txt") + osp.join("generator_files", "easy", "marked", "invalidtag.txt") ) tag = ";" From 625f1f2e20ea3be1a9e7c4eeea1b865ef5900e57 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 23 Jul 2024 17:15:43 -0500 Subject: [PATCH 15/17] mypy typing version update --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 05b6ef70b..5f90e791d 100644 --- a/setup.py +++ b/setup.py @@ -179,7 +179,7 @@ def has_ext_modules(_placeholder): "pydantic==1.10.14", "pyzmq>=25.1.2", "pygithub>=2.3.0", - "numpy<2" + "numpy<2", ] # Add SmartRedis at specific version @@ -203,7 +203,7 @@ def has_ext_modules(_placeholder): "types-tqdm", "types-tensorflow==2.12.0.9", "types-setuptools", - "typing_extensions>=4.1.0", + "typing_extensions>=4.1.0,<4.6", ], # see smartsim/_core/_install/buildenv.py for more details **versions.ml_extras_required(), From fcc1adc015578300bc90498d880cb9d0fd5f17ee Mon Sep 17 00:00:00 2001 From: Julia Putko <81587103+juliaputko@users.noreply.github.com> Date: Tue, 23 Jul 2024 16:22:55 -0700 Subject: [PATCH 16/17] Update tests/test_configs/generator_files/easy/marked/invalidtag.txt Co-authored-by: Matt Drozt --- tests/test_configs/generator_files/easy/marked/invalidtag.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt index 2165ae8d1..90a625319 100644 --- a/tests/test_configs/generator_files/easy/marked/invalidtag.txt +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -1,3 +1,3 @@ some text before -some params are valid and others are ;INVALID; but we mostly encounter valid params +some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params some text after From 92be5a35a5fea1788be770af6590ad6e381a387b Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Wed, 24 Jul 2024 12:58:01 -0500 Subject: [PATCH 17/17] from __future__ import annotations to fix mypy error for python<3.10 --- smartsim/_core/entrypoints/file_operations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 9ab23f299..c57192ea8 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -24,6 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import annotations + import argparse import base64 import functools