From 7f9ace85e044df412b7b436170d3e43129d73350 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 28 Dec 2023 10:38:48 -0700 Subject: [PATCH 1/4] Makefile changes 1. use nf-config instead of nc-config if available 2. add "-g" to gfortran compile flags 3. remove spurious comment --- tests/driver_src/Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/driver_src/Makefile b/tests/driver_src/Makefile index d6a33722..da5c7923 100644 --- a/tests/driver_src/Makefile +++ b/tests/driver_src/Makefile @@ -93,9 +93,13 @@ endif ifneq ($(COMP_NAME),NONE) INFO_FILE = $(EXE_SRC_DIR)/.netcdf_info.$(COMP_NAME) ifeq ($(USE_NETCDF),TRUE) - NETCDF_INC=-D_NETCDF -I$(shell nc-config --includedir) - # hobart_intel version of nc-config does not have --libdir option - NETCDF_LNK=-L$(shell nc-config --prefix)/lib -lnetcdf -lnetcdff + ifneq (${shell which nf-config},) + CONFIG=nf-config + else + CONFIG=nc-config + endif + NETCDF_INC=-D_NETCDF -I$(shell $(CONFIG) --includedir) + NETCDF_LNK=-L$(shell $(CONFIG) --prefix)/lib -lnetcdff endif # Determine if existing build used netCDF @@ -151,7 +155,7 @@ all: gnu .PHONY: gnu gnu: - $(MAKE) -f $(MAKE_DIR)/Makefile $(EXE) COMP_NAME=gnu$(MPISUF) FC=gfortran FCFLAGS="-Wall -Wextra -Wno-compare-reals -Werror -O2 -ffree-form -cpp -finit-real=snan -ffpe-trap=invalid" INC="-J $(OBJ_ROOT)/gnu$(MPISUF)" INC2="-J $(OBJ_ROOT)/gnu$(MPISUF)/driver" + $(MAKE) -f $(MAKE_DIR)/Makefile $(EXE) COMP_NAME=gnu$(MPISUF) FC=gfortran FCFLAGS="-g -Wall -Wextra -Wno-compare-reals -Werror -O2 -ffree-form -cpp -finit-real=snan -ffpe-trap=invalid" INC="-J $(OBJ_ROOT)/gnu$(MPISUF)" INC2="-J $(OBJ_ROOT)/gnu$(MPISUF)/driver" .PHONY: intel intel: From 1910c379bc0752cae2595e841d425799c6e7b62f Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 28 Dec 2023 10:41:27 -0700 Subject: [PATCH 2/4] Improve run_test_suite.sh output Besides marking failed tests with an asterisk, also provide a list of all failed tests (there are so many tests now it's challenging to scroll back through and look for the *s) --- MARBL_tools/run_test_suite.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MARBL_tools/run_test_suite.sh b/MARBL_tools/run_test_suite.sh index 9711701d..e1e1b051 100755 --- a/MARBL_tools/run_test_suite.sh +++ b/MARBL_tools/run_test_suite.sh @@ -20,6 +20,7 @@ function print_status() { TAIL="" if [ "${STATUS}" == "FAIL" ]; then FAIL_CNT=$((FAIL_CNT+1)) + echo "${FAIL_CNT}. $1" >> ${FAILURE_CACHE} HEAD="***" TAIL=" ***" fi @@ -34,9 +35,11 @@ function print_status() { MARBL_ROOT=`(cd ..; pwd -P)` RESULTS_CACHE=${MARBL_ROOT}/MARBL_tools/.test_suite.cache +FAILURE_CACHE=${MARBL_ROOT}/MARBL_tools/.test_suite.failures.cache TEST_CNT=0 FAIL_CNT=0 echo "Test Results:" > ${RESULTS_CACHE} +echo "Failed Tests:" > ${FAILURE_CACHE} ######### # TESTS # @@ -365,5 +368,9 @@ cat ${RESULTS_CACHE} rm -f ${RESULTS_CACHE} echo "" echo "${TEST_CNT} tests were run, and $FAIL_CNT failed." +if [ ${FAIL_CNT} -gt 0 ]; then + cat ${FAILURE_CACHE} +fi +rm -f ${FAILURE_CACHE} exit ${FAIL_CNT} From 9f62492659e630b2a25b57798722d4dd2bde50dc Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 28 Dec 2023 10:55:47 -0700 Subject: [PATCH 3/4] Update threshold in netcdf_comparison s With baselines from an intel chip, the Apple M2 chip was failing several of the netCDF Comparison tests; increasing the relative tolerance to 1e-9 lets those comparisons pass. This will be more useful if we need to update the baselines in the future, since the easiest machine to generate baselines on is my laptop (with an M2) but the continuous integration is run on a github machine with an intel or amd chip. --- MARBL_tools/netcdf_comparison.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/MARBL_tools/netcdf_comparison.py b/MARBL_tools/netcdf_comparison.py index 0ec81d91..48490fa4 100755 --- a/MARBL_tools/netcdf_comparison.py +++ b/MARBL_tools/netcdf_comparison.py @@ -30,7 +30,7 @@ # to make it easy to update the default values if necessary # rtol = 1e-11 fails the cgs vs mks comparison # (POC_REMIN_DIC and PON_REMIN_NH4 have rel errors of ~1.4e-11) -DEFAULT_TOLS = {'rtol' : 2e-11, 'atol' : 1e-16, 'thres' : 1e-16} +DEFAULT_TOLS = {'rtol' : 1e-9, 'atol' : 1e-16, 'thres' : 1e-16} ################## @@ -181,7 +181,7 @@ def _get_conversion_factor(ds_base, ds_new, var): conversion_factor = 1. / unit_conversion[old_units][new_units] found = True if not found: - raise KeyError(f'Can not convert from {new_units} to {old_units}') + raise KeyError(f'Can not convert from {new_units} to {old_units} for {var}') return conversion_factor def _variable_check_loose(ds_base, ds_new, rtol, atol, thres): @@ -209,6 +209,8 @@ def _variable_check_loose(ds_base, ds_new, rtol, atol, thres): conversion_factor = _get_conversion_factor(ds_base, ds_new, var) # (1) Are NaNs in the same place? + if var.lower() == 'time': + continue mask = np.isfinite(ds_base[var].data) if np.any(mask ^ np.isfinite(ds_new[var].data)): error_checking['messages'].append('NaNs are not in same place') From ddf969a7fdf34dc9f306392a2ef4ba1db4987cb5 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 28 Dec 2023 14:26:32 -0700 Subject: [PATCH 4/4] Update derecho modules, recognize compute nodes All tests pass on derecho (need to be on compute node for mpi test); I recommend the develop queue with 1 node and 2 cores --- tests/python_for_tests/machines.py | 18 ++++++++++++++---- tests/python_for_tests/marbl_testing_class.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/python_for_tests/machines.py b/tests/python_for_tests/machines.py index a53e0073..c6b50966 100644 --- a/tests/python_for_tests/machines.py +++ b/tests/python_for_tests/machines.py @@ -28,11 +28,21 @@ def load_module(mach, compiler, module_name): sys.path.insert(0, os.path.join(os.sep, 'glade', 'u', 'apps', 'derecho', '23.06', 'spack', 'opt', 'spack', 'lmod', '8.7.20', 'gcc', '7.5.0', 'pdxb', 'lmod', 'lmod', 'init')) from env_modules_python import module module('--force', 'purge') - module('load', 'ncarenv/23.06') - module('load', 'craype') + module('load', 'cesmdev/1.0') + if compiler == 'cray': + module('load', 'ncarenv/23.06') + module('load', 'craype') + elif compiler == 'nvhpc': + module('load', 'ncarenv/23.06') + else: + module('load', 'ncarenv/23.09') module('load', module_name) + module('load', 'ncarcompilers/1.0.0') + if compiler == 'cray': + module('load', 'cray-mpich/8.1.25') + else: + module('load', 'cray-mpich/8.1.27') module('load', 'netcdf/4.9.2') - module('load', 'ncarcompilers') if compiler in ['gnu', 'cray']: module('load', 'cray-libsci/23.02.1.1') @@ -88,7 +98,7 @@ def machine_specific(mach, supported_compilers, module_names): supported_compilers.append('gnu') supported_compilers.append('cray') supported_compilers.append('nvhpc') - module_names['intel'] = 'intel/2023.0.0' + module_names['intel'] = 'intel/2023.2.1' module_names['gnu'] = 'gcc/12.2.0' module_names['cray'] = 'cce/15.0.1' module_names['nvhpc'] = 'nvhpc/23.1' diff --git a/tests/python_for_tests/marbl_testing_class.py b/tests/python_for_tests/marbl_testing_class.py index b4964f84..e404451f 100644 --- a/tests/python_for_tests/marbl_testing_class.py +++ b/tests/python_for_tests/marbl_testing_class.py @@ -90,7 +90,7 @@ def parse_args(self, desc, HaveCompiler=True, HaveInputFile=True, from socket import gethostname self._hostname = gethostname() found = True - if 'derecho' in self._hostname: + if 'derecho' in self._hostname or self._hostname.startswith('dec'): self._machine = 'derecho' elif 'cheyenne' in self._hostname: self._machine = 'cheyenne'