From 4e4e5ccc48ec560b4197448354ed6b30476c6442 Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Mon, 6 May 2024 18:12:05 -0600 Subject: [PATCH 01/15] bugfix for dadadj ccpp refactor. remove length check as std_names can be much longer than 63 chars. Couldnt find a lenght limit for that field in the docs --- CIME/ParamGen/xml_schema/entry_id_pg.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/ParamGen/xml_schema/entry_id_pg.xsd b/CIME/ParamGen/xml_schema/entry_id_pg.xsd index 5ea4ba6b468..b2439523347 100644 --- a/CIME/ParamGen/xml_schema/entry_id_pg.xsd +++ b/CIME/ParamGen/xml_schema/entry_id_pg.xsd @@ -9,7 +9,7 @@ - + From 4877321232e3d0dd52d1a95f888864e5e8a601ef Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Tue, 7 May 2024 13:17:05 -0600 Subject: [PATCH 02/15] restrict standard name to have 1 [a-zA-Z] character followed by 0 or more [A-Za-z0-9_] characters --- CIME/ParamGen/xml_schema/entry_id_pg.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/ParamGen/xml_schema/entry_id_pg.xsd b/CIME/ParamGen/xml_schema/entry_id_pg.xsd index b2439523347..b9ca87e94be 100644 --- a/CIME/ParamGen/xml_schema/entry_id_pg.xsd +++ b/CIME/ParamGen/xml_schema/entry_id_pg.xsd @@ -9,7 +9,7 @@ - + From b5f8c5158497c94feae0f4e5c845f6c2908eb553 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 14 May 2024 17:54:45 -0700 Subject: [PATCH 03/15] Adds option to check lockedfiles quietly, preventing exception from being raised --- CIME/Tools/xmlchange | 2 +- CIME/locked_files.py | 15 +++++++++------ CIME/tests/test_unit_locked_files.py | 11 +++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index cc6b216a825..1c53e8f7b6f 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -227,7 +227,7 @@ def xmlchange_single_value( expect(result is not None, 'No variable "%s" found' % xmlid) - check_lockedfiles(case, skip=["env_case"]) + check_lockedfiles(case, skip=["env_case"], quiet=True) else: logger.warning("'%s' = '%s'", xmlid, xmlval) diff --git a/CIME/locked_files.py b/CIME/locked_files.py index 178a8af6942..efb3f0647e9 100644 --- a/CIME/locked_files.py +++ b/CIME/locked_files.py @@ -55,7 +55,7 @@ def is_locked(filename, caseroot): return os.path.exists(os.path.join(caseroot, LOCKED_DIR, filename)) -def check_lockedfiles(case, skip=None): +def check_lockedfiles(case, skip=None, quiet=False): """ Check that all lockedfiles match what's in case @@ -77,17 +77,17 @@ def check_lockedfiles(case, skip=None): if filename.count(".") > 1 or any([filename.startswith(x) for x in skip]): continue - check_lockedfile(case, filename, caseroot=caseroot) + check_lockedfile(case, filename, caseroot=caseroot, quiet=quiet) -def check_lockedfile(case, filebase, caseroot=None): +def check_lockedfile(case, filebase, caseroot=None, quiet=False): if caseroot is None: caseroot = case.get_value("CASEROOT") env_name, diff = diff_lockedfile(case, caseroot, filebase) if diff: - check_diff(case, filebase, env_name, diff) + check_diff(case, filebase, env_name, diff, quiet=quiet) def diff_lockedfile(case, caseroot, filename): @@ -138,7 +138,7 @@ def _get_case_env(case, caseroot, locked_file, env_name): return l_env, r_env -def check_diff(case, filename, env_name, diff): +def check_diff(case, filename, env_name, diff, quiet=False): logger.warning("Detected diff in locked file {!r}".format(filename)) # Remove BUILD_COMPLETE, invalid entry in diff @@ -207,4 +207,7 @@ def check_diff(case, filename, env_name, diff): message = f"{message}./case.build {clean_targets}\n./case.build" - expect(False, message) + if quiet: + logger.info(message) + else: + expect(False, message) diff --git a/CIME/tests/test_unit_locked_files.py b/CIME/tests/test_unit_locked_files.py index 05b2e9952dc..fc871e3b768 100644 --- a/CIME/tests/test_unit_locked_files.py +++ b/CIME/tests/test_unit_locked_files.py @@ -262,6 +262,17 @@ def test_check_lockedfiles(self): with self.assertRaises(CIMEError): locked_files.check_lockedfiles(case) + def test_check_lockedfiles_quiet(self): + case = mock.MagicMock() + + with tempfile.TemporaryDirectory() as tempdir: + case.get_value.side_effect = (tempdir,) + + create_fake_env(tempdir) + + # Should not raise exception + locked_files.check_lockedfiles(case, quiet=True) + def test_is_locked(self): with tempfile.TemporaryDirectory() as tempdir: src_path = Path(tempdir, locked_files.LOCKED_DIR, "env_case.xml") From 1c05311ec73b22b3f9d197eaab39ae068da50288 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 08:53:30 -0700 Subject: [PATCH 04/15] Fixes docs workflow failing when no branch previews exist --- .github/workflows/docs.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 70899438082..a6c4ced353d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,18 +35,20 @@ jobs: run: | pushd $GITHUB_WORKSPACE/gh-pages - for name in `ls branch/` - do - if [[ -z "$(git show-ref --quiet ${name})" ]] - then - git rm -rf branch/${name} - fi - done + if [[ -e "$GITHUB_WORKSPACE/gh-pages/branch" ]]; then + for name in `ls branch/` + do + if [[ -z "$(git show-ref --quiet ${name})" ]] + then + git rm -rf branch/${name} + fi + done - git config user.name github-actions[bot] - git config user.email github-actions[bot]@users.noreply.github.com - git commit -m "Clean up branch previews" - git push + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com + git commit -m "Clean up branch previews" + git push + fi build-and-deploy: permissions: contents: write # for peaceiris/actions-gh-pages to push From 539aebd3b5c7c279f097a0b8f4e2d84752848801 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 08:57:43 -0700 Subject: [PATCH 05/15] Allow branch cleanup on workflow dispatch --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a6c4ced353d..5a7191c8f37 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,7 @@ jobs: contents: write # for git push name: Cleanup branch previews runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' }} + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' steps: - uses: actions/checkout@v3 with: From 6cc400379c4439f3d4d76b517d15f4faa3316f83 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 08:59:48 -0700 Subject: [PATCH 06/15] Adds some debug messages --- .github/workflows/docs.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5a7191c8f37..4d1d33545c1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,12 +35,16 @@ jobs: run: | pushd $GITHUB_WORKSPACE/gh-pages + ls -la "$GITHUB_WORKSPACE/gh-pages/branch" + if [[ -e "$GITHUB_WORKSPACE/gh-pages/branch" ]]; then for name in `ls branch/` do if [[ -z "$(git show-ref --quiet ${name})" ]] then git rm -rf branch/${name} + + echo "Removed $GITHUB_WORKSPACE/gh-pages/branch/${name}" fi done From 3f5bb50d60278424a1a3360c4e7e073213b953a3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 15 May 2024 10:00:21 -0600 Subject: [PATCH 07/15] update test to handle new changes to xmlchange --- CIME/tests/test_sys_create_newcase.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index a0f07a001b6..800e690bb74 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -318,7 +318,10 @@ def test_e_xmlquery(self): self.assertTrue(output == str(STOP_N), msg="%s != %s" % (output, STOP_N)) cmd = xmlquery + " --non-local BUILD_COMPLETE --value" output = utils.run_cmd_no_fail(cmd, from_dir=casedir) - self.assertTrue(output == "TRUE", msg="%s != %s" % (output, BUILD_COMPLETE)) + output = output == "TRUE" + self.assertTrue( + output == BUILD_COMPLETE, msg="%s != %s" % (output, BUILD_COMPLETE) + ) # we expect DOCN_MODE to be undefined in this X compset # this test assures that we do not try to resolve this as a compvar cmd = xmlquery + " --non-local DOCN_MODE --value" From 036cce985763f42173c167a7666108333525b5f6 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 09:01:14 -0700 Subject: [PATCH 08/15] Fixes debug messages --- .github/workflows/docs.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4d1d33545c1..f819a5ccc96 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,9 +35,9 @@ jobs: run: | pushd $GITHUB_WORKSPACE/gh-pages - ls -la "$GITHUB_WORKSPACE/gh-pages/branch" - if [[ -e "$GITHUB_WORKSPACE/gh-pages/branch" ]]; then + ls -la "$GITHUB_WORKSPACE/gh-pages/branch" + for name in `ls branch/` do if [[ -z "$(git show-ref --quiet ${name})" ]] @@ -53,6 +53,8 @@ jobs: git commit -m "Clean up branch previews" git push fi + + echo "Done cleaning branches" build-and-deploy: permissions: contents: write # for peaceiris/actions-gh-pages to push From 8f5c0fecaf71297e1fd97c00f2d05783289cfb63 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 15 May 2024 14:25:23 -0600 Subject: [PATCH 09/15] cesm3 will not use manage-externals --- CIME/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CIME/utils.py b/CIME/utils.py index 7471c2e4f4c..2949508dbe3 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -534,7 +534,9 @@ def get_model(): if model is None: srcroot = get_src_root() - if os.path.isfile(os.path.join(srcroot, "Externals.cfg")): + if os.path.isfile(os.path.join(srcroot, "bin", "git-fleximod")): + model = "cesm" + elif os.path.isfile(os.path.join(srcroot, "Externals.cfg")): model = "cesm" with open(os.path.join(srcroot, "Externals.cfg")) as fd: for line in fd: From 8641b38bd1fd874d7802044a1b5d82b256a5b918 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 16 May 2024 11:28:58 -0600 Subject: [PATCH 10/15] remove reference to cice5 and ww3dev, no longer required --- CIME/XML/archive_base.py | 9 +-------- CIME/case/case_st_archive.py | 4 ---- CIME/data/config/cesm/config_files.xml | 18 +++++------------- CIME/hist_utils.py | 6 ------ 4 files changed, 6 insertions(+), 31 deletions(-) diff --git a/CIME/XML/archive_base.py b/CIME/XML/archive_base.py index fa42e186937..1b4ef9e7bd9 100644 --- a/CIME/XML/archive_base.py +++ b/CIME/XML/archive_base.py @@ -127,10 +127,6 @@ def get_all_hist_files(self, casename, model, from_dir, suffix="", ref_case=None # remove when component name is changed if model == "fv3gfs": model = "fv3" - if model == "cice5": - model = "cice" - if model == "ww3dev": - model = "ww3" hist_files = [] extensions = self.get_hist_file_extensions(self.get_entry(dmodel)) @@ -237,10 +233,7 @@ def _get_extension(model, filepath, ext_regexes): # Remove with component namechange if model == "fv3gfs": model = "fv3" - if model == "cice5": - model = "cice" - if model == "ww3dev": - model = "ww3" + basename = os.path.basename(filepath) m = None if ext_regexes is None: diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index 8238cf2f912..d735d6c587a 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -533,10 +533,6 @@ def _archive_restarts_date_comp( # the compname is drv but the files are named cpl if compname == "drv": compname = "cpl" - if compname == "cice5": - compname = "cice" - if compname == "ww3dev": - compname = "ww3" # get file_extension suffixes for suffix in archive.get_rest_file_extensions(archive_entry): diff --git a/CIME/data/config/cesm/config_files.xml b/CIME/data/config/cesm/config_files.xml index e128a870af9..688a8ad5437 100644 --- a/CIME/data/config/cesm/config_files.xml +++ b/CIME/data/config/cesm/config_files.xml @@ -179,7 +179,6 @@ unset $SRCROOT/components/ww3/ - $SRCROOT/components/ww3dev $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/dwav $SRCROOT/components/cdeps/dwav $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/swav @@ -309,14 +308,12 @@ $COMP_ROOT_DIR_ATM/cime_config/config_compsets.xml $COMP_ROOT_DIR_GLC/cime_config/config_compsets.xml $COMP_ROOT_DIR_LND/cime_config/config_compsets.xml - $COMP_ROOT_DIR_LND/cime_config/config_compsets.xml - $COMP_ROOT_DIR_ICE/cime_config/config_compsets.xml $COMP_ROOT_DIR_ICE/cime_config/config_compsets.xml $COMP_ROOT_DIR_OCN/cime_config/config_compsets.xml $COMP_ROOT_DIR_OCN/cime_config/config_compsets.xml $COMP_ROOT_DIR_OCN/cime_config/config_compsets.xml $COMP_ROOT_DIR_OCN/cime_config/config_compsets.xml - $COMP_ROOT_DIR_WAV/cime_config/config_compsets.xml + $COMP_ROOT_DIR_WAV/cime_config/config_compsets.xml case_last env_case.xml @@ -334,14 +331,12 @@ $COMP_ROOT_DIR_ATM/cime_config/config_pes.xml $COMP_ROOT_DIR_GLC/cime_config/config_pes.xml $COMP_ROOT_DIR_LND/cime_config/config_pes.xml - $COMP_ROOT_DIR_LND/cime_config/config_pes.xml - $COMP_ROOT_DIR_ICE/cime_config/config_pes.xml $COMP_ROOT_DIR_ICE/cime_config/config_pes.xml $COMP_ROOT_DIR_OCN/cime_config/config_pes.xml $COMP_ROOT_DIR_OCN/cime_config/config_pes.xml $COMP_ROOT_DIR_OCN/cime_config/config_pes.xml $COMP_ROOT_DIR_OCN/cime_config/config_pes.xml - $COMP_ROOT_DIR_WAV/cime_config/config_pes.xml + $COMP_ROOT_DIR_WAV/cime_config/config_pes.xml case_last env_case.xml @@ -366,8 +361,6 @@ $COMP_ROOT_DIR_ATM/cime_config/config_archive.xml $COMP_ROOT_DIR_GLC/cime_config/config_archive.xml $COMP_ROOT_DIR_LND/cime_config/config_archive.xml - $COMP_ROOT_DIR_LND/cime_config/config_archive.xml - $COMP_ROOT_DIR_ICE/cime_config/config_archive.xml $COMP_ROOT_DIR_ICE/cime_config/config_archive.xml $COMP_ROOT_DIR_OCN/cime_config/config_archive.xml $COMP_ROOT_DIR_OCN/cime_config/config_archive.xml @@ -376,7 +369,7 @@ $COMP_ROOT_DIR_ROF/cime_config/config_archive.xml $COMP_ROOT_DIR_ROF/cime_config/config_archive.xml $COMP_ROOT_DIR_ROF/cime_config/config_archive.xml - $COMP_ROOT_DIR_WAV/cime_config/config_archive.xml + $COMP_ROOT_DIR_WAV/cime_config/config_archive.xml case_last env_case.xml @@ -389,18 +382,17 @@ $CIMEROOT/CIME/SystemTests $COMP_ROOT_DIR_LND/cime_config/SystemTests - $COMP_ROOT_DIR_LND/cime_config/SystemTests $COMP_ROOT_DIR_ATM/cime_config/SystemTests $COMP_ROOT_DIR_OCN/cime_config/SystemTests $COMP_ROOT_DIR_OCN/cime_config/SystemTests $COMP_ROOT_DIR_OCN/cime_config/SystemTests $COMP_ROOT_DIR_OCN/cime_config/SystemTests - $COMP_ROOT_DIR_ICE/cime_config/SystemTests $COMP_ROOT_DIR_ICE/cime_config/SystemTests $COMP_ROOT_DIR_GLC/cime_config/SystemTests $COMP_ROOT_DIR_ROF/cime_config/SystemTests $COMP_ROOT_DIR_ROF/cime_config/SystemTests $COMP_ROOT_DIR_ROF/cime_config/SystemTests + $COMP_ROOT_DIR_WAV/cime_config/SystemTests test env_test.xml @@ -426,7 +418,7 @@ $COMP_ROOT_DIR_ROF/cime_config/testdefs/testlist_rtm.xml $COMP_ROOT_DIR_ROF/cime_config/testdefs/testlist_mosart.xml $COMP_ROOT_DIR_ROF/cime_config/testdefs/testlist_mizuRoute.xml - $COMP_ROOT_DIR_WAV/cime_config/testdefs/testlist_ww3dev.xml + $COMP_ROOT_DIR_WAV/cime_config/testdefs/testlist_ww3.xml $SRCROOT/components/cdeps/datm/cime_config/testdefs/testlist_datm.xml $SRCROOT/components/cdeps/dice/cime_config/testdefs/testlist_dice.xml $SRCROOT/components/cdeps/dlnd/cime_config/testdefs/testlist_dlnd.xml diff --git a/CIME/hist_utils.py b/CIME/hist_utils.py index 86ce16e6d43..b67301c2824 100644 --- a/CIME/hist_utils.py +++ b/CIME/hist_utils.py @@ -181,9 +181,6 @@ def _hists_match(model, hists1, hists2, suffix1="", suffix2=""): multi_normalized1, multi_normalized2 = [], [] multiinst = False - if model == "ww3dev": - model = "ww3" - for hists, suffix, normalized, multi_normalized in [ (hists1, suffix1, normalized1, multi_normalized1), (hists2, suffix2, normalized2, multi_normalized2), @@ -630,9 +627,6 @@ def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=Fa logger.debug("latest_files: {}".format(hists)) num_gen += len(hists) - if model == "ww3dev": - model = "ww3" - for hist in hists: offset = hist.rfind(model) expect( From a4907872862872364c16a741a92c7efa9d5f0f03 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 21 May 2024 12:52:51 -0700 Subject: [PATCH 11/15] Fixes xmlchange calling check_lockedfiles when using --file argument --- CIME/Tools/xmlchange | 31 +++++++++++++++++++++++++++---- CIME/locked_files.py | 18 ++++++++++++------ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 1c53e8f7b6f..ed141994d84 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -192,7 +192,7 @@ def parse_command_line(args, description): def xmlchange_single_value( - case, xmlid, xmlval, subgroup, append, force, dryrun, env_test + case, xmlid, xmlval, subgroup, append, force, dryrun, env_test, caseroot, xmlfile ): if xmlid in [ "THREAD_COUNT", @@ -227,7 +227,12 @@ def xmlchange_single_value( expect(result is not None, 'No variable "%s" found' % xmlid) - check_lockedfiles(case, skip=["env_case"], quiet=True) + if xmlfile is not None: + xmlfile = [xmlfile] + # Pass caseroot, in cases where --file is used and case does not have access to env_case.xml + check_lockedfiles( + case, skip=["env_case"], quiet=True, caseroot=caseroot, whitelist=xmlfile + ) else: logger.warning("'%s' = '%s'", xmlid, xmlval) @@ -283,11 +288,29 @@ def xmlchange( ) (xmlid, xmlval) = pair xmlchange_single_value( - case, xmlid, xmlval, subgroup, append, force, dryrun, env_test + case, + xmlid, + xmlval, + subgroup, + append, + force, + dryrun, + env_test, + caseroot, + xmlfile, ) else: xmlchange_single_value( - case, xmlid, xmlval, subgroup, append, force, dryrun, env_test + case, + xmlid, + xmlval, + subgroup, + append, + force, + dryrun, + env_test, + caseroot, + xmlfile, ) if not noecho: diff --git a/CIME/locked_files.py b/CIME/locked_files.py index efb3f0647e9..3be6f9c3573 100644 --- a/CIME/locked_files.py +++ b/CIME/locked_files.py @@ -1,4 +1,3 @@ -import glob from pathlib import Path from CIME.utils import safe_copy @@ -55,7 +54,7 @@ def is_locked(filename, caseroot): return os.path.exists(os.path.join(caseroot, LOCKED_DIR, filename)) -def check_lockedfiles(case, skip=None, quiet=False): +def check_lockedfiles(case, skip=None, quiet=False, caseroot=None, whitelist=None): """ Check that all lockedfiles match what's in case @@ -66,18 +65,25 @@ def check_lockedfiles(case, skip=None, quiet=False): elif isinstance(skip, str): skip = [skip] - caseroot = case.get_value("CASEROOT") + if caseroot is None: + caseroot = case.get_value("CASEROOT") + + locked_path = Path(caseroot, LOCKED_DIR) + + lockedfiles = locked_path.glob("*.xml") - lockedfiles = glob.glob(os.path.join(caseroot, LOCKED_DIR, "*.xml")) + # filter based on whitelist + if whitelist is not None: + lockedfiles = [x for x in lockedfiles if x.stem in whitelist] for file_path in lockedfiles: - filename = os.path.basename(file_path) + filename = file_path.name # Skip files used for tests e.g. env_mach_pes.ERP1.xml or included in skip list if filename.count(".") > 1 or any([filename.startswith(x) for x in skip]): continue - check_lockedfile(case, filename, caseroot=caseroot, quiet=quiet) + check_lockedfile(case, f"{filename}", caseroot=caseroot, quiet=quiet) def check_lockedfile(case, filebase, caseroot=None, quiet=False): From ead5c78fd46171be64a3807fdd4b1b640975edc6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 11 Jun 2024 10:33:26 -0600 Subject: [PATCH 12/15] More robust approach to waiting for many threads Using threading.active_count doesn't work if there are other threads being used in the process. Both jenkins_generic_job and wait_for_tests use threads and the jenkins_generic_job thread was causing wait_for_tests to wait when it shouldn't have. Instead, track the set of threads you want to wait on in a list. --- CIME/build.py | 6 ++++-- CIME/wait_for_tests.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CIME/build.py b/CIME/build.py index 3845de47f2d..a522b599763 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -338,6 +338,7 @@ def _build_model( thread_bad_results = [] libroot = os.path.join(exeroot, "lib") bldroot = None + bld_threads = [] for model, comp, nthrds, _, config_dir in complist: if buildlist is not None and model.lower() not in buildlist: continue @@ -391,12 +392,13 @@ def _build_model( ), ) t.start() + bld_threads.append(t) logs.append(file_build) # Wait for threads to finish - while threading.active_count() > 1: - time.sleep(1) + for bld_thread in bld_threads: + bld_thread.join() expect(not thread_bad_results, "\n".join(thread_bad_results)) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index a6e87eaac8c..7636d5f8986 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -658,6 +658,7 @@ def wait_for_tests_impl( ############################################################################### results = queue.Queue() + wft_threads = [] for test_path in test_paths: t = threading.Thread( target=wait_for_test, @@ -675,9 +676,10 @@ def wait_for_tests_impl( ) t.daemon = True t.start() + wft_threads.append(t) - while threading.active_count() > 1: - time.sleep(1) + for wft_thread in wft_threads: + wft_thread.join() test_results = {} completed_test_paths = [] From bb93451b46c7192d2132cc772a8efe78d54012b5 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 11 Jun 2024 16:05:08 -0600 Subject: [PATCH 13/15] Fix test_cime_case_test_walltime_mgmt_8 Theta machine is no longer known to E3SM. --- CIME/tests/test_sys_cime_case.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index dabea7d9600..4b226ff3b46 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -509,8 +509,10 @@ def test_cime_case_test_walltime_mgmt_8(self): if self._config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") - test_name = "SMS_P25600.f19_g16_rx1.A" - machine, compiler = "theta", "gnu" + # Frontier has 56 MAX_MPITASKS_PER_NODE so 5600 should require 100 nodes + # which should land us in 6 hour queue + test_name = "SMS_P5600.f19_g16_rx1.A" + machine, compiler = "frontier", "gnu" casedir = self._create_test( [ "--no-setup", @@ -528,12 +530,12 @@ def test_cime_case_test_walltime_mgmt_8(self): "./xmlquery JOB_WALLCLOCK_TIME -N --subgroup=case.test --value", from_dir=casedir, ) - self.assertEqual(result, "09:00:00") + self.assertEqual(result, "06:00:00") result = self.run_cmd_assert_result( "./xmlquery JOB_QUEUE -N --subgroup=case.test --value", from_dir=casedir ) - self.assertEqual(result, "default") + self.assertEqual(result, "batch") def test_cime_case_test_custom_project(self): test_name = "ERS_P1.f19_g16_rx1.A" From 4e14c865fbd7d42d8dd79d049bf8366d36458cc0 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 12 Jun 2024 14:25:43 -0600 Subject: [PATCH 14/15] remove mct from all cesm testing --- CIME/data/config/cesm/config_files.xml | 105 +++++++++---------------- CIME/tests/base.py | 2 +- CIME/tests/test_sys_full_system.py | 4 +- 3 files changed, 42 insertions(+), 69 deletions(-) diff --git a/CIME/data/config/cesm/config_files.xml b/CIME/data/config/cesm/config_files.xml index e128a870af9..456ba8439ad 100644 --- a/CIME/data/config/cesm/config_files.xml +++ b/CIME/data/config/cesm/config_files.xml @@ -28,7 +28,7 @@ char $SRCROOT/ccs_config/config_grids.xml - $SRCROOT/ccs_config/config_grids_nuopc.xml + $SRCROOT/ccs_config/config_grids_nuopc.xml case_last env_case.xml @@ -122,12 +122,10 @@ char - $SRCROOT/components/cdeps/datm - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/datm - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/satm - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/satm - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xatm - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xatm + $SRCROOT/components/cdeps/datm + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/satm + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xatm + $SRCROOT/components/cam/ $SRCROOT/components/fv3/ @@ -140,12 +138,9 @@ char - $SRCROOT/components/cmeps - $SRCROOT/components/cmeps - $SRCROOT/components/cmeps - $SRCROOT/components/cpl7/driver - $SRCROOT/components/cpl7/driver - $SRCROOT/components/cpl7/driver + $SRCROOT/components/cmeps + $SRCROOT/components/cmeps + $SRCROOT/components/cmeps case_comps env_case.xml @@ -161,12 +156,9 @@ $SRCROOT/components/mom/ $SRCROOT/components/nemo/ $SRCROOT/components/blom/ - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/docn - $SRCROOT/components/cdeps/docn - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/socn - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/socn - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xocn - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xocn + $SRCROOT/components/cdeps/docn + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/socn + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xocn case_comps env_case.xml @@ -179,13 +171,9 @@ unset $SRCROOT/components/ww3/ - $SRCROOT/components/ww3dev - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/dwav - $SRCROOT/components/cdeps/dwav - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/swav - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/swav - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xwav - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xwav + $SRCROOT/components/cdeps/dwav + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/swav + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xwav case_comps env_case.xml @@ -198,12 +186,9 @@ unset $SRCROOT/components/cism/ - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/dglc - $SRCROOT/components/cdeps/dglc - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/sglc - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/sglc - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xglc - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xglc + $SRCROOT/components/cdeps/dglc + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/sglc + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xglc case_comps env_case.xml @@ -217,12 +202,9 @@ $SRCROOT/components/cice5/ $SRCROOT/components/cice/ - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/dice - $SRCROOT/components/cdeps/dice - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/sice - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/sice - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xice - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xice + $SRCROOT/components/cdeps/dice + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/sice + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xice case_comps env_case.xml @@ -237,12 +219,9 @@ $SRCROOT/components/rtm/ $SRCROOT/components/mosart/ $SRCROOT/components/mizuRoute/ - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/drof - $SRCROOT/components/cdeps/drof - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/srof - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/srof - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xrof - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xrof + $SRCROOT/components/cdeps/drof + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/srof + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xrof case_comps env_case.xml @@ -256,12 +235,9 @@ $SRCROOT/components/clm/ $SRCROOT/components/slim/ - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/dlnd - $SRCROOT/components/cdeps/dlnd - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/slnd - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/slnd - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xlnd - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xlnd + $SRCROOT/components/cdeps/dlnd + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/slnd + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xlnd case_comps env_case.xml @@ -273,10 +249,8 @@ char unset - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/siac - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/siac - $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xiac - $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xiac + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/siac + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xiac case_comps env_case.xml @@ -288,10 +262,8 @@ char unset - $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/desp - $CIMEROOT/CIME/non_py/src/components/data_comps_$COMP_INTERFACE/desp - $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/sesp - $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/sesp + $CIMEROOT/CIME/non_py/src/components/data_comps_$COMP_INTERFACE/desp + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/sesp case_comps env_case.xml @@ -460,13 +432,13 @@ $COMP_ROOT_DIR_OCN/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_OCN/cime_config/testdefs/testmods_dirs $COMP_ROOT_DIR_OCN/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/datm/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/dice/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/dlnd/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/docn/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/drof/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/dglc/cime_config/testdefs/testmods_dirs - $SRCROOT/components/cdeps/dwav/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/datm/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/dice/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/dlnd/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/docn/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/drof/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/dglc/cime_config/testdefs/testmods_dirs + $SRCROOT/components/cdeps/dwav/cime_config/testdefs/testmods_dirs case_last env_case.xml @@ -551,8 +523,7 @@ char - $SRCROOT/components/cmeps/cime_config/config_component_$MODEL.xml --> - $SRCROOT/components/cpl7/driver/cime_config/config_component_$MODEL.xml --> + $SRCROOT/components/cmeps/cime_config/config_component_$MODEL.xml --> case_last env_case.xml diff --git a/CIME/tests/base.py b/CIME/tests/base.py index 524900fac82..b2845575455 100644 --- a/CIME/tests/base.py +++ b/CIME/tests/base.py @@ -211,7 +211,7 @@ def _create_test( driver = utils.get_cime_default_driver() if driver == "nuopc" and "cime_developer" in extra_args: extra_args.append( - " ^SMS_Ln3.T42_T42.S ^PRE.f19_f19.ADESP_TEST ^PRE.f19_f19.ADESP ^DAE.ww3a.ADWAV" + " ^SMS_Ln3.T42_T42.S ^PRE.f19_f19.ADESP_TEST ^PRE.f19_f19.ADESP ^DAE.ww3a.ADWAV ^IRT_N2_Vmct_Ln9.f19_g16_rx1.A" ) test_id = ( diff --git a/CIME/tests/test_sys_full_system.py b/CIME/tests/test_sys_full_system.py index 854e782ac1f..186bb256eba 100644 --- a/CIME/tests/test_sys_full_system.py +++ b/CIME/tests/test_sys_full_system.py @@ -47,6 +47,7 @@ def test_full_system(self): "PRE.f19_f19.ADESP_TEST", "PRE.f19_f19.ADESP", "DAE.ww3a.ADWAV", + "IRT_N2_Vmct_Ln9.f19_g16_rx1.A", ] tests = get_tests.get_test_suite( "cime_developer", @@ -54,7 +55,8 @@ def test_full_system(self): compiler=self._compiler, skip_tests=skip_tests, ) - + print(f"tests are {tests}") + self.assertTrue(False,msg="Stop here") for test in tests: casedir = self.get_casedir(test, cases) From 70dc6cd99b890874db07dcf5c495b02b71acda8e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 12 Jun 2024 16:17:17 -0600 Subject: [PATCH 15/15] black reformat file --- CIME/tests/test_sys_full_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/tests/test_sys_full_system.py b/CIME/tests/test_sys_full_system.py index 186bb256eba..22a6708436b 100644 --- a/CIME/tests/test_sys_full_system.py +++ b/CIME/tests/test_sys_full_system.py @@ -56,7 +56,7 @@ def test_full_system(self): skip_tests=skip_tests, ) print(f"tests are {tests}") - self.assertTrue(False,msg="Stop here") + self.assertTrue(False, msg="Stop here") for test in tests: casedir = self.get_casedir(test, cases)