From 76569cdb32f1861ff81b2ebc1878ae980a950321 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 28 Jun 2023 10:20:10 -0600 Subject: [PATCH 001/173] More confined regular expressing fixing #2039 --- cime_config/usermods_dirs/NEON/defaults/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 437201f2b2..1f5427ca98 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -32,7 +32,7 @@ else fi # If needed for SP simulations: & set history file variables -if [[ $compset =~ .*CLM[0-9]+%.*SP.* ]]; then +if [[ $compset =~ .*CLM[0-9]+%[^_]*SP.* ]]; then if [[ $TEST != "TRUE" ]]; then ./xmlchange STOP_OPTION=nyears fi From 8aa0a680db01f7cc345900aa5c194bc182f15e10 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 11 Aug 2023 21:18:51 -0600 Subject: [PATCH 002/173] Put NEON site fsurdat files in the namelist_defaults file, and get it connected, fixing #2028 --- bld/CLMBuildNamelist.pm | 56 ++++++++++++++----- bld/env_run.xml | 1 + bld/namelist_files/namelist_defaults_ctsm.xml | 11 ++++ bld/unit_testers/build-namelist_test.pl | 4 +- 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 69b3a8c3dc..98cce06fca 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -611,7 +611,7 @@ sub process_namelist_user_input { process_namelist_commandline_infile($opts, $definition, $nl, $envxml_ref); # Apply the commandline options and make sure the user didn't change it above - process_namelist_commandline_options($opts, $nl_flags, $definition, $defaults, $nl, $physv); + process_namelist_commandline_options($opts, $nl_flags, $definition, $defaults, $nl, $envxml_ref, $physv); # The last two process command line arguments for usr_name and use_case # They require that process_namelist_commandline_options was called before this @@ -632,10 +632,10 @@ sub process_namelist_commandline_options { # Obtain default values for the following build-namelist input arguments # : res, mask, ssp_rcp, sim_year, sim_year_range, and clm_accelerated_spinup. - my ($opts, $nl_flags, $definition, $defaults, $nl, $physv) = @_; + my ($opts, $nl_flags, $definition, $defaults, $nl, $envxml_ref, $physv) = @_; setup_cmdl_chk_res($opts, $defaults); - setup_cmdl_resolution($opts, $nl_flags, $definition, $defaults); + setup_cmdl_resolution($opts, $nl_flags, $definition, $defaults, $envxml_ref); setup_cmdl_mask($opts, $nl_flags, $definition, $defaults, $nl); setup_cmdl_configuration_and_structure($opts, $nl_flags, $definition, $defaults, $nl); setup_cmdl_bgc($opts, $nl_flags, $definition, $defaults, $nl); @@ -666,7 +666,7 @@ sub setup_cmdl_chk_res { } sub setup_cmdl_resolution { - my ($opts, $nl_flags, $definition, $defaults) = @_; + my ($opts, $nl_flags, $definition, $defaults, $envxml_ref) = @_; my $var = "res"; my $val; @@ -690,10 +690,20 @@ sub setup_cmdl_resolution { } } # For NEON sites - if ($nl_flags->{'res'} =~ /NEON/) { - $nl_flags->{'neon'} = ".true." - } else { - $nl_flags->{'neon'} = ".false." + $nl_flags->{'neon'} = ".false."; + $nl_flags->{'neonsite'} = ""; + if ( $nl_flags->{'res'} eq "CLM_USRDAT" ) { + if ( ! defined($opts->{'clm_usr_name'}) ) { + $log->fatal_error("Resolution is CLM_USRDAT, but --clm_usr_name option is NOT set, and it is required for CLM_USRDAT resolutions"); + } + if ( $opts->{'clm_usr_name'} eq "NEON" ) { + $nl_flags->{'neon'} = ".true."; + $nl_flags->{'neonsite'} = $envxml_ref->{'NEONSITE'}; + $log->verbose_message( "This is a NEON site with NEONSITE = " . $nl_flags->{'neonsite'} ); + } + } + if ( ! &value_is_true( $nl_flags->{'neon'} ) ) { + $log->verbose_message( "This is NOT a NEON site" ); } } @@ -1583,7 +1593,7 @@ sub process_namelist_inline_logic { setup_logic_grainproduct($opts, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_soilstate($opts, $nl_flags, $definition, $defaults, $nl); setup_logic_demand($opts, $nl_flags, $definition, $defaults, $nl); - setup_logic_surface_dataset($opts, $nl_flags, $definition, $defaults, $nl); + setup_logic_surface_dataset($opts, $nl_flags, $definition, $defaults, $nl, $envxml_ref); setup_logic_dynamic_subgrid($opts, $nl_flags, $definition, $defaults, $nl); if ( remove_leading_and_trailing_quotes($nl_flags->{'clm_start_type'}) ne "branch" ) { setup_logic_initial_conditions($opts, $nl_flags, $definition, $defaults, $nl, $physv); @@ -2305,7 +2315,7 @@ sub setup_logic_surface_dataset { # consistent with it # MUST BE AFTER: setup_logic_demand which is where flanduse_timeseries is set # - my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; + my ($opts, $nl_flags, $definition, $defaults, $nl, $xmlvar_ref) = @_; $nl_flags->{'flanduse_timeseries'} = "null"; my $flanduse_timeseries = $nl->get_value('flanduse_timeseries'); @@ -2330,26 +2340,42 @@ sub setup_logic_surface_dataset { if ( ! &value_is_true($nl_flags->{'use_fates'}) ) { add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, 'hgrid'=>$nl_flags->{'res'}, 'ssp_rcp'=>$nl_flags->{'ssp_rcp'}, + 'neon'=>$nl_flags->{'neon'}, 'neonsite'=>$nl_flags->{'neonsite'}, 'sim_year'=>$nl_flags->{'sim_year'}, 'irrigate'=>".true.", 'use_vichydro'=>$nl_flags->{'use_vichydro'}, - 'use_crop'=>".true.", 'glc_nec'=>$nl_flags->{'glc_nec'}, 'nofail'=>1); + 'use_crop'=>".true.", 'glc_nec'=>$nl_flags->{'glc_nec'}, 'use_fates'=>$nl_flags->{'use_fates'}, 'nofail'=>1); } # If didn't find the crop version check for the exact match - if ( ! defined($nl->get_value($var) ) ) { + my $fsurdat = $nl->get_value($var); + if ( ! defined($fsurdat) ) { if ( ! &value_is_true($nl_flags->{'use_fates'}) ) { $log->verbose_message( "Crop version of $var NOT found, searching for an exact match" ); } add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, 'hgrid'=>$nl_flags->{'res'}, 'ssp_rcp'=>$nl_flags->{'ssp_rcp'}, 'use_vichydro'=>$nl_flags->{'use_vichydro'}, - 'sim_year'=>$nl_flags->{'sim_year'}, 'irrigate'=>$nl_flags->{'irrigate'}, + 'sim_year'=>$nl_flags->{'sim_year'}, 'irrigate'=>$nl_flags->{'irrigate'}, 'use_fates'=>$nl_flags->{'use_fates'}, + 'neon'=>$nl_flags->{'neon'}, 'neonsite'=>$nl_flags->{'neonsite'}, 'use_crop'=>$nl_flags->{'use_crop'}, 'glc_nec'=>$nl_flags->{'glc_nec'}, 'nofail'=>1 ); - if ( ! defined($nl->get_value($var) ) ) { + if ( ! defined($fsurdat) ) { $log->verbose_message( "Exact match of $var NOT found, searching for version with irrigate true" ); } add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, 'hgrid'=>$nl_flags->{'res'}, 'ssp_rcp'=>$nl_flags->{'ssp_rcp'}, 'use_vichydro'=>$nl_flags->{'use_vichydro'}, - 'sim_year'=>$nl_flags->{'sim_year'}, 'irrigate'=>".true.", + 'sim_year'=>$nl_flags->{'sim_year'}, 'irrigate'=>".true.", 'use_fates'=>$nl_flags->{'use_fates'}, + 'neon'=>$nl_flags->{'neon'}, 'neonsite'=>$nl_flags->{'neonsite'}, 'use_crop'=>$nl_flags->{'use_crop'}, 'glc_nec'=>$nl_flags->{'glc_nec'} ); } + # + # Expand the XML variables for NEON cases so that NEONSITE will be used + # + if ( &value_is_true($nl_flags->{'neon'}) ) { + my $fsurdat = $nl->get_value($var); + my $newval = SetupTools::expand_xml_var( $fsurdat, $xmlvar_ref ); + if ( $newval ne $fsurdat ) { + my $group = $definition->get_group_name($var); + $nl->set_variable_value($group, $var, $newval); + $log->verbose_message( "This is a NEON site and the fsurdat file selected is: $newval" ); + } + } } #------------------------------------------------------------------------------- diff --git a/bld/env_run.xml b/bld/env_run.xml index 8bf59d0911..f3b7467168 100644 --- a/bld/env_run.xml +++ b/bld/env_run.xml @@ -9,5 +9,6 @@ Sample env_run.xml file that allows build-namelist to be run for testing in this --> + diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 1d895524e6..666b19a08f 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1275,6 +1275,17 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/surfdata_map/surfdata_0.9x1.25_hist_16pfts_nourb_CMIP6_simyrPtVg_c181114.nc + + +lnd/clm2/surfdata_map/NEON/16PFT_mixed/surfdata_1x1_NEON_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230120.nc + +lnd/clm2/surfdata_map/NEON/surfdata_1x1_NEON_${NEONSITE}_hist_78pfts_CMIP6_simyr2000_c230601.nc + + + "MYDINLOCROOT", GLC_TWO_WAY_COUPLING=>"FALSE" ); + my %env_vars = ( DIN_LOC_ROOT=>"MYDINLOCROOT", GLC_TWO_WAY_COUPLING=>"FALSE", NEONSITE=>"" ); # Set any settings that came in from function call foreach my $item ( keys(%settings) ) { $env_vars{$item} = $settings{$item}; @@ -380,7 +380,7 @@ sub cat_and_create_namelistinfile { "JORN", "LAJA", "MOAB", "OAES", "OSBS", "SCBI", "SOAP", "STER", "TOOL", "UNDE", "YELL" ) { - &make_env_run(); + &make_env_run( NEONSITE=>"$site" ); # # Concatonate default usermods and specific sitetogether expanding env variables while doing that # From df4c392756de1fde33019a817aa05c97a0bf0af1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 14 Aug 2023 16:38:54 -0600 Subject: [PATCH 003/173] Fix for #2103 making sure --res sent to build-namelist is CLM_USRDAT rather than \$CLM_USRDAT_NAME as it is now --- cime_config/buildnml | 1 - 1 file changed, 1 deletion(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index e239f0ec58..84e1581406 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -136,7 +136,6 @@ def buildnml(case, caseroot, compname): clmusr = "" if lnd_grid == "CLM_USRDAT": clm_usrdat_name = case.get_value("CLM_USRDAT_NAME") - lnd_grid = clm_usrdat_name clmusr = " -clm_usr_name %s " % clm_usrdat_name # Write warning about initial condition data if "NEON" in clm_usrdat_name and clm_force_coldstart == "off": From 7f2475c711089d90813807ed16aa7e59554c96dc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 14 Aug 2023 16:53:44 -0600 Subject: [PATCH 004/173] Complete build-namelist work needed for #2103 and #2028 having res be CLM_USRDAT when clm_usr_name option is set and setting fsurdat and flanduse in namelist defaults for NEON sites --- bld/CLMBuildNamelist.pm | 13 +++++++++---- bld/namelist_files/namelist_defaults_ctsm.xml | 6 ++++++ bld/namelist_files/namelist_defaults_overall.xml | 6 ++++-- bld/namelist_files/use_cases/2018-PD_transient.xml | 6 +++++- bld/namelist_files/use_cases/2018_control.xml | 3 +++ bld/unit_testers/build-namelist_test.pl | 2 +- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 98cce06fca..5150d05473 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -684,18 +684,22 @@ sub setup_cmdl_resolution { $val = "e_string( $nl_flags->{'res'} ); if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); - if ( ! defined($opts->{'clm_usr_name'}) || $nl_flags->{'res'} ne $opts->{'clm_usr_name'} ) { + if ( $nl_flags->{'res'} ne "CLM_USRDAT" ) { $log->fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values"); } } } - # For NEON sites - $nl_flags->{'neon'} = ".false."; - $nl_flags->{'neonsite'} = ""; if ( $nl_flags->{'res'} eq "CLM_USRDAT" ) { if ( ! defined($opts->{'clm_usr_name'}) ) { $log->fatal_error("Resolution is CLM_USRDAT, but --clm_usr_name option is NOT set, and it is required for CLM_USRDAT resolutions"); } + } + # + # For NEON sites + # + $nl_flags->{'neon'} = ".false."; + $nl_flags->{'neonsite'} = ""; + if ( $nl_flags->{'res'} eq "CLM_USRDAT" ) { if ( $opts->{'clm_usr_name'} eq "NEON" ) { $nl_flags->{'neon'} = ".true."; $nl_flags->{'neonsite'} = $envxml_ref->{'NEONSITE'}; @@ -2263,6 +2267,7 @@ sub setup_logic_demand { $settings{'use_lch4'} = $nl_flags->{'use_lch4'}; $settings{'use_nitrif_denitrif'} = $nl_flags->{'use_nitrif_denitrif'}; $settings{'use_crop'} = $nl_flags->{'use_crop'}; + $settings{'neon'} = $nl_flags->{'neon'}; my $demand = $nl->get_value('clm_demand'); if (defined($demand)) { diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 666b19a08f..7dff7ce4e9 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1288,6 +1288,12 @@ lnd/clm2/surfdata_map/NEON/surfdata_1x1_NEON_TOOL_hist_78pfts_CMIP6_simyr2000_c2 + + lnd/clm2/surfdata_map/landuse.timeseries_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c170824.nc -flanduse_timeseries -flanduse_timeseries +null +null +flanduse_timeseries +flanduse_timeseries diff --git a/bld/namelist_files/use_cases/2018-PD_transient.xml b/bld/namelist_files/use_cases/2018-PD_transient.xml index d838efbd00..96f14207ad 100644 --- a/bld/namelist_files/use_cases/2018-PD_transient.xml +++ b/bld/namelist_files/use_cases/2018-PD_transient.xml @@ -1,8 +1,12 @@ + + -Simulate transient land-use, and aerosol deposition changes from 2018 to current day with a mix of historical data, and future scenario data +Simulate transient Nitrogen-deposition, aerosol deposition, urban, and fire related (pop-density, lightning) changes from 2018 to current day with a mix of historical data, and future scenario data +Simulate transient Nitrogen-deposition, aerosol deposition, urban, and fire related (pop-density, lightning) changes from 2018 to current day with a mix of historical data, and future scenario data +Simulate transient urban and aerosol deposition changes from 2018 to current day with a mix of historical data, and future scenario data diff --git a/bld/namelist_files/use_cases/2018_control.xml b/bld/namelist_files/use_cases/2018_control.xml index e5e572d749..28554074c4 100644 --- a/bld/namelist_files/use_cases/2018_control.xml +++ b/bld/namelist_files/use_cases/2018_control.xml @@ -1,5 +1,8 @@ + + + Conditions to simulate 2018 land-use diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index cdad9d63c8..dc96984bb3 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -399,7 +399,7 @@ sub cat_and_create_namelistinfile { # # Now run the site # - my $options = "-res CLM_USRDAT -clm_usr_name NEON -no-megan -bgc bgc -sim_year 2018 -infile $namelistfile"; + my $options = "--res CLM_USRDAT --clm_usr_name NEON --no-megan --bgc bgc --use_case 2018_control --infile $namelistfile"; eval{ system( "$bldnml -envxml_dir . $options > $tempfile 2>&1 " ); }; is( $@, '', "options: $options" ); $cfiles->checkfilesexist( "$options", $mode ); From ab94ab484bc15405788ffe6c1e061778eeb3d337 Mon Sep 17 00:00:00 2001 From: AdrienDams Date: Fri, 8 Sep 2023 10:44:52 +0200 Subject: [PATCH 005/173] Replace thermal conductivity of snow of Jordan with Sturm --- src/biogeophys/SoilTemperatureMod.F90 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/biogeophys/SoilTemperatureMod.F90 b/src/biogeophys/SoilTemperatureMod.F90 index b868224a60..caeedd1603 100644 --- a/src/biogeophys/SoilTemperatureMod.F90 +++ b/src/biogeophys/SoilTemperatureMod.F90 @@ -47,7 +47,7 @@ module SoilTemperatureMod ! o The thermal conductivity of soil is computed from ! the algorithm of Johansen (as reported by Farouki 1981), and the ! conductivity of snow is from the formulation used in - ! SNTHERM (Jordan 1991). + ! Sturm (1997). ! o Boundary conditions: ! F = Rnet - Hg - LEg (top), F= 0 (base of the soil column). ! o Soil / snow temperature is predicted from heat conduction @@ -100,7 +100,7 @@ subroutine SoilTemperature(bounds, num_urbanl, filter_urbanl, num_urbanc, filter ! o The thermal conductivity of soil is computed from ! the algorithm of Johansen (as reported by Farouki 1981), and the ! conductivity of snow is from the formulation used in - ! SNTHERM (Jordan 1991). + ! Sturm (1997). ! o Boundary conditions: ! F = Rnet - Hg - LEg (top), F= 0 (base of the soil column). ! o Soil / snow temperature is predicted from heat conduction @@ -611,7 +611,7 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter ! ! (2) The thermal conductivity of soil is computed from the algorithm of ! Johansen (as reported by Farouki 1981), and of snow is from the - ! formulation used in SNTHERM (Jordan 1991). + ! formulation used in Sturm (1997). ! The thermal conductivities at the interfaces between two neighboring ! layers (j, j+1) are derived from an assumption that the flux across ! the interface is equal to that from the node j to the interface and the @@ -734,11 +734,16 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter endif endif - ! Thermal conductivity of snow, which from Jordan (1991) pp. 18 + ! Thermal conductivity of snow, which from Sturm (1997) ! Only examine levels from snl(c)+1 -> 0 where snl(c) < 1 if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then - bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) - thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) + bw(c,j) = ((h2osoi_ice(c,j)*1)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) ! ==RHOS + ! thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) ! Original (Jordan) Parameterisation + if (bw(c,j) <= 156) then !LMW or 0.156 ? + thk(c,j) = 0.023 + 0.234*(bw(c,j)/1000) !LMW - units changed by VRD + else !LMW + thk(c,j) = 0.138 - 1.01*(bw(c,j)/1000) +(3.233*((bw(c,j)/1000)*(bw(c,j)/1000))) ! LMW Sturm I think + end if end if end do From 8b75c9b03dd17f61680c17023c0c5b7066a44fb4 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 13 Sep 2023 10:00:23 -0600 Subject: [PATCH 006/173] Moving run_neon.py to /python and creating wrapper script for it --- .../ctsm}/site_and_regional/run_neon.py | 5 -- tools/site_and_regional/run_neon | 47 +++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) rename {tools => python/ctsm}/site_and_regional/run_neon.py (99%) create mode 100755 tools/site_and_regional/run_neon diff --git a/tools/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py similarity index 99% rename from tools/site_and_regional/run_neon.py rename to python/ctsm/site_and_regional/run_neon.py index 84c00715fb..aa6e0c3ffb 100755 --- a/tools/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -875,8 +875,3 @@ def main(description): rerun, experiment, ) - - - -if __name__ == "__main__": - main(__doc__) diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon new file mode 100755 index 0000000000..ad930f50e3 --- /dev/null +++ b/tools/site_and_regional/run_neon @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +""" +This is a just top-level skeleton script that calls +run_neon.py. +The original code (run_neon.py) is located under +python/ctsm/site_and_regional folder. + +For full instructions on how to run the code and different options, +please check python/ctsm/site_and_regional/run_neon.py file. + +This script first creates and builds a generic base case. +Next, it will clone the base_case for different neon sites and run +types to reduce the need to build ctsm everytime. + +This script will do the following: + 1) Create a generic base case for cloning. + 2) Make the case for the specific neon site(s). + 3) Make changes to the case, for: + a. AD spinup + b. post-AD spinup + c. transient + #--------------- + d. SASU or Matrix spinup + 4) Build and submit the case. + +---------------------------------------------------------------- +To see all available options for running tower sites: + ./run_neon --help +---------------------------------------------------------------- +Instructions for running using conda python environments: +../../py_env_create +conda activate ctsm_pylib +""" + +import os +import sys + +# -- add python/ctsm to path +_CTSM_PYTHON = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" +) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.site_and_regional.run_neon import main + +if __name__ == "__main__": + main(__doc__) From 8260253b8bc49b3976e578183f5ccd19389c6a26 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 13 Sep 2023 11:06:31 -0600 Subject: [PATCH 007/173] Moving modify_singlept_site_neon.py to /python and creating wrapper script for it --- .../modify_singlept_site_neon.py | 6 +-- .../modify_singlept_site_neon | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) rename {tools => python/ctsm}/site_and_regional/modify_singlept_site_neon.py (99%) create mode 100755 tools/site_and_regional/modify_singlept_site_neon diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py similarity index 99% rename from tools/site_and_regional/modify_singlept_site_neon.py rename to python/ctsm/site_and_regional/modify_singlept_site_neon.py index e135760a48..c5ae7eb31b 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -153,7 +153,7 @@ def get_parser(): parser.add_argument( "--inputdata-dir", help=""" - Directory to write updated single point surface dataset. + Directory containing standard input files from CESM input data such as the surf_soildepth_file. [default: %(default)s] """, action="store", @@ -735,7 +735,3 @@ def main(): + "):\n - " + wfile ) - - -if __name__ == "__main__": - main() diff --git a/tools/site_and_regional/modify_singlept_site_neon b/tools/site_and_regional/modify_singlept_site_neon new file mode 100755 index 0000000000..1b790a74ca --- /dev/null +++ b/tools/site_and_regional/modify_singlept_site_neon @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +This is a just top-level skeleton script that calls +modify_singlept_site_neon.py. +The original code (modify_singlept_site_neon.py) is located under +python/ctsm/site_and_regional folder. + +For full instructions on how to run the code and different options, +please check python/ctsm/site_and_regional/modify_singlept_site_neon.py file. + +This script is for modifying surface dataset at neon sites +using data available from the neon server. + +After creating a single point surface data file from a global +surface data file using subset_data.py, use this script to +overwrite some fields with site-specific data for neon sites. + +This script will do the following: +- Download neon data for the specified site if it does not exist + in the specified directory : (i.e. ../../../neon_surf_files). +- Modify surface dataset with downloaded data. + +---------------------------------------------------------------- +To see all available options for modifying surface datasets at +tower sites: + ./modify_singlept_site_neon --help +---------------------------------------------------------------- +Instructions for running using conda python environments: +../../py_env_create +conda activate ctsm_pylib +""" + +import os +import sys + +# -- add python/ctsm to path +_CTSM_PYTHON = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" +) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.site_and_regional.modify_singlept_site_neon import main + +if __name__ == "__main__": + main() From 83d36edc75080c5d9fbec97bd284bae02423780e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 13 Sep 2023 11:13:11 -0600 Subject: [PATCH 008/173] Moving neon_surf_wrapper.py to /python and creating wrapper script for it --- .../site_and_regional/neon_surf_wrapper.py | 14 +++---- tools/site_and_regional/neon_surf_wrapper | 37 +++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) rename {tools => python/ctsm}/site_and_regional/neon_surf_wrapper.py (92%) create mode 100755 tools/site_and_regional/neon_surf_wrapper diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/python/ctsm/site_and_regional/neon_surf_wrapper.py similarity index 92% rename from tools/site_and_regional/neon_surf_wrapper.py rename to python/ctsm/site_and_regional/neon_surf_wrapper.py index 3271c72f08..ce9efc4ed5 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/python/ctsm/site_and_regional/neon_surf_wrapper.py @@ -14,7 +14,7 @@ Instructions for running using conda python environments: ../../py_env_create -conda activate ctsm_py +conda activate ctsm_pylib """ # TODO @@ -118,7 +118,7 @@ def main(): subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), '--site',clmsite, '--create-surface','--uniform-snowpack', '--cap-saturation','--verbose','--overwrite'] - modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', 'subset_data_single_point', '--16pft'] elif args.pft_16: # use surface dataset with 16 pfts, but overwrite to 100% 1 dominant PFT @@ -127,7 +127,7 @@ def main(): subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), '--site',clmsite,'--dompft',str(pft),'--create-surface', '--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', 'subset_data_single_point', '--16pft'] elif args.mixed: # use surface dataset with 78 pfts, and don't overwrite with 100% 1 dominant PFT @@ -137,7 +137,7 @@ def main(): subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), '--site',clmsite,'--crop','--create-surface', '--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', 'subset_data_single_point'] else: # use surface dataset with 78 pfts, and overwrite to 100% 1 dominant PFT @@ -147,11 +147,7 @@ def main(): subset_command = ['./subset_data', 'point', '--lat', str(lat), '--lon', str(lon), '--site', clmsite,'--crop', '--dompft', str(pft), '--create-surface', '--uniform-snowpack', '--cap-saturation', '--verbose', '--overwrite'] - modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', 'subset_data_single_point'] execute(subset_command) execute(modify_command) - -if __name__ == "__main__": - main() - diff --git a/tools/site_and_regional/neon_surf_wrapper b/tools/site_and_regional/neon_surf_wrapper new file mode 100755 index 0000000000..306d38a774 --- /dev/null +++ b/tools/site_and_regional/neon_surf_wrapper @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +""" +This is a just top-level skeleton script that calls +neon_surf_wrapper.py. +The original code (neon_surf_wrapper.py) is located under +python/ctsm/site_and_regional folder. + +For full instructions on how to run the code and different options, +please check python/ctsm/site_and_regional/neon_surf_wrapper.py file. + +This script is a simple wrapper for neon sites that performs the +following: + 1) For neon sites, subset surface dataset from global dataset + (i.e. ./subset_data.py ) + 2) Download neon and update the created surface dataset + based on the downloaded neon data. + (i.e. modify_singlept_site_neon.py) + +---------------------------------------------------------------- +Instructions for running using conda python environments: +../../py_env_create +conda activate ctsm_pylib +""" + +import os +import sys + +# -- add python/ctsm to path +_CTSM_PYTHON = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" +) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.site_and_regional.neon_surf_wrapper import main + +if __name__ == "__main__": + main() From 4ee49e3e516ca7dee5df378f65664f93a7db4415 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 13 Sep 2023 11:21:04 -0600 Subject: [PATCH 009/173] reformatting with black --- .../modify_singlept_site_neon.py | 63 +++--- .../site_and_regional/neon_surf_wrapper.py | 200 ++++++++++++------ python/ctsm/site_and_regional/run_neon.py | 119 +++++------ 3 files changed, 216 insertions(+), 166 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index c5ae7eb31b..a33323ed7a 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -160,7 +160,7 @@ def get_parser(): dest="inputdatadir", type=str, required=False, - default="/glade/p/cesmdata/cseg/inputdata" + default="/glade/p/cesmdata/cseg/inputdata", ) parser.add_argument( "-d", @@ -233,10 +233,7 @@ def get_neon(neon_dir, site_name): print("Download finished successfully for", site_name) elif response.status_code == 404: sys.exit( - "Data for this site " - + site_name - + " was not available on the neon server:" - + url + "Data for this site " + site_name + " was not available on the neon server:" + url ) print("Download exit status code: ", response.status_code) @@ -270,12 +267,12 @@ def find_surffile(surf_dir, site_name, pft_16): """ if pft_16: - sf_name = "surfdata_1x1_NEON_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" + sf_name = "surfdata_1x1_NEON_" + site_name + "*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" else: - sf_name = "surfdata_1x1_NEON_" +site_name+"*hist_78pfts_CMIP6_simyr2000_*.nc" + sf_name = "surfdata_1x1_NEON_" + site_name + "*hist_78pfts_CMIP6_simyr2000_*.nc" - print (os.path.join(surf_dir , sf_name)) - surf_file = sorted(glob.glob(os.path.join(surf_dir , sf_name))) + print(os.path.join(surf_dir, sf_name)) + surf_file = sorted(glob.glob(os.path.join(surf_dir, sf_name))) if len(surf_file) > 1: print("The following files found :", *surf_file, sep="\n- ") @@ -287,10 +284,14 @@ def find_surffile(surf_dir, site_name, pft_16): surf_file = surf_file[0] else: sys.exit( - "Surface data for this site " + str(site_name) + " was not found:" + str(surf_dir) + str(sf_name) + - "." + - "\n" + - "Please run ./subset_data.py for this site." + "Surface data for this site " + + str(site_name) + + " was not found:" + + str(surf_dir) + + str(sf_name) + + "." + + "\n" + + "Please run ./subset_data.py for this site." ) return surf_file @@ -324,10 +325,8 @@ def find_soil_structure(args, surf_file): print("------------") # print (f1.attrs["Soil_texture_raw_data_file_name"]) - clm_input_dir = os.path.join( args.inputdatadir, "lnd/clm2/rawdata/" ) - surf_soildepth_file = os.path.join( - clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"] - ) + clm_input_dir = os.path.join(args.inputdatadir, "lnd/clm2/rawdata/") + surf_soildepth_file = os.path.join(clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"]) if os.path.exists(surf_soildepth_file): print( @@ -345,9 +344,7 @@ def find_soil_structure(args, surf_file): else: sys.exit( - "Cannot find soil structure file : " - + surf_soildepth_file - + "for the surface dataset." + "Cannot find soil structure file : " + surf_soildepth_file + "for the surface dataset." ) return soil_bot, soil_top @@ -492,9 +489,9 @@ def download_file(url, fname): elif response.status_code == 404: print("File " + fname + "was not available on the neon server:" + url) except Exception as err: - print ('The server could not fulfill the request.') - print ('Something went wrong in downloading', fname) - print ('Error code:', err.code) + print("The server could not fulfill the request.") + print("Something went wrong in downloading", fname) + print("Error code:", err.code) def fill_interpolate(f2, var, method): @@ -536,8 +533,10 @@ def main(): # Check if pandas is a recent enough version pdvers = pd.__version__ if version.parse(pdvers) < version.parse("1.1.0"): - sys.exit("The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", pdvers ) - + sys.exit( + "The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", + pdvers, + ) file_time = check_neon_time() @@ -639,8 +638,7 @@ def main(): estimated_oc = carbon_tot layer_depth = ( - df["biogeoBottomDepth"][bin_index[soil_lev]] - - df["biogeoTopDepth"][bin_index[soil_lev]] + df["biogeoBottomDepth"][bin_index[soil_lev]] - df["biogeoTopDepth"][bin_index[soil_lev]] ) # f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58 @@ -709,9 +707,9 @@ def main(): print("Updated : ", f2.PCT_CROP.values) print("Updating PCT_NAT_PFT") - #print (f2.PCT_NAT_PFT) + # print (f2.PCT_NAT_PFT) print(f2.PCT_NAT_PFT.values[0]) - #f2.PCT_NAT_PFT.values[0] = [[100.0]] + # f2.PCT_NAT_PFT.values[0] = [[100.0]] print(f2.PCT_NAT_PFT[0].values) out_dir = args.out_dir @@ -729,9 +727,4 @@ def main(): print(f2.attrs) f2.to_netcdf(path=wfile, mode="w", format="NETCDF3_64BIT") - print( - "Successfully updated surface data file for neon site(" - + site_name - + "):\n - " - + wfile - ) + print("Successfully updated surface data file for neon site(" + site_name + "):\n - " + wfile) diff --git a/python/ctsm/site_and_regional/neon_surf_wrapper.py b/python/ctsm/site_and_regional/neon_surf_wrapper.py index ce9efc4ed5..c2e5ac61b8 100755 --- a/python/ctsm/site_and_regional/neon_surf_wrapper.py +++ b/python/ctsm/site_and_regional/neon_surf_wrapper.py @@ -19,8 +19,8 @@ """ # TODO # Automatic downloading of missing files if they are missing -#-[ ] Download neon sites and dom pft file -#-[ ] Make sure verbose works for printing out commands running +# -[ ] Download neon sites and dom pft file +# -[ ] Make sure verbose works for printing out commands running # Import libraries from __future__ import print_function @@ -35,35 +35,41 @@ import pandas as pd - - -def get_parser(): +def get_parser(): """ Get parser object for this script. """ - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.print_usage = parser.print_help - parser.add_argument('-v','--verbose', - help='Verbose mode will print more information. ', - action="store_true", - dest="verbose", - default=False) - - parser.add_argument('--16pft', - help='Create and/or modify 16-PFT surface datasets (e.g. for a FATES run) ', - action="store_true", - dest="pft_16", - default=False) - - parser.add_argument('-m', '--mixed', - help='Do not overwrite surface dataset to be just one dominant PFT at 100%', - action="store_true", - dest="mixed", - default=False) - + parser.add_argument( + "-v", + "--verbose", + help="Verbose mode will print more information. ", + action="store_true", + dest="verbose", + default=False, + ) + + parser.add_argument( + "--16pft", + help="Create and/or modify 16-PFT surface datasets (e.g. for a FATES run) ", + action="store_true", + dest="pft_16", + default=False, + ) + + parser.add_argument( + "-m", + "--mixed", + help="Do not overwrite surface dataset to be just one dominant PFT at 100%", + action="store_true", + dest="mixed", + default=False, + ) return parser @@ -77,19 +83,15 @@ def execute(command): Raises: Error with the return code from shell. """ - print ('\n',' >> ',*command,'\n') + print("\n", " >> ", *command, "\n") try: subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) - #print (e.ouput) - print (e) - - - - + # raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + # print (e.ouput) + print(e) def main(): @@ -99,55 +101,129 @@ def main(): if args.verbose: logging.basicConfig(level=logging.DEBUG) - - neon_sites = pd.read_csv('neon_sites_dompft.csv') - + neon_sites = pd.read_csv("neon_sites_dompft.csv") for i, row in tqdm.tqdm(neon_sites.iterrows()): - lat = row['Lat'] - lon = row['Lon'] - site = row['Site'] - pft = row['pft'] - clmsite = "1x1_NEON_"+site - print ("Now processing site :", site) + lat = row["Lat"] + lon = row["Lon"] + site = row["Site"] + pft = row["pft"] + clmsite = "1x1_NEON_" + site + print("Now processing site :", site) if args.mixed and args.pft_16: # use surface dataset with 16 pfts, and don't overwrite with 100% 1 dominant PFT # don't set crop flag # don't set a dominant pft - subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), - '--site',clmsite, '--create-surface','--uniform-snowpack', - '--cap-saturation','--verbose','--overwrite'] - modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', - 'subset_data_single_point', '--16pft'] + subset_command = [ + "./subset_data", + "point", + "--lat", + str(lat), + "--lon", + str(lon), + "--site", + clmsite, + "--create-surface", + "--uniform-snowpack", + "--cap-saturation", + "--verbose", + "--overwrite", + ] + modify_command = [ + "./modify_singlept_site_neon", + "--neon_site", + site, + "--surf_dir", + "subset_data_single_point", + "--16pft", + ] elif args.pft_16: # use surface dataset with 16 pfts, but overwrite to 100% 1 dominant PFT # don't set crop flag # set dominant pft - subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), - '--site',clmsite,'--dompft',str(pft),'--create-surface', - '--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', - 'subset_data_single_point', '--16pft'] + subset_command = [ + "./subset_data", + "point", + "--lat", + str(lat), + "--lon", + str(lon), + "--site", + clmsite, + "--dompft", + str(pft), + "--create-surface", + "--uniform-snowpack", + "--cap-saturation", + "--verbose", + "--overwrite", + ] + modify_command = [ + "./modify_singlept_site_neon", + "--neon_site", + site, + "--surf_dir", + "subset_data_single_point", + "--16pft", + ] elif args.mixed: # use surface dataset with 78 pfts, and don't overwrite with 100% 1 dominant PFT # NOTE: FATES will currently not run with a 78-PFT surface dataset # set crop flag # don't set dominant pft - subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), - '--site',clmsite,'--crop','--create-surface', - '--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', - 'subset_data_single_point'] + subset_command = [ + "./subset_data", + "point", + "--lat", + str(lat), + "--lon", + str(lon), + "--site", + clmsite, + "--crop", + "--create-surface", + "--uniform-snowpack", + "--cap-saturation", + "--verbose", + "--overwrite", + ] + modify_command = [ + "./modify_singlept_site_neon", + "--neon_site", + site, + "--surf_dir", + "subset_data_single_point", + ] else: # use surface dataset with 78 pfts, and overwrite to 100% 1 dominant PFT # NOTE: FATES will currently not run with a 78-PFT surface dataset # set crop flag # set dominant pft - subset_command = ['./subset_data', 'point', '--lat', str(lat), '--lon', str(lon), - '--site', clmsite,'--crop', '--dompft', str(pft), '--create-surface', - '--uniform-snowpack', '--cap-saturation', '--verbose', '--overwrite'] - modify_command = ['./modify_singlept_site_neon', '--neon_site', site, '--surf_dir', - 'subset_data_single_point'] + subset_command = [ + "./subset_data", + "point", + "--lat", + str(lat), + "--lon", + str(lon), + "--site", + clmsite, + "--crop", + "--dompft", + str(pft), + "--create-surface", + "--uniform-snowpack", + "--cap-saturation", + "--verbose", + "--overwrite", + ] + modify_command = [ + "./modify_singlept_site_neon", + "--neon_site", + site, + "--surf_dir", + "subset_data_single_point", + ] execute(subset_command) execute(modify_command) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index aa6e0c3ffb..8188f6084b 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -67,9 +67,7 @@ from getpass import getuser # Get the ctsm util tools and then the cime tools. -_CTSM_PYTHON = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "python") -) +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) from ctsm import add_cime_to_path @@ -194,7 +192,7 @@ def get_parser(args, description, valid_neon_sites): ) parser.add_argument( - "--prism", + "--prism", help=""" Uses the PRISM reanaylsis precipitation data for the site instead of the NEON data (only available over Continental US) @@ -205,9 +203,8 @@ def get_parser(args, description, valid_neon_sites): default=False, ) - parser.add_argument( - "--experiment", + "--experiment", help=""" Appends the case name with string for model experiment """, @@ -274,12 +271,11 @@ def get_parser(args, description, valid_neon_sites): """, action="store", dest="user_version", - required = False, - type = str, - choices= ['v1','v2','v3'], + required=False, + type=str, + choices=["v1", "v2", "v3"], ) - args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if "all" in args.neon_sites: @@ -381,9 +377,7 @@ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): def __str__(self): return ( - str(self.__class__) - + "\n" - + "\n".join((str(item) + " = " for item in (self.__dict__))) + str(self.__class__) + "\n" + "\n".join((str(item) + " = " for item in (self.__dict__))) ) def build_base_case( @@ -408,9 +402,7 @@ def build_base_case( """ print("---- building a base case -------") self.base_case_root = output_root - user_mods_dirs = [ - os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) - ] + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] if not output_root: output_root = os.getcwd() case_path = os.path.join(output_root, self.name) @@ -449,9 +441,15 @@ def build_base_case( existingcompname = case.get_value("COMPSET") match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( match == None, "Existing base case is a historical type and should not be -- rerun with the --orverwrite option" ) + expect( + match == None, + "Existing base case is a historical type and should not be -- rerun with the --orverwrite option", + ) else: - expect( match != None, "Existing base case should be a historical type and is not -- rerun with the --orverwrite option" ) + expect( + match != None, + "Existing base case should be a historical type and is not -- rerun with the --orverwrite option", + ) # reset the case case.case_setup(reset=True) case_path = case.get_value("CASEROOT") @@ -474,22 +472,22 @@ def diff_month(self): d1 = datetime.datetime(self.end_year, self.end_month, 1) d2 = datetime.datetime(self.start_year, self.start_month, 1) return (d1.year - d2.year) * 12 + d1.month - d2.month - + def get_batch_query(self, case): """ - Function for querying the batch queue query command for a case, depending on the - user's batch system. + Function for querying the batch queue query command for a case, depending on the + user's batch system. Args: case: case object """ - + if case.get_value("BATCH_SYSTEM") == "none": - return "none" + return "none" else: - return case.get_value("batch_query") - + return case.get_value("batch_query") + def run_case( self, base_case_root, @@ -504,9 +502,7 @@ def run_case( experiment=False, ): user_mods_dirs = [ - os.path.join( - self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name - ) + os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) ] expect( os.path.isdir(base_case_root), @@ -516,15 +512,13 @@ def run_case( if user_version: version = user_version else: - version = 'latest' + version = "latest" - print ("using this version:", version) + print("using this version:", version) if experiment != None: self.name = self.name + "." + experiment - case_root = os.path.abspath( - os.path.join(base_case_root, "..", self.name + "." + run_type) - ) + case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) rundir = None if os.path.isdir(case_root): @@ -538,15 +532,17 @@ def run_case( existingcompname = case.get_value("COMPSET") match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( match == None, "Existing base case is a historical type and should not be -- rerun with the --orverwrite option" ) + expect( + match == None, + "Existing base case is a historical type and should not be -- rerun with the --orverwrite option", + ) else: - expect( match != None, "Existing base case should be a historical type and is not -- rerun with the --orverwrite option" ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print( - "Case {} appears to be complete, not rerunning.".format( - case_root - ) + expect( + match != None, + "Existing base case should be a historical type and is not -- rerun with the --orverwrite option", ) + if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): + print("Case {} appears to be complete, not rerunning.".format(case_root)) elif not setup_only: print("Resubmitting case {}".format(case_root)) case.submit(no_batch=no_batch) @@ -557,17 +553,13 @@ def run_case( print(f"Use {batch_query} to check its run status") return else: - logger.warning( - "Case already exists in {}, not overwritting.".format(case_root) - ) + logger.warning("Case already exists in {}, not overwritting.".format(case_root)) return if run_type == "postad": adcase_root = case_root.replace(".postad", ".ad") if not os.path.isdir(adcase_root): - logger.warning( - "postad requested but no ad case found in {}".format(adcase_root) - ) + logger.warning("postad requested but no ad case found in {}".format(adcase_root)) return if not os.path.isdir(case_root): @@ -580,15 +572,13 @@ def run_case( # that the shell_commands file is copied, as well as taking care of the DATM inputs. # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 # - basecase.create_clone( - case_root, keepexe=True, user_mods_dirs=user_mods_dirs - ) + basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) with Case(case_root, read_only=False) as case: if run_type != "transient": - # in order to avoid the complication of leap years we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") + # in order to avoid the complication of leap years we always set the run_length in units of days. + case.set_value("STOP_OPTION", "ndays") + case.set_value("REST_OPTION", "end") case.set_value("CONTINUE_RUN", False) case.set_value("NEONVERSION", version) if prism: @@ -660,9 +650,7 @@ def set_ref_case(self, case): case.set_value("RUN_REFDIR", refrundir) case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) refdate = None - for reffile in glob.iglob( - refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root) - ): + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) if m: refdate = m.group(1) @@ -677,9 +665,7 @@ def set_ref_case(self, case): if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( os.path.join(refrundir, "inputdata") ): - symlink_force( - os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata") - ) + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) case.set_value("RUN_REFDATE", refdate) if case_root.endswith(".postad"): @@ -693,9 +679,7 @@ def modify_user_nl(self, case_root, run_type, rundir): if run_type == "transient": if self.finidat: user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format( - rundir, self.finidat - ) + "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) ] else: user_nl_lines = [ @@ -767,15 +751,14 @@ def parse_neon_listing(listing_file, valid_neon_sites): # -- find all the data versions versions = tmp_df[7].unique() - #print ("all versions available for ", site_name,":", *versions) + # print ("all versions available for ", site_name,":", *versions) latest_version = tmp_df[7].iloc[-1] - #print ("latests version available for ", site_name,":", latest_version) + # print ("latests version available for ", site_name,":", latest_version) tmp_df = tmp_df[tmp_df[7].str.contains(latest_version)] # -- remove .nc from the file names tmp_df[9] = tmp_df[9].str.replace(".nc", "", regex=False) - tmp_df2 = tmp_df[9].str.split("-", expand=True) # ignore any prefix in file name and just get year @@ -800,9 +783,7 @@ def parse_neon_listing(listing_file, valid_neon_sites): if site_name in line: finidat = line.split(",")[0].split("/")[-1] - neon_site = NeonSite( - site_name, start_year, end_year, start_month, end_month, finidat - ) + neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month, finidat) logger.debug(neon_site) available_list.append(neon_site) @@ -847,9 +828,9 @@ def main(description): res = "CLM_USRDAT" if run_type == "transient": - compset = "IHist1PtClm51Bgc" + compset = "IHist1PtClm51Bgc" else: - compset = "I1PtClm51Bgc" + compset = "I1PtClm51Bgc" # -- Looping over neon sites From 9dd60f3a3065269db662e5f70a143cec6fa70e17 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 13 Sep 2023 11:35:20 -0600 Subject: [PATCH 010/173] Updating .git-blame-ignore-revs --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 9b7cb3c036..1b7a0d75b3 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -10,6 +10,7 @@ b88e1cd1b28e3609684c79a2ec0e88f26cfc362b b771971e3299c4fa56534b93421f7a2b9c7282fd 9de88bb57ea9855da408cbec1dc8acb9079eda47 8bc4688e52ea23ef688e283698f70a44388373eb +4ee49e3e516ca7dee5df378f65664f93a7db4415 # Ran SystemTests and python/ctsm through black python formatter 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c 8056ae649c1b37f5e10aaaac79005d6e3a8b2380 From b3327d09d59b8dd92c6c1bcca328b59334c488d5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 2 Oct 2023 11:50:38 -0600 Subject: [PATCH 011/173] Fixed "anomoly" typo. --- tools/contrib/ssp_anomaly_forcing_smooth | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index ae3d189a8b..f70ccd7a73 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -3,7 +3,7 @@ ssp_anomaly_forcing_smooth -Create anomoly forcing datasets for SSP scenarios that can be used by CESM datm model +Create anomaly forcing datasets for SSP scenarios that can be used by CESM datm model load proper modules first, i.e. @@ -48,7 +48,7 @@ if args.sspnum == 0: # ------------------------------------------------------- -print("Create anomoly forcing data that can be used by CTSM in CESM") +print("Create anomaly forcing data that can be used by CTSM in CESM") # Input and output directories make sure they exist datapath = "/glade/campaign/collections/cmip/CMIP6/timeseries-cmip6/" # Path on casper @@ -710,4 +710,4 @@ for f in range(nfields): # -- End Loop over forcing fields ------------------------------------ outfile.close() -print("\n\nSuccessfully made anomoly forcing datasets\n") +print("\n\nSuccessfully made anomaly forcing datasets\n") From b578626a17bb97642da619f51240d0d64e509350 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 2 Oct 2023 11:52:51 -0600 Subject: [PATCH 012/173] Removed unnecessary hist_case. --- tools/contrib/ssp_anomaly_forcing_smooth | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index f70ccd7a73..27eaed5588 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -165,9 +165,6 @@ if args.print_ssps: sspnum = args.sspnum -# hist_case needed? -hist_case = "b.e21.BHIST.f09_g17.CMIP6-historical.010" - if sspnum == 1: # SSP1-26 ssptag = "SSP1-2.6" From b5ce1209ffb2cfe98fc483788657dbe249dc1c27 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 2 Oct 2023 11:53:10 -0600 Subject: [PATCH 013/173] num_ens now automatically set. --- tools/contrib/ssp_anomaly_forcing_smooth | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 27eaed5588..c453237402 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -193,9 +193,6 @@ if num_ens != len(histdir): print("number of ensemble members not the same") sys.exit("number of members different") -# test w/ 1 ensemble member -num_ens = 3 - # Setup output directory sspoutdir = "anomaly_forcing/CMIP6-" + ssptag From 8d06313f53bf524e1f410c163f4ebfef53c3a1db Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 2 Oct 2023 11:53:41 -0600 Subject: [PATCH 014/173] Removed unused hist_suffix and ssp_suffix. --- tools/contrib/ssp_anomaly_forcing_smooth | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index c453237402..6ac76d1682 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -202,13 +202,6 @@ if not os.path.exists(outdir): print("Output specific data directory :" + outdir) -# historical files are split by 50 year periods; use last period -hist_suffix = ["200001-201412.nc"] # not standardized?! -# hist_suffix = ['-201412.nc'] -# projections are split 2015/2064 2065/2100 -ssp_suffix = ["201501-206412.nc", "206501-210012.nc"] -# ssp_suffix = ['-206412.nc','-210012.nc'] - climo_year = 2015 # ten years on either side (21 years total) climo_base_nyrs = 21 From 2f55d1eb1a538d069c91435232b4aabb47ad3eac Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 2 Oct 2023 11:55:20 -0600 Subject: [PATCH 015/173] Added output_format variable. --- tools/contrib/ssp_anomaly_forcing_smooth | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 6ac76d1682..33f0fc3e72 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -231,6 +231,8 @@ field_out_wind = ["uas", "vas"] nfields = len(field_in) +output_format = "NETCDF3_64BIT_DATA" + # -- Loop over forcing fields ------------------------------------ for f in range(nfields): @@ -471,7 +473,7 @@ for f in range(nfields): w = netcdf4.Dataset( outdir + field_out[f] + "_climo" + creationdate + ".nc", "w", - format="NETCDF3_64BIT_DATA" + format=output_format, ) w.createDimension("lat", int(nlat)) w.createDimension("lon", int(nlon)) @@ -502,7 +504,7 @@ for f in range(nfields): w = netcdf4.Dataset( outdir + field_out[f] + "_smooth" + creationdate + ".nc", "w", - format="NETCDF3_64BIT_DATA" + format=output_format, ) w.createDimension("lat", int(nlat)) w.createDimension("lon", int(nlon)) @@ -545,7 +547,7 @@ for f in range(nfields): # output in CDF5 format using netCDF4 python interfaces outfilename = outdir + "/" + "af.allvars" + outfile_suffix print("Creating: " + outfilename) - outfile = netcdf4.Dataset(outfilename, "w", format="NETCDF3_64BIT_DATA") + outfile = netcdf4.Dataset(outfilename, "w", format=output_format) # creation date on the file command = 'date "+%Y/%m/%d"' From 0ca6d62af04c0dc9dc56492eb724ff54d8dbe8ce Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 2 Oct 2023 12:07:09 -0600 Subject: [PATCH 016/173] Can now specify --output-dir. Also fixes bug of write_climo file paths missing a /. --- tools/contrib/ssp_anomaly_forcing_smooth | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 33f0fc3e72..a0b6419744 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -41,6 +41,13 @@ parser.add_argument( action="store_true", default=False, ) +parser.add_argument( + "--output_dir", "--output-dir", + help="Top-level output directory (default: ./anomaly_forcing/). Sub-directory will be created for the selected scenario.", + type=str, + default=os.path.join(".", "anomaly_forcing"), +) + args = parser.parse_args() if args.sspnum == 0: @@ -109,15 +116,14 @@ _v2 is just used for restart files that have been spatially interpolated """ -spath = "./" if os.path.exists(datapath): print("Input data directory:" + datapath) else: sys.exit("Could not find input directory: " + datapath) -if os.path.exists(spath): - print("Output data directory:" + spath) +if os.path.exists(args.output_dir): + print("Output data directory:" + args.output_dir) else: - sys.exit("Could not find output directory: " + spath) + sys.exit("Could not find output directory: " + args.output_dir) # Settings to run with today = datetime.date.today() @@ -194,9 +200,9 @@ if num_ens != len(histdir): sys.exit("number of members different") # Setup output directory -sspoutdir = "anomaly_forcing/CMIP6-" + ssptag +sspoutdir = "CMIP6-" + ssptag -outdir = spath + sspoutdir +outdir = os.path.join(args.output_dir, sspoutdir) if not os.path.exists(outdir): os.makedirs(outdir) @@ -471,7 +477,7 @@ for f in range(nfields): if write_climo: # Use NetCDF4 format, because using older NetCDF formats are too slow w = netcdf4.Dataset( - outdir + field_out[f] + "_climo" + creationdate + ".nc", + os.path.join(outdir, field_out[f] + "_climo" + creationdate + ".nc"), "w", format=output_format, ) @@ -502,7 +508,7 @@ for f in range(nfields): # Use NetCDF4 format, because using older NetCDF formats are too slow w = netcdf4.Dataset( - outdir + field_out[f] + "_smooth" + creationdate + ".nc", + os.path.join(outdir, field_out[f] + "_smooth" + creationdate + ".nc"), "w", format=output_format, ) @@ -545,7 +551,7 @@ for f in range(nfields): # Use NetCDF4 format, because using older NetCDF formats are too slow # Will need to convert to CDF5 format at the end, as we can't seem to # output in CDF5 format using netCDF4 python interfaces - outfilename = outdir + "/" + "af.allvars" + outfile_suffix + outfilename = os.path.join(outdir, "af.allvars" + outfile_suffix) print("Creating: " + outfilename) outfile = netcdf4.Dataset(outfilename, "w", format=output_format) From c58e0ee80d8653961dd9af29c5bd8d3c6c92029b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Oct 2023 10:04:43 -0600 Subject: [PATCH 017/173] Move iso functions to utils --- python/ctsm/site_and_regional/run_neon.py | 28 +------------------- python/ctsm/utils.py | 32 ++++++++++++++++++++++- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 8188f6084b..b0ccc4b687 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -71,7 +71,7 @@ sys.path.insert(1, _CTSM_PYTHON) from ctsm import add_cime_to_path -from ctsm.path_utils import path_to_ctsm_root +from ctsm.path_utils import path_to_ctsm_root, parse_isoduration from ctsm.download_utils import download_file import CIME.build as build @@ -327,32 +327,6 @@ def get_parser(args, description, valid_neon_sites): args.user_version, ) - -def get_isosplit(s, split): - if split in s: - n, s = s.split(split) - else: - n = 0 - return n, s - - -def parse_isoduration(s): - """ - simple ISO 8601 duration parser, does not account for leap years and assumes 30 day months - """ - # Remove prefix - s = s.split("P")[-1] - - # Step through letter dividers - years, s = get_isosplit(s, "Y") - months, s = get_isosplit(s, "M") - days, s = get_isosplit(s, "D") - - # Convert all to timedelta - dt = datetime.timedelta(days=int(days) + 365 * int(years) + 30 * int(months)) - return int(dt.total_seconds() / 86400) - - class NeonSite: """ A class for encapsulating neon sites. diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 42444e32c5..add7655418 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -7,7 +7,7 @@ import re import pdb -from datetime import date +from datetime import date, timedelta from getpass import getuser from ctsm.git_utils import get_ctsm_git_short_hash @@ -189,3 +189,33 @@ def write_output(file, file_in, file_out, file_type): file.to_netcdf(path=file_out, mode="w", format="NETCDF3_64BIT") logger.info("Successfully created: %s", file_out) file.close() + + +def get_isosplit(s, split): + """ + Split a string (s) by the character sent in from split + Only used by parse_isoduration + """ + if split in s: + n, s = s.split(split) + else: + n = 0 + return n, s + + +def parse_isoduration(s): + """ + simple ISO 8601 duration parser, does not account for leap years and assumes 30 day months + """ + # Remove prefix + s = s.split("P")[-1] + + # Step through letter dividers + years, s = get_isosplit(s, "Y") + months, s = get_isosplit(s, "M") + days, s = get_isosplit(s, "D") + + # Convert all to timedelta + dt = timedelta(days=int(days) + 365 * int(years) + 30 * int(months)) + return int(dt.total_seconds() / 86400) + From 47e72881e80c94473284b23de2d6ad714faf7f9d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Oct 2023 10:23:16 -0600 Subject: [PATCH 018/173] Some lint and black updates for the new iso utility code --- python/ctsm/utils.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index add7655418..8578ea860c 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -191,31 +191,31 @@ def write_output(file, file_in, file_out, file_type): file.close() -def get_isosplit(s, split): +def get_isosplit(iso_string, split): """ - Split a string (s) by the character sent in from split + Split a string (iso_string) by the character sent in from split + Returns the number for that character split Only used by parse_isoduration """ - if split in s: - n, s = s.split(split) + if split in iso_string: + num, iso_string = iso_string.split(split) else: - n = 0 - return n, s + num = 0 + return num, iso_string -def parse_isoduration(s): +def parse_isoduration(iso_string): """ simple ISO 8601 duration parser, does not account for leap years and assumes 30 day months """ # Remove prefix - s = s.split("P")[-1] + iso_string = iso_string.split("P")[-1] # Step through letter dividers - years, s = get_isosplit(s, "Y") - months, s = get_isosplit(s, "M") - days, s = get_isosplit(s, "D") + years, iso_string = get_isosplit(iso_string, "Y") + months, iso_string = get_isosplit(iso_string, "M") + days, iso_string = get_isosplit(iso_string, "D") # Convert all to timedelta - dt = timedelta(days=int(days) + 365 * int(years) + 30 * int(months)) - return int(dt.total_seconds() / 86400) - + delta_t = timedelta(days=int(days) + 365 * int(years) + 30 * int(months)) + return int(delta_t.total_seconds() / 86400) From e56552d9b2334f80f93189efd91c72d96c9fdf01 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Oct 2023 11:03:11 -0600 Subject: [PATCH 019/173] Add unit tester for get_isosplit function --- python/ctsm/site_and_regional/run_neon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index b0ccc4b687..41719adf33 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -71,7 +71,8 @@ sys.path.insert(1, _CTSM_PYTHON) from ctsm import add_cime_to_path -from ctsm.path_utils import path_to_ctsm_root, parse_isoduration +from ctsm.path_utils import path_to_ctsm_root +from ctsm.utils import parse_isoduration from ctsm.download_utils import download_file import CIME.build as build From 017b021c4bff1b3592e7b067083b210868a437b1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Oct 2023 11:03:58 -0600 Subject: [PATCH 020/173] Add unit tester for get_isosplit function --- python/ctsm/test/test_unit_iso_utils.py | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 python/ctsm/test/test_unit_iso_utils.py diff --git a/python/ctsm/test/test_unit_iso_utils.py b/python/ctsm/test/test_unit_iso_utils.py new file mode 100755 index 0000000000..c087b23f64 --- /dev/null +++ b/python/ctsm/test/test_unit_iso_utils.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +"""Unit tests for the iso functions in utils +""" + +import unittest +import os + +from ctsm import unit_testing +from ctsm.utils import parse_isoduration, get_isosplit + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +class TestIsoUtils(unittest.TestCase): + """Tests of iso functions in utils""" + + def test_iso_split_for_Year(self): + """ + Tests the get_isosplit function for a strings with Years + """ + iso_string = "0Y" + self.assertEqual( get_isosplit(iso_string, "Y"), ("0", "") ) + iso_string = "1Y" + self.assertEqual( get_isosplit(iso_string, "Y"), ("1", "") ) + iso_string = "4Y" + self.assertEqual( get_isosplit(iso_string, "Y"), ("4", "") ) + iso_string = "100Y" + self.assertEqual( get_isosplit(iso_string, "Y"), ("100", "") ) + iso_string = "999999Y" + self.assertEqual( get_isosplit(iso_string, "Y"), ("999999", "") ) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 3830a6de2b10cbe3a9e2a864fb16b426639a7abc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Oct 2023 11:21:09 -0600 Subject: [PATCH 021/173] Add a test of parse_isoduration and run it in black --- python/ctsm/test/test_unit_iso_utils.py | 27 +++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/python/ctsm/test/test_unit_iso_utils.py b/python/ctsm/test/test_unit_iso_utils.py index c087b23f64..c58beef52e 100755 --- a/python/ctsm/test/test_unit_iso_utils.py +++ b/python/ctsm/test/test_unit_iso_utils.py @@ -4,7 +4,6 @@ """ import unittest -import os from ctsm import unit_testing from ctsm.utils import parse_isoduration, get_isosplit @@ -22,15 +21,31 @@ def test_iso_split_for_Year(self): Tests the get_isosplit function for a strings with Years """ iso_string = "0Y" - self.assertEqual( get_isosplit(iso_string, "Y"), ("0", "") ) + self.assertEqual(get_isosplit(iso_string, "Y"), ("0", "")) iso_string = "1Y" - self.assertEqual( get_isosplit(iso_string, "Y"), ("1", "") ) + self.assertEqual(get_isosplit(iso_string, "Y"), ("1", "")) iso_string = "4Y" - self.assertEqual( get_isosplit(iso_string, "Y"), ("4", "") ) + self.assertEqual(get_isosplit(iso_string, "Y"), ("4", "")) iso_string = "100Y" - self.assertEqual( get_isosplit(iso_string, "Y"), ("100", "") ) + self.assertEqual(get_isosplit(iso_string, "Y"), ("100", "")) iso_string = "999999Y" - self.assertEqual( get_isosplit(iso_string, "Y"), ("999999", "") ) + self.assertEqual(get_isosplit(iso_string, "Y"), ("999999", "")) + + def test_parse_isoduration_for_Years(self): + """ + Tests the parse_isoduration function for iso strings with Years + """ + days_in_year = 365 + iso_string = "0Y" + self.assertEqual(parse_isoduration(iso_string), 0) + iso_string = "1Y" + self.assertEqual(parse_isoduration(iso_string), days_in_year) + iso_string = "4Y" + self.assertEqual(parse_isoduration(iso_string), 4 * days_in_year) + iso_string = "100Y" + self.assertEqual(parse_isoduration(iso_string), 100 * days_in_year) + iso_string = "999999Y" + self.assertEqual(parse_isoduration(iso_string), 999999 * days_in_year) if __name__ == "__main__": From 4b6874eec19b41c540bfa3587a8ef95c56c04962 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 4 Oct 2023 16:06:10 -0600 Subject: [PATCH 022/173] add unit tests --- .../test_unit_modify_singlept_site_neon.py | 109 ++++++++++++++++++ .../ctsm/test/test_unit_neon_surf_wrapper.py | 42 +++++++ python/ctsm/test/test_unit_run_neon.py | 60 ++++++++++ 3 files changed, 211 insertions(+) create mode 100755 python/ctsm/test/test_unit_modify_singlept_site_neon.py create mode 100755 python/ctsm/test/test_unit_neon_surf_wrapper.py create mode 100755 python/ctsm/test/test_unit_run_neon.py diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py new file mode 100755 index 0000000000..f67bf2ecc8 --- /dev/null +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Unit tests for modify_singlept_site_neon + +You can run this by: + python -m unittest test_unit_modify_singlept_site_neon.py +""" + +import unittest +import tempfile +import shutil +import configparser +import argparse +import os +import sys +from getpass import getuser +from datetime import date + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.modify_singlept_site_neon import get_parser, get_neon, find_surffile, find_soil_structure, update_metadata, update_time_tag, sort_print_soil_layers, check_neon_time, download_file, fill_interpolate +from ctsm.path_utils import path_to_ctsm_root + +# pylint: disable=invalid-name + + +class TestModifySingleptSiteNeon(unittest.TestCase): + """ + Basic class for testing modify_singlept_site_neon.py. + """ + + def setUp(self): + """ + Make /_tempdir for use by these tests. + Check tempdir for history files + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_get_neon(self): + """ + Test to see if neon data for valid site name is found + """ + site_name = "ABBY" + neon_dir = self._tempdir + file = get_neon(neon_dir, site_name) + self.assertEqual(file.split('/')[-1][:4], 'ABBY', "CSV file did not download as expected") + + def test_get_neon_false_site(self): + """ + Test to see if neon data for invalid site name is found + """ + site_name = "ABY" + neon_dir = self._tempdir + with self.assertRaises(SystemExit): + get_neon(neon_dir, site_name) + + def test_find_surffile(self): + """ + Test that surface file does not exist in tempdir and raises system exit error + """ + surf_dir = self._tempdir + site_name = "BART" + pft_16 = True + with self.assertRaises(SystemExit): + find_surffile(surf_dir, site_name, pft_16) + # TODO: it would be useful to also include an example where the surface file is actually found or correctly searched for + # Is there a default surface data file directory we can point to? + + #def test_find_soil_structure(self): + # """ + # Test + # """ + # find_soil_structure() + # TODO: need file to use in test + + #def test_update_metadata(self): + # TODO: do we have a surface data file that we can put in tempdir to update? + # update_metadata(nc, surf_file, neon_file, zb_flag) + + def test_update_time_tag(self): + """ + Test that file ending is updated + """ + self.assertEqual(update_time_tag("test_YYMMDD.nc")[-9:-3], date.today().strftime("%y%m%d"), "File ending not as expected") + + def test_check_neon_time(self): + """ + Test that dictionary containing last modified information is correctly downloaded + """ + last_abby_download = check_neon_time()['https://storage.neonscience.org/neon-ncar/NEON/surf_files/v1/ABBY_surfaceData.csv'] + self.assertEqual(len(last_abby_download), 19, "last ABBY download has unexpected date format or does not exist") + # TODO: this checks that data is not pulled from before 2021; we may want to update this occassionally or find another way to check data is actually newest? + self.assertGreater(int(last_abby_download[:4]), 2021, "ABBY download is older than expected") + + # TODO: test sort_print_soil_layers, fill_interpolate, find_soil_structure, and update_metadata + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_neon_surf_wrapper.py b/python/ctsm/test/test_unit_neon_surf_wrapper.py new file mode 100755 index 0000000000..f46cbdb47c --- /dev/null +++ b/python/ctsm/test/test_unit_neon_surf_wrapper.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +""" +Unit tests for neon_surf_wrapper + +You can run this by: + python -m unittest test_unit_neon_surf_wrapper.py +""" + +import unittest +import configparser +import argparse +import os +import sys + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.neon_surf_wrapper import get_parser +from ctsm.path_utils import path_to_ctsm_root + +# pylint: disable=invalid-name + + +class TestNeonSurfWrapper(unittest.TestCase): + """ + Basic class for testing neon_surf_wrapper.py. + """ + + def test_parser(self): + """ + Test that parser has same defaults as expected + """ + + self.assertEqual(get_parser().argument_default, None, "Parser not working as expected") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py new file mode 100755 index 0000000000..d3ceb52a3f --- /dev/null +++ b/python/ctsm/test/test_unit_run_neon.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +""" +Unit tests for run_neon + +You can run this by: + python -m unittest test_unit_run_neon.py +""" + +import unittest +import configparser +import argparse +import os +import sys +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.run_neon import check_neon_listing +from ctsm.path_utils import path_to_ctsm_root + +# pylint: disable=invalid-name + + +class TestRunNeon(unittest.TestCase): + """ + Basic class for testing run_neon.py. + """ + + #def setUp(self): + # sys.argv = ["subset_data", "point", "--create-surface"] + # DEFAULTS_FILE = os.path.join(os.getcwd(), "ctsm/test/testinputs/default_data.cfg") + # self.parser = get_parser() + # self.args = self.parser.parse_args() + # self.cesmroot = path_to_ctsm_root() + # self.defaults = configparser.ConfigParser() + # self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) + + def test_check_neon_listing(self): + """ + Test that neon listing is available for valid sites + """ + valid_neon_sites = ['ABBY', 'BART'] + available_list = check_neon_listing(valid_neon_sites) + self.assertEqual(available_list[0].name, 'ABBY', "available list of actual sites not as expected") + self.assertEqual(available_list[1].name, 'BART', "available list of actual sites not as expected") + + + def test_check_neon_listing_misspelled(self): + """ + Test that neon listing is not available for invalid sites + """ + valid_neon_sites = ['ABY', 'BRT'] + available_list = check_neon_listing(valid_neon_sites) + self.assertEqual(available_list, [], "available list of incorrect dummy site not as expected") + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 528d4f05c84bee4e2de72c2c481ff31f3d305ccd Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 5 Oct 2023 08:32:21 -0600 Subject: [PATCH 023/173] remove hard-coded neon site list & use dir structure instead, similarly to in run_neon --- .../modify_singlept_site_neon.py | 54 ++----------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index a33323ed7a..a131671a6e 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -31,7 +31,6 @@ """ # TODO (NS) # --[] If subset file not found run subset_data.py -# --[] List of valid neon sites for all scripts come from one place. # --[] Download files only when available. # Import libraries @@ -52,60 +51,15 @@ from datetime import date from getpass import getuser +from ctsm.path_utils import path_to_ctsm_root myname = getuser() # -- valid neon sites -valid_neon_sites = [ - "ABBY", - "BARR", - "BART", - "BLAN", - "BONA", - "CLBJ", - "CPER", - "DCFS", - "DEJU", - "DELA", - "DSNY", - "GRSM", - "GUAN", - "HARV", - "HEAL", - "JERC", - "JORN", - "KONA", - "KONZ", - "LAJA", - "LENO", - "MLBS", - "MOAB", - "NIWO", - "NOGP", - "OAES", - "ONAQ", - "ORNL", - "OSBS", - "PUUM", - "RMNP", - "SCBI", - "SERC", - "SJER", - "SOAP", - "SRER", - "STEI", - "STER", - "TALL", - "TEAK", - "TOOL", - "TREE", - "UKFS", - "UNDE", - "WOOD", - "WREF", - "YELL", -] +valid_neon_sites = glob.glob( + os.path.join(path_to_ctsm_root(), "cime_config", "usermods_dirs", "NEON", "[!d]*") + ) def get_parser(): From 0bc3f00115d86d026a977918661c93779b3b19f9 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 5 Oct 2023 08:41:15 -0600 Subject: [PATCH 024/173] black reformatting --- .../modify_singlept_site_neon.py | 5 +-- python/ctsm/site_and_regional/run_neon.py | 1 + .../test_unit_modify_singlept_site_neon.py | 44 ++++++++++++++----- .../ctsm/test/test_unit_neon_surf_wrapper.py | 2 +- python/ctsm/test/test_unit_run_neon.py | 23 ++++++---- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index a131671a6e..1ddcd949a8 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -58,8 +58,8 @@ # -- valid neon sites valid_neon_sites = glob.glob( - os.path.join(path_to_ctsm_root(), "cime_config", "usermods_dirs", "NEON", "[!d]*") - ) + os.path.join(path_to_ctsm_root(), "cime_config", "usermods_dirs", "NEON", "[!d]*") +) def get_parser(): @@ -477,7 +477,6 @@ def fill_interpolate(f2, var, method): def main(): - args = get_parser().parse_args() # -- debugging option diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 41719adf33..3f540845e4 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -328,6 +328,7 @@ def get_parser(args, description, valid_neon_sites): args.user_version, ) + class NeonSite: """ A class for encapsulating neon sites. diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index f67bf2ecc8..d1d738ffa8 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -22,7 +22,18 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.site_and_regional.modify_singlept_site_neon import get_parser, get_neon, find_surffile, find_soil_structure, update_metadata, update_time_tag, sort_print_soil_layers, check_neon_time, download_file, fill_interpolate +from ctsm.site_and_regional.modify_singlept_site_neon import ( + get_parser, + get_neon, + find_surffile, + find_soil_structure, + update_metadata, + update_time_tag, + sort_print_soil_layers, + check_neon_time, + download_file, + fill_interpolate, +) from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name @@ -50,10 +61,10 @@ def test_get_neon(self): """ Test to see if neon data for valid site name is found """ - site_name = "ABBY" + site_name = "ABBY" neon_dir = self._tempdir file = get_neon(neon_dir, site_name) - self.assertEqual(file.split('/')[-1][:4], 'ABBY', "CSV file did not download as expected") + self.assertEqual(file.split("/")[-1][:4], "ABBY", "CSV file did not download as expected") def test_get_neon_false_site(self): """ @@ -76,34 +87,47 @@ def test_find_surffile(self): # TODO: it would be useful to also include an example where the surface file is actually found or correctly searched for # Is there a default surface data file directory we can point to? - #def test_find_soil_structure(self): + # def test_find_soil_structure(self): # """ # Test # """ # find_soil_structure() # TODO: need file to use in test - #def test_update_metadata(self): - # TODO: do we have a surface data file that we can put in tempdir to update? + # def test_update_metadata(self): + # TODO: do we have a surface data file that we can put in tempdir to update? # update_metadata(nc, surf_file, neon_file, zb_flag) def test_update_time_tag(self): """ Test that file ending is updated """ - self.assertEqual(update_time_tag("test_YYMMDD.nc")[-9:-3], date.today().strftime("%y%m%d"), "File ending not as expected") + self.assertEqual( + update_time_tag("test_YYMMDD.nc")[-9:-3], + date.today().strftime("%y%m%d"), + "File ending not as expected", + ) def test_check_neon_time(self): """ Test that dictionary containing last modified information is correctly downloaded """ - last_abby_download = check_neon_time()['https://storage.neonscience.org/neon-ncar/NEON/surf_files/v1/ABBY_surfaceData.csv'] - self.assertEqual(len(last_abby_download), 19, "last ABBY download has unexpected date format or does not exist") + last_abby_download = check_neon_time()[ + "https://storage.neonscience.org/neon-ncar/NEON/surf_files/v1/ABBY_surfaceData.csv" + ] + self.assertEqual( + len(last_abby_download), + 19, + "last ABBY download has unexpected date format or does not exist", + ) # TODO: this checks that data is not pulled from before 2021; we may want to update this occassionally or find another way to check data is actually newest? - self.assertGreater(int(last_abby_download[:4]), 2021, "ABBY download is older than expected") + self.assertGreater( + int(last_abby_download[:4]), 2021, "ABBY download is older than expected" + ) # TODO: test sort_print_soil_layers, fill_interpolate, find_soil_structure, and update_metadata + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_neon_surf_wrapper.py b/python/ctsm/test/test_unit_neon_surf_wrapper.py index f46cbdb47c..0092bdb91c 100755 --- a/python/ctsm/test/test_unit_neon_surf_wrapper.py +++ b/python/ctsm/test/test_unit_neon_surf_wrapper.py @@ -33,7 +33,7 @@ def test_parser(self): """ Test that parser has same defaults as expected """ - + self.assertEqual(get_parser().argument_default, None, "Parser not working as expected") diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index d3ceb52a3f..a8031fff1d 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -11,6 +11,7 @@ import argparse import os import sys + # -- add python/ctsm to path (needed if we want to run the test stand-alone) _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) @@ -28,7 +29,7 @@ class TestRunNeon(unittest.TestCase): Basic class for testing run_neon.py. """ - #def setUp(self): + # def setUp(self): # sys.argv = ["subset_data", "point", "--create-surface"] # DEFAULTS_FILE = os.path.join(os.getcwd(), "ctsm/test/testinputs/default_data.cfg") # self.parser = get_parser() @@ -37,23 +38,29 @@ class TestRunNeon(unittest.TestCase): # self.defaults = configparser.ConfigParser() # self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) - def test_check_neon_listing(self): + def test_check_neon_listing(self): """ Test that neon listing is available for valid sites """ - valid_neon_sites = ['ABBY', 'BART'] + valid_neon_sites = ["ABBY", "BART"] available_list = check_neon_listing(valid_neon_sites) - self.assertEqual(available_list[0].name, 'ABBY', "available list of actual sites not as expected") - self.assertEqual(available_list[1].name, 'BART', "available list of actual sites not as expected") - + self.assertEqual( + available_list[0].name, "ABBY", "available list of actual sites not as expected" + ) + self.assertEqual( + available_list[1].name, "BART", "available list of actual sites not as expected" + ) def test_check_neon_listing_misspelled(self): """ Test that neon listing is not available for invalid sites """ - valid_neon_sites = ['ABY', 'BRT'] + valid_neon_sites = ["ABY", "BRT"] available_list = check_neon_listing(valid_neon_sites) - self.assertEqual(available_list, [], "available list of incorrect dummy site not as expected") + self.assertEqual( + available_list, [], "available list of incorrect dummy site not as expected" + ) + if __name__ == "__main__": unit_testing.setup_for_tests() From 30ab529d8fe8fe01f0faf11a23913d0a2e392a4e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 5 Oct 2023 08:45:32 -0600 Subject: [PATCH 025/173] Updating .git-blame-ignore-revs --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 1b7a0d75b3..49d3229d8e 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -14,3 +14,4 @@ b771971e3299c4fa56534b93421f7a2b9c7282fd # Ran SystemTests and python/ctsm through black python formatter 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c 8056ae649c1b37f5e10aaaac79005d6e3a8b2380 +0bc3f00115d86d026a977918661c93779b3b19f9 From 807ca5c51979f6b4d26103d4b18d978fe7e67dda Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 5 Oct 2023 09:30:10 -0600 Subject: [PATCH 026/173] addressed some pylint suggestions --- .../modify_singlept_site_neon.py | 23 +++-- python/ctsm/site_and_regional/run_neon.py | 91 +++++++++++++------ 2 files changed, 75 insertions(+), 39 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index 1ddcd949a8..465fcda25c 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -36,21 +36,20 @@ # Import libraries from __future__ import print_function +import argparse +from datetime import date +from getpass import getuser +import glob +import logging import os import sys -import glob -import argparse import requests -import logging import numpy as np import pandas as pd import xarray as xr from packaging import version -from datetime import date -from getpass import getuser - from ctsm.path_utils import path_to_ctsm_root myname = getuser() @@ -487,11 +486,12 @@ def main(): pdvers = pd.__version__ if version.parse(pdvers) < version.parse("1.1.0"): sys.exit( - "The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", + """The pandas version in your python environment is too old, + update to a newer version of pandas (>=1.1.0): version=%s""", pdvers, ) - file_time = check_neon_time() + # file_time = check_neon_time() # -- specify site from which to extract data site_name = args.site_name @@ -502,7 +502,7 @@ def main(): # -- directory structure current_dir = os.getcwd() - parent_dir = os.path.dirname(current_dir) + # parent_dir = os.path.dirname(current_dir) clone_dir = os.path.abspath(os.path.join(__file__, "../../../..")) neon_dir = os.path.join(clone_dir, "neon_surffiles") @@ -538,7 +538,7 @@ def main(): soil_mid = 0.5 * (soil_bot - soil_top) + soil_top # print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) - obs_top = df["biogeoTopDepth"] / 100 + # obs_top = df["biogeoTopDepth"] / 100 obs_bot = df["biogeoBottomDepth"] / 100 # -- Mapping surface dataset and neon soil levels @@ -587,8 +587,7 @@ def main(): # -- Check to make sure the rounded oc is not higher than carbon_tot. # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. - if estimated_oc > carbon_tot: - estimated_oc = carbon_tot + estimated_oc = min(estimated_oc, carbon_tot) layer_depth = ( df["biogeoBottomDepth"][bin_index[soil_lev]] - df["biogeoTopDepth"][bin_index[soil_lev]] diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 3f540845e4..59ddf3f2eb 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -51,36 +51,30 @@ # Import libraries - +import argparse +import datetime +import glob +import logging import os +import re +import shutil import sys import time -import shutil -import logging -import requests -import argparse -import re -import subprocess import pandas as pd -import glob -import datetime -from getpass import getuser # Get the ctsm util tools and then the cime tools. _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) -from ctsm import add_cime_to_path +from CIME import build +from CIME.case import Case +from CIME.utils import safe_copy, expect, symlink_force + from ctsm.path_utils import path_to_ctsm_root from ctsm.utils import parse_isoduration from ctsm.download_utils import download_file -import CIME.build as build from standard_script_setup import * -from CIME.case import Case -from CIME.utils import safe_copy, expect, symlink_force, run_cmd_no_fail -from argparse import RawTextHelpFormatter -from CIME.locked_files import lock_file, unlock_file logger = logging.getLogger(__name__) @@ -296,7 +290,8 @@ def get_parser(args, description, valid_neon_sites): elif args.run_type == "postad": run_length = "100Y" else: - # The transient run length is set by cdeps atm buildnml to the last date of the available tower data + # The transient run length is set by cdeps atm buildnml to + # the last date of the available tower data # this value is not used run_length = "4Y" else: @@ -307,7 +302,8 @@ def get_parser(args, description, valid_neon_sites): if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) - # Reduce output level for this script unless --debug or --verbose is provided on the command line + # Reduce output level for this script unless --debug or + # --verbose is provided on the command line if not args.debug and not args.verbose: root_logger = logging.getLogger() root_logger.setLevel(logging.WARN) @@ -418,12 +414,12 @@ def build_base_case( match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) if re.search("^HIST", compset, flags=re.IGNORECASE) is None: expect( - match == None, + match is None, "Existing base case is a historical type and should not be -- rerun with the --orverwrite option", ) else: expect( - match != None, + match is not None, "Existing base case should be a historical type and is not -- rerun with the --orverwrite option", ) # reset the case @@ -445,6 +441,9 @@ def build_base_case( return case_path def diff_month(self): + """ + Determine difference between two dates in months + """ d1 = datetime.datetime(self.end_year, self.end_month, 1) d2 = datetime.datetime(self.start_year, self.start_month, 1) return (d1.year - d2.year) * 12 + d1.month - d2.month @@ -461,8 +460,7 @@ def get_batch_query(self, case): if case.get_value("BATCH_SYSTEM") == "none": return "none" - else: - return case.get_value("batch_query") + return case.get_value("batch_query") def run_case( self, @@ -477,6 +475,33 @@ def run_case( rerun=False, experiment=False, ): + + """ + Run case. + + Args: + self + base_case_root: str, opt + file path of base case + run_type: str, opt + transient, post_ad, or ad case, default transient + prism: bool, opt + if True, use PRISM precipitation, default False + run_length: str, opt + length of run, default '4Y' + user_version: str, opt + default 'latest' + overwrite: bool, opt + default False + setup_only: bool, opt + default False; if True, set up but do not run case + no_batch: bool, opt + default False + rerun: bool, opt + default False + experiment: str, opt + name of experiment, default False + """ user_mods_dirs = [ os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) ] @@ -492,7 +517,7 @@ def run_case( print("using this version:", version) - if experiment != None: + if experiment is not None: self.name = self.name + "." + experiment case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) @@ -509,12 +534,12 @@ def run_case( match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) if re.search("^HIST", compset, flags=re.IGNORECASE) is None: expect( - match == None, + match is None, "Existing base case is a historical type and should not be -- rerun with the --orverwrite option", ) else: expect( - match != None, + match is not None, "Existing base case should be a historical type and is not -- rerun with the --orverwrite option", ) if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): @@ -552,7 +577,8 @@ def run_case( with Case(case_root, read_only=False) as case: if run_type != "transient": - # in order to avoid the complication of leap years we always set the run_length in units of days. + # in order to avoid the complication of leap years, + # we always set the run_length in units of days. case.set_value("STOP_OPTION", "ndays") case.set_value("REST_OPTION", "end") case.set_value("CONTINUE_RUN", False) @@ -605,6 +631,9 @@ def run_case( print(f"Use {batch_query} to check its run status") def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ rundir = case.get_value("RUNDIR") case_root = case.get_value("CASEROOT") if case_root.endswith(".postad"): @@ -650,6 +679,10 @@ def set_ref_case(self, case): return True def modify_user_nl(self, case_root, run_type, rundir): + """ + Modify user namelist. If transient, include finidat in user_nl; + Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. + """ user_nl_fname = os.path.join(case_root, "user_nl_clm") user_nl_lines = None if run_type == "transient": @@ -726,7 +759,7 @@ def parse_neon_listing(listing_file, valid_neon_sites): tmp_df = tmp_df[tmp_df[9].str.contains("\d\d\d\d-\d\d.nc")] # -- find all the data versions - versions = tmp_df[7].unique() + # versions = tmp_df[7].unique() # print ("all versions available for ", site_name,":", *versions) latest_version = tmp_df[7].iloc[-1] # print ("latests version available for ", site_name,":", latest_version) @@ -767,6 +800,10 @@ def parse_neon_listing(listing_file, valid_neon_sites): def main(description): + """ + Determine valid neon sites. Make an output directory if it does not exist. + Loop through requested sites and run CTSM at that site. + """ cesmroot = path_to_ctsm_root() # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob( From 0207bc98dd5c75cd69a0e788bc53e41093712f5c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 5 Oct 2023 09:32:04 -0600 Subject: [PATCH 027/173] black reformatting --- python/ctsm/site_and_regional/run_neon.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 59ddf3f2eb..987883fe93 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -475,7 +475,6 @@ def run_case( rerun=False, experiment=False, ): - """ Run case. @@ -801,7 +800,7 @@ def parse_neon_listing(listing_file, valid_neon_sites): def main(description): """ - Determine valid neon sites. Make an output directory if it does not exist. + Determine valid neon sites. Make an output directory if it does not exist. Loop through requested sites and run CTSM at that site. """ cesmroot = path_to_ctsm_root() From a6c3c64d16fe745f17e98661a1638ed8ef0037c0 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 5 Oct 2023 10:22:57 -0600 Subject: [PATCH 028/173] Updating .git-blame-ignore-revs --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 49d3229d8e..c520df4357 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -11,6 +11,7 @@ b771971e3299c4fa56534b93421f7a2b9c7282fd 9de88bb57ea9855da408cbec1dc8acb9079eda47 8bc4688e52ea23ef688e283698f70a44388373eb 4ee49e3e516ca7dee5df378f65664f93a7db4415 +0207bc98dd5c75cd69a0e788bc53e41093712f5c # Ran SystemTests and python/ctsm through black python formatter 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c 8056ae649c1b37f5e10aaaac79005d6e3a8b2380 From ca9977c83bcb17dfa150d5124324e91e7e57e59d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 10 Oct 2023 10:30:20 -0600 Subject: [PATCH 029/173] modify single point site neon unit tests and minor changes to modify_singlept_site_neon.py --- .../modify_singlept_site_neon.py | 4 +- .../test_unit_modify_singlept_site_neon.py | 49 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index 465fcda25c..ca85d68636 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -252,7 +252,7 @@ def find_surffile(surf_dir, site_name, pft_16): def find_soil_structure(args, surf_file): """ Function for finding surface dataset soil - strucutre using surface data metadata. + structure using surface data metadata. In CLM surface data, soil layer information is in a file from surface data metadata @@ -306,7 +306,7 @@ def find_soil_structure(args, surf_file): def update_metadata(nc, surf_file, neon_file, zb_flag): """ Function for updating modified surface dataset - metadat for neon sites. + metadata for neon sites. Args: nc (xr Dataset): netcdf file including updated neon surface data diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index d1d738ffa8..f782075c43 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -6,15 +6,13 @@ python -m unittest test_unit_modify_singlept_site_neon.py """ -import unittest -import tempfile -import shutil -import configparser -import argparse import os +import shutil import sys -from getpass import getuser +import tempfile +import unittest from datetime import date +import xarray as xr # -- add python/ctsm to path (needed if we want to run the test stand-alone) _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) @@ -23,18 +21,12 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing from ctsm.site_and_regional.modify_singlept_site_neon import ( - get_parser, get_neon, find_surffile, - find_soil_structure, update_metadata, update_time_tag, - sort_print_soil_layers, check_neon_time, - download_file, - fill_interpolate, ) -from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name @@ -84,19 +76,32 @@ def test_find_surffile(self): pft_16 = True with self.assertRaises(SystemExit): find_surffile(surf_dir, site_name, pft_16) - # TODO: it would be useful to also include an example where the surface file is actually found or correctly searched for - # Is there a default surface data file directory we can point to? + # TODO: need to fix unkown file type error? # def test_find_soil_structure(self): # """ - # Test + # Test to ensure that correct attributes are found for find_soil_structure. + # soil_texture_raw_data_file_name should be found, and test should go through sysexit. # """ - # find_soil_structure() - # TODO: need file to use in test + # surf_file = "testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + # f1 = xr.open_dataset(surf_file) + # print(f1.attrs["Soil_texture_raw_data_file_name"]) + # self.assertEqual(f1.attrs["Soil_texture_raw_data_file_name"], + # "FIND FILENAME FROM ATTRIBUTES AND ADD HERE", + # "did not retrieve expected surface soil texture filename from surf file") # def test_update_metadata(self): - # TODO: do we have a surface data file that we can put in tempdir to update? - # update_metadata(nc, surf_file, neon_file, zb_flag) + # """ + # Test to ensure that an attribute has changed when update_metadata() is run. + # """ + # surf_file = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + # neon_file = "dummy_neon_file.nc" + # zb_flag = True + # f1 = xr.open_dataset("testinputs/" + surf_file) + # f2 = update_metadata(f1, surf_file, neon_file, zb_flag) + # self.assertNotEqual(f1.attrs["Updated_on"], + # f2.attrs["Updated_on"], + # "File was not updated as expected") def test_update_time_tag(self): """ @@ -120,13 +125,13 @@ def test_check_neon_time(self): 19, "last ABBY download has unexpected date format or does not exist", ) - # TODO: this checks that data is not pulled from before 2021; we may want to update this occassionally or find another way to check data is actually newest? + # Note: this checks that data is not pulled from before 2021; + # we may want to update this occassionally, + # but in any case it confirms that the oldest data is not found self.assertGreater( int(last_abby_download[:4]), 2021, "ABBY download is older than expected" ) - # TODO: test sort_print_soil_layers, fill_interpolate, find_soil_structure, and update_metadata - if __name__ == "__main__": unit_testing.setup_for_tests() From 86fba583b477ef6f554400e6a134d1703351fe56 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 11 Oct 2023 12:02:00 -0600 Subject: [PATCH 030/173] unit test updates after using LFS for referenced files --- .../modify_singlept_site_neon.py | 3 -- .../test_unit_modify_singlept_site_neon.py | 49 +++++++++---------- python/ctsm/test/testinputs/README.md | 2 +- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index ca85d68636..fc4964c0b4 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -502,12 +502,9 @@ def main(): # -- directory structure current_dir = os.getcwd() - # parent_dir = os.path.dirname(current_dir) clone_dir = os.path.abspath(os.path.join(__file__, "../../../..")) neon_dir = os.path.join(clone_dir, "neon_surffiles") - print("Present Directory", current_dir) - # -- download neon data if needed neon_file = get_neon(neon_dir, site_name) diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index f782075c43..e15e2c13ef 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -77,31 +77,30 @@ def test_find_surffile(self): with self.assertRaises(SystemExit): find_surffile(surf_dir, site_name, pft_16) - # TODO: need to fix unkown file type error? - # def test_find_soil_structure(self): - # """ - # Test to ensure that correct attributes are found for find_soil_structure. - # soil_texture_raw_data_file_name should be found, and test should go through sysexit. - # """ - # surf_file = "testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" - # f1 = xr.open_dataset(surf_file) - # print(f1.attrs["Soil_texture_raw_data_file_name"]) - # self.assertEqual(f1.attrs["Soil_texture_raw_data_file_name"], - # "FIND FILENAME FROM ATTRIBUTES AND ADD HERE", - # "did not retrieve expected surface soil texture filename from surf file") - - # def test_update_metadata(self): - # """ - # Test to ensure that an attribute has changed when update_metadata() is run. - # """ - # surf_file = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" - # neon_file = "dummy_neon_file.nc" - # zb_flag = True - # f1 = xr.open_dataset("testinputs/" + surf_file) - # f2 = update_metadata(f1, surf_file, neon_file, zb_flag) - # self.assertNotEqual(f1.attrs["Updated_on"], - # f2.attrs["Updated_on"], - # "File was not updated as expected") + def test_find_soil_structure(self): + """ + Test to ensure that correct attributes are found for find_soil_structure. + soil_texture_raw_data_file_name should be found, and test should go through sysexit. + """ + surf_file = "testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + f1 = xr.open_dataset(surf_file) + self.assertEqual(f1.attrs["Soil_texture_raw_data_file_name"], + "mksrf_soitex.10level.c010119.nc", + "did not retrieve expected surface soil texture filename from surf file") + + def test_update_metadata(self): + """ + Test to ensure that the file was updated today. + """ + surf_file = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + neon_file = "dummy_neon_file.nc" + zb_flag = True + f1 = xr.open_dataset("testinputs/" + surf_file) + f2 = update_metadata(f1, surf_file, neon_file, zb_flag) + today = date.today() + today_string = today.strftime("%Y-%m-%d") + self.assertEqual(f1.attrs["Updated_on"], today_string, + "File was not updated as expected") def test_update_time_tag(self): """ diff --git a/python/ctsm/test/testinputs/README.md b/python/ctsm/test/testinputs/README.md index 45451b53e1..1775de48b7 100644 --- a/python/ctsm/test/testinputs/README.md +++ b/python/ctsm/test/testinputs/README.md @@ -6,7 +6,7 @@ Installing Git LFS on your machine is a two-step process; step (1) needs to be done once per machine, and step (2) needs to be done once per user: 1. Install the Git LFS tool: Follow the instructions on the [Git LFS page](https://git-lfs.github.com/) for installing Git LFS on your platform. - - On cheyenne, Git LFS is already available as long as you are using a git + - On cheyenne and casper, Git LFS is already available as long as you are using a git module rather than the default system-level git. So just make sure that you are always using git via a git module (`module load git`). - On a Mac using homebrew, this can be done with `brew install git-lfs`. From ea307e0b1d3cd536f4ea72303d0358993615106a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 11 Oct 2023 12:21:33 -0600 Subject: [PATCH 031/173] small update to test + minor black reformatting --- .../test/test_unit_modify_singlept_site_neon.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index e15e2c13ef..b8315c7bae 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -82,11 +82,15 @@ def test_find_soil_structure(self): Test to ensure that correct attributes are found for find_soil_structure. soil_texture_raw_data_file_name should be found, and test should go through sysexit. """ - surf_file = "testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + surf_file = ( + "testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + ) f1 = xr.open_dataset(surf_file) - self.assertEqual(f1.attrs["Soil_texture_raw_data_file_name"], - "mksrf_soitex.10level.c010119.nc", - "did not retrieve expected surface soil texture filename from surf file") + self.assertEqual( + f1.attrs["Soil_texture_raw_data_file_name"], + "mksrf_soitex.10level.c010119.nc", + "did not retrieve expected surface soil texture filename from surf file", + ) def test_update_metadata(self): """ @@ -99,8 +103,7 @@ def test_update_metadata(self): f2 = update_metadata(f1, surf_file, neon_file, zb_flag) today = date.today() today_string = today.strftime("%Y-%m-%d") - self.assertEqual(f1.attrs["Updated_on"], today_string, - "File was not updated as expected") + self.assertEqual(f2.attrs["Updated_on"], today_string, "File was not updated as expected") def test_update_time_tag(self): """ From a851a75715d64ca300463c0bdb79b8a1e0fc4f59 Mon Sep 17 00:00:00 2001 From: John Alex Date: Fri, 13 Oct 2023 10:48:43 -0600 Subject: [PATCH 032/173] Call the new FATES-side FatesReadParameters (the one which doesn't need to call back into the HLM). No-op. --- src/utils/clmfates_interfaceMod.F90 | 18 ++++++---- src/utils/clmfates_paraminterfaceMod.F90 | 42 ++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/utils/clmfates_interfaceMod.F90 b/src/utils/clmfates_interfaceMod.F90 index e7e31b854a..1039a0c4b0 100644 --- a/src/utils/clmfates_interfaceMod.F90 +++ b/src/utils/clmfates_interfaceMod.F90 @@ -7,11 +7,9 @@ module CLMFatesInterfaceMod ! ! This is also the only location where CLM code is allowed to see FATES memory ! structures. - ! The routines here, that call FATES library routines, will not pass any types defined - ! by the driving land model (HLM). - ! - ! either native type arrays (int,real,log, etc) or packed into fates boundary condition - ! structures. + ! The routines here, that call FATES library routines, cannot pass most types defined + ! by the driving land model (HLM), only native type arrays (int,real,log, etc), implementations + ! of fates abstract classes, and references into fates boundary condition structures. ! ! Note that CLM/ALM does use Shared Memory Parallelism (SMP), where processes such as ! the update of state variables are forked. However, IO is not assumed to be @@ -110,6 +108,7 @@ module CLMFatesInterfaceMod use shr_log_mod , only : errMsg => shr_log_errMsg use clm_varcon , only : dzsoi_decomp use FuncPedotransferMod, only: get_ipedof + use CLMFatesParamInterfaceMod, only: fates_param_reader_ctsm_impl ! use SoilWaterPlantSinkMod, only : Compute_EffecRootFrac_And_VertTranSink_Default ! Used FATES Modules @@ -127,6 +126,9 @@ module CLMFatesInterfaceMod use FatesInterfaceMod , only : UpdateFatesRMeansTStep use FatesInterfaceMod , only : InitTimeAveragingGlobals + use FatesParametersInterface, only : fates_param_reader_type + use FatesParametersInterface, only : fates_parameters_type + use FatesHistoryInterfaceMod, only : fates_hist use FatesRestartInterfaceMod, only : fates_restart_interface_type @@ -272,6 +274,7 @@ subroutine CLMFatesGlobals1(surf_numpft,surf_numcft,maxsoil_patches) integer :: pass_sp integer :: pass_masterproc logical :: verbose_output + type(fates_param_reader_ctsm_impl) :: var_reader call t_startf('fates_globals1') @@ -315,6 +318,7 @@ subroutine CLMFatesGlobals1(surf_numpft,surf_numcft,maxsoil_patches) end if + ! The following call reads in the parameter file ! and then uses that to determine the number of patches ! FATES requires. We pass that to CLM here @@ -323,7 +327,7 @@ subroutine CLMFatesGlobals1(surf_numpft,surf_numcft,maxsoil_patches) ! and allocations on the FATES side, which require ! some allocations from CLM (like soil layering) - call SetFatesGlobalElements1(use_fates,surf_numpft,surf_numcft) + call SetFatesGlobalElements1(use_fates,surf_numpft,surf_numcft,var_reader) maxsoil_patches = fates_maxPatchesPerSite @@ -3393,4 +3397,6 @@ subroutine GetAndSetTime() end subroutine GetAndSetTime + !----------------------------------------------------------------------- + end module CLMFatesInterfaceMod diff --git a/src/utils/clmfates_paraminterfaceMod.F90 b/src/utils/clmfates_paraminterfaceMod.F90 index dedd8629cc..8f63c68965 100644 --- a/src/utils/clmfates_paraminterfaceMod.F90 +++ b/src/utils/clmfates_paraminterfaceMod.F90 @@ -5,15 +5,29 @@ module CLMFatesParamInterfaceMod use shr_kind_mod, only : r8 => shr_kind_r8, SHR_KIND_CL use FatesGlobals, only : fates_log use FatesParametersInterface, only : fates_parameters_type + use FatesParametersInterface, only : fates_param_reader_type use EDParamsMod, only : FatesRegisterParams, FatesReceiveParams use SFParamsMod, only : SpitFireRegisterParams, SpitFireReceiveParams use PRTInitParamsFATESMod, only : PRTRegisterParams, PRTReceiveParams use FatesSynchronizedParamsMod, only : FatesSynchronizedParamsInst implicit none + public :: fates_param_reader_ctsm_impl + ! + type, extends(fates_param_reader_type) :: fates_param_reader_ctsm_impl + private - ! NOTE(bja, 2017-01) these methods can NOT be part of the clmi-fates - ! nterface type because they are called before the instance is + ! !PRIVATE MEMBER DATA: + + contains + ! !PUBLIC MEMBER FUNCTIONS: + procedure, public :: Read ! Read params from disk + + end type fates_param_reader_ctsm_impl + + + ! NOTE(bja, 2017-01) these methods can NOT be part of the clm-fates + ! interface type because they are called before the instance is ! initialized. public :: FatesReadParameters public :: FatesReadPFTs @@ -238,4 +252,28 @@ subroutine ParametersFromNetCDF(filename, is_host_file, fates_params) end subroutine ParametersFromNetCDF !----------------------------------------------------------------------- + subroutine Read(this, is_host_file, fates_params ) + ! + ! !DESCRIPTION: + ! Read 'fates_params' parameters from appropriate filename given 'is_host_file'. + ! + ! USES + use clm_varctl, only : fname_len, paramfile, fates_paramfile + ! !ARGUMENTS: + class(fates_param_reader_ctsm_impl) :: this + logical, intent(in) :: is_host_file + class(fates_parameters_type), intent(inout) :: fates_params + !----------------------------------------------------------------------- + character(len=fname_len) :: filename ! file to read + + filename = fates_paramfile + if (is_host_file) then + filename = paramfile + endif + call ParametersFromNetCDF(filename, is_host_file, fates_params) + + end subroutine Read + + !----------------------------------------------------------------------- + end module CLMFatesParamInterfaceMod From 509bba32bad6c2ca4bad527cf711d4f0e346748c Mon Sep 17 00:00:00 2001 From: John Alex Date: Wed, 25 Oct 2023 09:58:05 -0600 Subject: [PATCH 033/173] Remove is_host_file param from fates_param_reader_type::Read implementation --- src/utils/clmfates_paraminterfaceMod.F90 | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/utils/clmfates_paraminterfaceMod.F90 b/src/utils/clmfates_paraminterfaceMod.F90 index 8f63c68965..95b1e428da 100644 --- a/src/utils/clmfates_paraminterfaceMod.F90 +++ b/src/utils/clmfates_paraminterfaceMod.F90 @@ -252,25 +252,20 @@ subroutine ParametersFromNetCDF(filename, is_host_file, fates_params) end subroutine ParametersFromNetCDF !----------------------------------------------------------------------- - subroutine Read(this, is_host_file, fates_params ) + subroutine Read(this, fates_params ) ! ! !DESCRIPTION: - ! Read 'fates_params' parameters from appropriate filename given 'is_host_file'. + ! Read 'fates_params' parameters from storage. ! ! USES use clm_varctl, only : fname_len, paramfile, fates_paramfile ! !ARGUMENTS: class(fates_param_reader_ctsm_impl) :: this - logical, intent(in) :: is_host_file class(fates_parameters_type), intent(inout) :: fates_params !----------------------------------------------------------------------- - character(len=fname_len) :: filename ! file to read + logical :: is_host_file = .false. - filename = fates_paramfile - if (is_host_file) then - filename = paramfile - endif - call ParametersFromNetCDF(filename, is_host_file, fates_params) + call ParametersFromNetCDF(fates_paramfile, is_host_file, fates_params) end subroutine Read From 85610b5d3b005acfd2d3a86f8946551ba43fb3ee Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 26 Oct 2023 14:10:27 -0600 Subject: [PATCH 034/173] include temporary system test files that still need adapting. Update unit tests to have more clearly invalid site names --- python/ctsm/site_and_regional/run_neon.py | 8 +- .../test_sys_modify_singlept_site_neon.py | 72 ++++++++++++++++ python/ctsm/test/test_sys_run_neon.py | 86 +++++++++++++++++++ .../test_unit_modify_singlept_site_neon.py | 8 +- python/ctsm/test/test_unit_run_neon.py | 11 +-- 5 files changed, 168 insertions(+), 17 deletions(-) create mode 100755 python/ctsm/test/test_sys_modify_singlept_site_neon.py create mode 100755 python/ctsm/test/test_sys_run_neon.py diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 987883fe93..e8706e4cce 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -415,12 +415,12 @@ def build_base_case( if re.search("^HIST", compset, flags=re.IGNORECASE) is None: expect( match is None, - "Existing base case is a historical type and should not be -- rerun with the --orverwrite option", + "Existing base case is a historical type and should not be -- rerun with the --overwrite option", ) else: expect( match is not None, - "Existing base case should be a historical type and is not -- rerun with the --orverwrite option", + "Existing base case should be a historical type and is not -- rerun with the --overwrite option", ) # reset the case case.case_setup(reset=True) @@ -534,12 +534,12 @@ def run_case( if re.search("^HIST", compset, flags=re.IGNORECASE) is None: expect( match is None, - "Existing base case is a historical type and should not be -- rerun with the --orverwrite option", + "Existing base case is a historical type and should not be -- rerun with the --overwrite option", ) else: expect( match is not None, - "Existing base case should be a historical type and is not -- rerun with the --orverwrite option", + "Existing base case should be a historical type and is not -- rerun with the --overwrite option", ) if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): print("Case {} appears to be complete, not rerunning.".format(case_root)) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py new file mode 100755 index 0000000000..9eb9ed9256 --- /dev/null +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +""" +System tests for modify_singlept_site_neon.py +""" + +import os +import re + +import unittest +import tempfile +import shutil +import sys + +import xarray as xr +import numpy as np + +# THESE LINES ARE JUST HERE FOR TESTING +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.path_utils import path_to_ctsm_root +from ctsm import unit_testing +from ctsm.site_and_regional import modify_singlept_site_neon +from ctsm.site_and_regional.modify_singlept_site_neon import main, get_parser + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + + +class TestSysModifySingleptSiteNeon(unittest.TestCase): + """System tests for modify_singlept_site_neon""" + + def setUp(self): + """ + Make /_tempdir for use by these tests. + Check tempdir for history files + """ + self._tempdir = tempfile.mkdtemp() + testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") + self._cfg_file_path = os.path.join( + testinputs_path, "modify_singlept_site_neon_opt_sections.cfg" + ) + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_modify_site(self): + """ + Test modifying a singple point site + The primary items to test here are the following: + 1) Fields are overwritten with site-specific data for neon sites + 2) Downloaded data is used in surface dataset + 3) Check specific fields listed in update_metadata for correct output + 4) Check that a netcdf with correct formatting is created + """ + sys.argv = ["--neon_site", ["ABBY"]] # self._cfg_file_path] #, "ABBY"] + print("sys.argv:") + print(sys.argv) + parser = get_parser() + print("parser:") + print(parser) + main() # running into error here because main doesn't take in any arguments and doesn't know what arguments are needed. + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py new file mode 100755 index 0000000000..f654c03111 --- /dev/null +++ b/python/ctsm/test/test_sys_run_neon.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 + +"""System tests for run_neon + +""" + +import os +import re + +import unittest +import tempfile +import shutil +import sys + +import xarray as xr +import numpy as np + +# THESE LINES ARE JUST HERE FOR TESTING +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.path_utils import path_to_ctsm_root, path_to_cime +from ctsm import unit_testing +from ctsm.site_and_regional import run_neon +from ctsm.site_and_regional.run_neon import main, get_parser + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + + +class TestSysRunNeon(unittest.TestCase): + """System tests for run_neon""" + + def setUp(self): + """ + Make /_tempdir for use by these tests. + Check tempdir for history files + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_one_site(self): + """ + This test specifies a site to run + Run the tool, check history file output exists + Could also check log files? Although this functionally doesn't change output, + it might be good to ensure the log files are working as expected? + Test running transient, ad and post ad cases. + Test use of base case root. + Test for using prism? + Test setup_only? This should be encapsulated within the full run? + """ + + # run the run_neon tool + # sys.argv = ["run_neon", "--neon_sites 'ABBY'"] + # sys.argv = ["--neon_sites", ["ABBY"]] + sys.argv = ["--neon-sites 'ABBY'"] + print("sys.argv:") + print(sys.argv) + valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] + parser = get_parser(sys.argv, "description_for_parser", valid_neon_sites) + print("parser:") + print(parser) + main("") + # this seems to run OSBS (default site, instead of ABBY), + # but does create files! It takes a while though, should we do setup-only? + # Could assert that dir is created with files; we should also move this into a tempdir? + + # the critical piece of this test is that the above command + # doesn't generate errors; however, we also do some assertions below + + # fsurdat_in_data = xr.open_dataset(self._fsurdat_in) + # fsurdat_out_data = xr.open_dataset(self._fsurdat_out) + # assert that fsurdat_out equals fsurdat_in + # self.assertTrue(fsurdat_out_data.equals(fsurdat_in_data)) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index b8315c7bae..21f7c839c2 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -18,6 +18,8 @@ _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) +from ctsm.path_utils import path_to_ctsm_root + # pylint: disable=wrong-import-position from ctsm import unit_testing from ctsm.site_and_regional.modify_singlept_site_neon import ( @@ -62,7 +64,7 @@ def test_get_neon_false_site(self): """ Test to see if neon data for invalid site name is found """ - site_name = "ABY" + site_name = "INVALID_SITE" neon_dir = self._tempdir with self.assertRaises(SystemExit): get_neon(neon_dir, site_name) @@ -83,7 +85,7 @@ def test_find_soil_structure(self): soil_texture_raw_data_file_name should be found, and test should go through sysexit. """ surf_file = ( - "testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc") ) f1 = xr.open_dataset(surf_file) self.assertEqual( @@ -99,7 +101,7 @@ def test_update_metadata(self): surf_file = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" neon_file = "dummy_neon_file.nc" zb_flag = True - f1 = xr.open_dataset("testinputs/" + surf_file) + f1 = xr.open_dataset(os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs/") + surf_file) f2 = update_metadata(f1, surf_file, neon_file, zb_flag) today = date.today() today_string = today.strftime("%Y-%m-%d") diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index a8031fff1d..72c1b67a31 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -29,15 +29,6 @@ class TestRunNeon(unittest.TestCase): Basic class for testing run_neon.py. """ - # def setUp(self): - # sys.argv = ["subset_data", "point", "--create-surface"] - # DEFAULTS_FILE = os.path.join(os.getcwd(), "ctsm/test/testinputs/default_data.cfg") - # self.parser = get_parser() - # self.args = self.parser.parse_args() - # self.cesmroot = path_to_ctsm_root() - # self.defaults = configparser.ConfigParser() - # self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) - def test_check_neon_listing(self): """ Test that neon listing is available for valid sites @@ -55,7 +46,7 @@ def test_check_neon_listing_misspelled(self): """ Test that neon listing is not available for invalid sites """ - valid_neon_sites = ["ABY", "BRT"] + valid_neon_sites = ["INVALID_SITE1", "INVALID_SITE2"] available_list = check_neon_listing(valid_neon_sites) self.assertEqual( available_list, [], "available list of incorrect dummy site not as expected" From 5714ffb8eb16382aefa7d0f9ab327784e2888026 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 26 Oct 2023 14:26:49 -0600 Subject: [PATCH 035/173] sys test assertion and formatting modify singlept site neon --- python/ctsm/test/test_sys_run_neon.py | 19 ++++++------------- .../test_unit_modify_singlept_site_neon.py | 9 ++++++--- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index f654c03111..c2599d782d 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -4,6 +4,7 @@ """ +import glob import os import re @@ -60,26 +61,18 @@ def test_one_site(self): # run the run_neon tool # sys.argv = ["run_neon", "--neon_sites 'ABBY'"] # sys.argv = ["--neon_sites", ["ABBY"]] + sys.argv = ["--neon-sites 'ABBY'"] - print("sys.argv:") - print(sys.argv) valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] parser = get_parser(sys.argv, "description_for_parser", valid_neon_sites) - print("parser:") - print(parser) main("") + # this seems to run OSBS (default site, instead of ABBY), # but does create files! It takes a while though, should we do setup-only? - # Could assert that dir is created with files; we should also move this into a tempdir? - - # the critical piece of this test is that the above command - # doesn't generate errors; however, we also do some assertions below - - # fsurdat_in_data = xr.open_dataset(self._fsurdat_in) - # fsurdat_out_data = xr.open_dataset(self._fsurdat_out) - # assert that fsurdat_out equals fsurdat_in - # self.assertTrue(fsurdat_out_data.equals(fsurdat_in_data)) + # Could assert that dir is created with files + #we should also move this into a tempdir and delete files after running (and cancel queue?) + self.assertTrue("OSBS" in glob.glob("*")) if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index 21f7c839c2..7817dd535e 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -84,8 +84,9 @@ def test_find_soil_structure(self): Test to ensure that correct attributes are found for find_soil_structure. soil_texture_raw_data_file_name should be found, and test should go through sysexit. """ - surf_file = ( - os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc") + surf_file = os.path.join( + path_to_ctsm_root(), + "python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc", ) f1 = xr.open_dataset(surf_file) self.assertEqual( @@ -101,7 +102,9 @@ def test_update_metadata(self): surf_file = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" neon_file = "dummy_neon_file.nc" zb_flag = True - f1 = xr.open_dataset(os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs/") + surf_file) + f1 = xr.open_dataset( + os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs/") + surf_file + ) f2 = update_metadata(f1, surf_file, neon_file, zb_flag) today = date.today() today_string = today.strftime("%Y-%m-%d") From 0bc8c04c60a1af06210c89d4d7dd575397085d15 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 26 Oct 2023 16:07:02 -0600 Subject: [PATCH 036/173] cleanup modify_singlept_site_neon.py --- .../ctsm/site_and_regional/modify_singlept_site_neon.py | 8 -------- python/ctsm/test/test_sys_run_neon.py | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index fc4964c0b4..a7838bedac 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -524,12 +524,6 @@ def main(): # better suggestion by WW to write dzsoi to neon surface dataset # This todo needs to go to the subset_data - # TODO Will: if I sum them up , are they 3.5? (m) YES - print("soil_top:", soil_top) - print("soil_bot:", soil_bot) - print("Sum of soil top depths :", sum(soil_top)) - print("Sum of soil bottom depths :", sum(soil_bot)) - soil_top = np.cumsum(soil_top) soil_bot = np.cumsum(soil_bot) soil_mid = 0.5 * (soil_bot - soil_top) + soil_top @@ -656,9 +650,7 @@ def main(): print("Updated : ", f2.PCT_CROP.values) print("Updating PCT_NAT_PFT") - # print (f2.PCT_NAT_PFT) print(f2.PCT_NAT_PFT.values[0]) - # f2.PCT_NAT_PFT.values[0] = [[100.0]] print(f2.PCT_NAT_PFT[0].values) out_dir = args.out_dir diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index c2599d782d..db90dc8bb0 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -70,10 +70,11 @@ def test_one_site(self): # this seems to run OSBS (default site, instead of ABBY), # but does create files! It takes a while though, should we do setup-only? # Could assert that dir is created with files - #we should also move this into a tempdir and delete files after running (and cancel queue?) + # we should also move this into a tempdir and delete files after running (and cancel queue?) self.assertTrue("OSBS" in glob.glob("*")) + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() From 2bf01c969656dc41cc3a1a32decdb76d1baeecc1 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 27 Oct 2023 11:43:22 -0600 Subject: [PATCH 037/173] minor updates to clean up modify_singlept_site_neon.py --- python/ctsm/site_and_regional/modify_singlept_site_neon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ctsm/site_and_regional/modify_singlept_site_neon.py b/python/ctsm/site_and_regional/modify_singlept_site_neon.py index a7838bedac..1013ba944c 100755 --- a/python/ctsm/site_and_regional/modify_singlept_site_neon.py +++ b/python/ctsm/site_and_regional/modify_singlept_site_neon.py @@ -529,7 +529,6 @@ def main(): soil_mid = 0.5 * (soil_bot - soil_top) + soil_top # print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) - # obs_top = df["biogeoTopDepth"] / 100 obs_bot = df["biogeoBottomDepth"] / 100 # -- Mapping surface dataset and neon soil levels From 37db306904dbc8649a76c0b5b7d04eb022013116 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 30 Oct 2023 17:56:15 -0600 Subject: [PATCH 038/173] Update comments, add placeholder for thermal_cond_snow option --- src/biogeophys/SoilTemperatureMod.F90 | 36 +++++++++++++++++++-------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/src/biogeophys/SoilTemperatureMod.F90 b/src/biogeophys/SoilTemperatureMod.F90 index caeedd1603..d3e797631b 100644 --- a/src/biogeophys/SoilTemperatureMod.F90 +++ b/src/biogeophys/SoilTemperatureMod.F90 @@ -47,7 +47,7 @@ module SoilTemperatureMod ! o The thermal conductivity of soil is computed from ! the algorithm of Johansen (as reported by Farouki 1981), and the ! conductivity of snow is from the formulation used in - ! Sturm (1997). + ! Sturm (1997) or Jordan (1991) p. 18 depending on namelist option. ! o Boundary conditions: ! F = Rnet - Hg - LEg (top), F= 0 (base of the soil column). ! o Soil / snow temperature is predicted from heat conduction @@ -100,7 +100,7 @@ subroutine SoilTemperature(bounds, num_urbanl, filter_urbanl, num_urbanc, filter ! o The thermal conductivity of soil is computed from ! the algorithm of Johansen (as reported by Farouki 1981), and the ! conductivity of snow is from the formulation used in - ! Sturm (1997). + ! Sturm (1997) or Jordan (1991) p. 18 depending on namelist option. ! o Boundary conditions: ! F = Rnet - Hg - LEg (top), F= 0 (base of the soil column). ! o Soil / snow temperature is predicted from heat conduction @@ -611,7 +611,8 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter ! ! (2) The thermal conductivity of soil is computed from the algorithm of ! Johansen (as reported by Farouki 1981), and of snow is from the - ! formulation used in Sturm (1997). + ! formulation used in Sturm (1997) or Jordan (1991) p. 18 depending on + ! namelist option. ! The thermal conductivities at the interfaces between two neighboring ! layers (j, j+1) are derived from an assumption that the flux across ! the interface is equal to that from the node j to the interface and the @@ -734,16 +735,29 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter endif endif - ! Thermal conductivity of snow, which from Sturm (1997) + ! Thermal conductivity of snow ! Only examine levels from snl(c)+1 -> 0 where snl(c) < 1 if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then - bw(c,j) = ((h2osoi_ice(c,j)*1)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) ! ==RHOS - ! thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) ! Original (Jordan) Parameterisation - if (bw(c,j) <= 156) then !LMW or 0.156 ? - thk(c,j) = 0.023 + 0.234*(bw(c,j)/1000) !LMW - units changed by VRD - else !LMW - thk(c,j) = 0.138 - 1.01*(bw(c,j)/1000) +(3.233*((bw(c,j)/1000)*(bw(c,j)/1000))) ! LMW Sturm I think - end if + bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) +! TODO slevis: Add namelist option thermal_cond_snow and then +! uncomment relevant lines below +! select case (thermal_cond_snow) +! case ('Jordan1991') +! thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) +! case ('Sturm1997') + ! Implemented by Vicky Dutch (VRD), Nick Rutter, and + ! Leanne Wake (LMW) + ! https://tc.copernicus.org/articles/16/4201/2022/ + ! Code provided by Adrien Dams to Will Wieder + if (bw(c,j) <= 156) then !LMW or 0.156 ? + thk(c,j) = 0.023 + 0.234*(bw(c,j)/1000) !LMW - units changed by VRD + else !LMW + thk(c,j) = 0.138 - 1.01*(bw(c,j)/1000) +(3.233*((bw(c,j)/1000)*(bw(c,j)/1000))) ! LMW Sturm I think + end if +! case default +! write(iulog,*) subname//' ERROR: unknown thermal_cond_snow value: ', thermal_cond_snow +! call endrun(msg=errMsg(sourcefile, __LINE__)) +! end select end if end do From 1bc6e7205fa9b33b608d26c4460b68386c275edf Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 31 Oct 2023 09:37:35 -0600 Subject: [PATCH 039/173] functional (but simple) sys tests --- .../test_sys_modify_singlept_site_neon.py | 17 ++++++------- python/ctsm/test/test_sys_run_neon.py | 25 +++++++------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 9eb9ed9256..eb0575789d 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -51,21 +51,20 @@ def tearDown(self): def test_modify_site(self): """ - Test modifying a singple point site - The primary items to test here are the following: + Test modifying a singple point site. This test currently checks that the run fails due to dir structure + + TODO: The primary items to test here are the following: 1) Fields are overwritten with site-specific data for neon sites 2) Downloaded data is used in surface dataset 3) Check specific fields listed in update_metadata for correct output 4) Check that a netcdf with correct formatting is created """ - sys.argv = ["--neon_site", ["ABBY"]] # self._cfg_file_path] #, "ABBY"] - print("sys.argv:") - print(sys.argv) + sys.argv = ["modify_singlept_site_neon", "--neon_site", "/glade/work/tking/ctsm/cime_config/usermods_dirs/NEON/ABBY"] + # TODO: the above requires a full path instead of site name because of how run_neon is configured... This needs to be fixed/generalized. parser = get_parser() - print("parser:") - print(parser) - main() # running into error here because main doesn't take in any arguments and doesn't know what arguments are needed. - + with self.assertRaises(SystemExit): + print("This should currently fail due to directory structure in run_neon and the directory structure listed in sys.argv") + main() if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index db90dc8bb0..7f83eb6453 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -49,30 +49,23 @@ def tearDown(self): def test_one_site(self): """ This test specifies a site to run - Run the tool, check history file output exists - Could also check log files? Although this functionally doesn't change output, - it might be good to ensure the log files are working as expected? - Test running transient, ad and post ad cases. - Test use of base case root. - Test for using prism? - Test setup_only? This should be encapsulated within the full run? + Run the tool, check that file structure is set up correctly """ # run the run_neon tool - # sys.argv = ["run_neon", "--neon_sites 'ABBY'"] - # sys.argv = ["--neon_sites", ["ABBY"]] - - sys.argv = ["--neon-sites 'ABBY'"] + sys.argv = ["run_neon", "--neon-sites", "BART", "--setup-only", "--output-root", self._tempdir] valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] parser = get_parser(sys.argv, "description_for_parser", valid_neon_sites) main("") - # this seems to run OSBS (default site, instead of ABBY), - # but does create files! It takes a while though, should we do setup-only? - # Could assert that dir is created with files - # we should also move this into a tempdir and delete files after running (and cancel queue?) + # assert that BART directories were created during setup + self.assertTrue("BART" in glob.glob(self._tempdir+"/*")) - self.assertTrue("OSBS" in glob.glob("*")) + # TODO: Would also be useful to test the following items: + # It might be good to ensure the log files are working as expected? + # Test running transient, ad and post ad cases. + # Test use of base case root. + # Test for using prism? if __name__ == "__main__": From 6be5dae9f83b6172da6a33676e71a6cb60c5054d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 31 Oct 2023 09:46:03 -0600 Subject: [PATCH 040/173] minor reformatting --- .../test_sys_modify_singlept_site_neon.py | 24 +++++++++++-------- python/ctsm/test/test_sys_run_neon.py | 18 +++++++------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index eb0575789d..d6d4acce74 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -5,23 +5,17 @@ """ import os -import re - import unittest import tempfile import shutil import sys -import xarray as xr -import numpy as np - # THESE LINES ARE JUST HERE FOR TESTING _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) from ctsm.path_utils import path_to_ctsm_root from ctsm import unit_testing -from ctsm.site_and_regional import modify_singlept_site_neon from ctsm.site_and_regional.modify_singlept_site_neon import main, get_parser # Allow test names that pylint doesn't like; otherwise hard to make them @@ -51,7 +45,8 @@ def tearDown(self): def test_modify_site(self): """ - Test modifying a singple point site. This test currently checks that the run fails due to dir structure + Test modifying a singple point site. + This test currently checks that the run fails due to dir structure TODO: The primary items to test here are the following: 1) Fields are overwritten with site-specific data for neon sites @@ -59,13 +54,22 @@ def test_modify_site(self): 3) Check specific fields listed in update_metadata for correct output 4) Check that a netcdf with correct formatting is created """ - sys.argv = ["modify_singlept_site_neon", "--neon_site", "/glade/work/tking/ctsm/cime_config/usermods_dirs/NEON/ABBY"] - # TODO: the above requires a full path instead of site name because of how run_neon is configured... This needs to be fixed/generalized. + sys.argv = [ + "modify_singlept_site_neon", + "--neon_site", + "/glade/work/tking/ctsm/cime_config/usermods_dirs/NEON/ABBY", + ] + # TODO: the above requires a full path instead of site name because of how run_neon is configured. + # This needs to be fixed/generalized. parser = get_parser() with self.assertRaises(SystemExit): - print("This should currently fail due to directory structure in run_neon and the directory structure listed in sys.argv") + print( + """This should currently fail due to directory structure in run_neon + and the directory structure listed in sys.argv""" + ) main() + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index 7f83eb6453..d19c18002c 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -6,23 +6,16 @@ import glob import os -import re - import unittest import tempfile import shutil import sys -import xarray as xr -import numpy as np - # THESE LINES ARE JUST HERE FOR TESTING _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) sys.path.insert(1, _CTSM_PYTHON) -from ctsm.path_utils import path_to_ctsm_root, path_to_cime from ctsm import unit_testing -from ctsm.site_and_regional import run_neon from ctsm.site_and_regional.run_neon import main, get_parser # Allow test names that pylint doesn't like; otherwise hard to make them @@ -53,13 +46,20 @@ def test_one_site(self): """ # run the run_neon tool - sys.argv = ["run_neon", "--neon-sites", "BART", "--setup-only", "--output-root", self._tempdir] + sys.argv = [ + "run_neon", + "--neon-sites", + "BART", + "--setup-only", + "--output-root", + self._tempdir, + ] valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] parser = get_parser(sys.argv, "description_for_parser", valid_neon_sites) main("") # assert that BART directories were created during setup - self.assertTrue("BART" in glob.glob(self._tempdir+"/*")) + self.assertTrue("BART" in glob.glob(self._tempdir + "/*")) # TODO: Would also be useful to test the following items: # It might be good to ensure the log files are working as expected? From 4dc31b1a47c45e856d8489c116d32a84b3af541a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 31 Oct 2023 13:24:10 -0600 Subject: [PATCH 041/173] update path --- python/ctsm/test/test_sys_modify_singlept_site_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index d6d4acce74..9ea1322141 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -57,7 +57,7 @@ def test_modify_site(self): sys.argv = [ "modify_singlept_site_neon", "--neon_site", - "/glade/work/tking/ctsm/cime_config/usermods_dirs/NEON/ABBY", + path_to_ctsm_root()+"/ctsm/cime_config/usermods_dirs/NEON/ABBY", ] # TODO: the above requires a full path instead of site name because of how run_neon is configured. # This needs to be fixed/generalized. From a83784b177dcf3dfb15344fa64802f9d3c6c8aaf Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 31 Oct 2023 13:45:31 -0600 Subject: [PATCH 042/173] reformat --- python/ctsm/test/test_sys_modify_singlept_site_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 9ea1322141..ab285f1a8b 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -57,7 +57,7 @@ def test_modify_site(self): sys.argv = [ "modify_singlept_site_neon", "--neon_site", - path_to_ctsm_root()+"/ctsm/cime_config/usermods_dirs/NEON/ABBY", + path_to_ctsm_root() + "/ctsm/cime_config/usermods_dirs/NEON/ABBY", ] # TODO: the above requires a full path instead of site name because of how run_neon is configured. # This needs to be fixed/generalized. From d708fb3aa0f72391affedbe63f5fdee2e25a2e38 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 1 Nov 2023 11:31:44 -0600 Subject: [PATCH 043/173] minor fix --- python/ctsm/test/test_sys_run_neon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index d19c18002c..eb22ac45d5 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -59,7 +59,8 @@ def test_one_site(self): main("") # assert that BART directories were created during setup - self.assertTrue("BART" in glob.glob(self._tempdir + "/*")) + print(glob.glob(self._tempdir + "/BART*")) + self.assertTrue("BART" in glob.glob(self._tempdir + "/BART*")[0]) # TODO: Would also be useful to test the following items: # It might be good to ensure the log files are working as expected? From 338b1800ac73dc6015b82dae40b20ef24314456c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 1 Nov 2023 11:44:59 -0600 Subject: [PATCH 044/173] remove print statement --- python/ctsm/test/test_sys_run_neon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index eb22ac45d5..d14b2ce312 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -59,7 +59,6 @@ def test_one_site(self): main("") # assert that BART directories were created during setup - print(glob.glob(self._tempdir + "/BART*")) self.assertTrue("BART" in glob.glob(self._tempdir + "/BART*")[0]) # TODO: Would also be useful to test the following items: From 07d1ec326b69d61a01eee883670a8d2e2796e5f5 Mon Sep 17 00:00:00 2001 From: John Alex Date: Wed, 1 Nov 2023 12:24:12 -0600 Subject: [PATCH 045/173] Remove CLM-side FatesReadParameters (The FATES repo will be updated to define this method within FATES) --- src/utils/clmfates_paraminterfaceMod.F90 | 39 ------------------------ 1 file changed, 39 deletions(-) diff --git a/src/utils/clmfates_paraminterfaceMod.F90 b/src/utils/clmfates_paraminterfaceMod.F90 index 95b1e428da..ea27f563bf 100644 --- a/src/utils/clmfates_paraminterfaceMod.F90 +++ b/src/utils/clmfates_paraminterfaceMod.F90 @@ -29,7 +29,6 @@ module CLMFatesParamInterfaceMod ! NOTE(bja, 2017-01) these methods can NOT be part of the clm-fates ! interface type because they are called before the instance is ! initialized. - public :: FatesReadParameters public :: FatesReadPFTs private :: ParametersFromNetCDF private :: SetParameterDimensions @@ -42,44 +41,6 @@ module CLMFatesParamInterfaceMod contains - !----------------------------------------------------------------------- - subroutine FatesReadParameters() - use clm_varctl, only : use_fates, paramfile, fates_paramfile - use spmdMod, only : masterproc - - implicit none - - character(len=32) :: subname = 'FatesReadParameters' - class(fates_parameters_type), allocatable :: fates_params - logical :: is_host_file - - if (masterproc) then - write(fates_log(), *) 'clmfates_interfaceMod.F90::'//trim(subname)//' :: CLM reading ED/FATES '//' parameters ' - end if - - allocate(fates_params) - call fates_params%Init() ! fates_params class, in FatesParameterInterfaceMod - call FatesRegisterParams(fates_params) !EDParamsMod, only operates on fates_params class - call SpitFireRegisterParams(fates_params) !SpitFire Mod, only operates of fates_params class - call PRTRegisterParams(fates_params) ! PRT mod, only operates on fates_params class - call FatesSynchronizedParamsInst%RegisterParams(fates_params) !Synchronized params class in Synchronized params mod, only operates on fates_params class - - is_host_file = .false. - call ParametersFromNetCDF(fates_paramfile, is_host_file, fates_params) - - is_host_file = .true. - call ParametersFromNetCDF(paramfile, is_host_file, fates_params) - - call FatesReceiveParams(fates_params) - call SpitFireReceiveParams(fates_params) - call PRTReceiveParams(fates_params) - call FatesSynchronizedParamsInst%ReceiveParams(fates_params) - - call fates_params%Destroy() - deallocate(fates_params) - - end subroutine FatesReadParameters - !----------------------------------------------------------------------- subroutine FatesReadPFTs() From 75d5f10f09fd7b9ef3e2cb535a5e4d30976d28a1 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 1 Nov 2023 16:53:23 -0600 Subject: [PATCH 046/173] Add new namelist variable snow_thermal_cond_method (not tested) --- bld/CLMBuildNamelist.pm | 7 +++++++ bld/namelist_files/namelist_defaults_ctsm.xml | 1 + .../namelist_definition_ctsm.xml | 5 +++++ src/biogeophys/SoilTemperatureMod.F90 | 21 +++++++++---------- src/main/clm_varctl.F90 | 2 ++ src/main/controlMod.F90 | 4 +++- 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index c12c54645f..5afaaab40d 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -2031,6 +2031,13 @@ sub setup_logic_snicar_methods { sub setup_logic_snow { my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'snow_thermal_cond_method' ); + + my $var = $nl->get_value('snow_thermal_cond_method'); + if ( $var ne 'Jordan1991' && $var ne 'Sturm1997' ) { + $log->fatal_error("$var is incorrect entry for the namelist variable snow_thermal_cond_method; expected Jordan1991 or Sturm1997"); + } + my $numrad_snw = $nl->get_value('snicar_numrad_snw'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fsnowoptics', 'snicar_numrad_snw' => $numrad_snw); diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2e57391df7..b23e67ed6b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -445,6 +445,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 1.e9 SwensonLawrence2012 +Jordan1991 diff --git a/src/biogeophys/SoilTemperatureMod.F90 b/src/biogeophys/SoilTemperatureMod.F90 index d3e797631b..69f9958efa 100644 --- a/src/biogeophys/SoilTemperatureMod.F90 +++ b/src/biogeophys/SoilTemperatureMod.F90 @@ -623,7 +623,8 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter use clm_varcon , only : denh2o, denice, tfrz, tkwat, tkice, tkair, cpice, cpliq, thk_bedrock, csol_bedrock use landunit_varcon , only : istice, istwet use column_varcon , only : icol_roof, icol_sunwall, icol_shadewall, icol_road_perv, icol_road_imperv - use clm_varctl , only : iulog + use clm_varctl , only : iulog, snow_thermal_cond_method + ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -739,12 +740,10 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter ! Only examine levels from snl(c)+1 -> 0 where snl(c) < 1 if (snl(c)+1 < 1 .AND. (j >= snl(c)+1) .AND. (j <= 0)) then bw(c,j) = (h2osoi_ice(c,j)+h2osoi_liq(c,j))/(frac_sno(c)*dz(c,j)) -! TODO slevis: Add namelist option thermal_cond_snow and then -! uncomment relevant lines below -! select case (thermal_cond_snow) -! case ('Jordan1991') -! thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) -! case ('Sturm1997') + select case (snow_thermal_cond_method) + case ('Jordan1991') + thk(c,j) = tkair + (7.75e-5_r8 *bw(c,j) + 1.105e-6_r8*bw(c,j)*bw(c,j))*(tkice-tkair) + case ('Sturm1997') ! Implemented by Vicky Dutch (VRD), Nick Rutter, and ! Leanne Wake (LMW) ! https://tc.copernicus.org/articles/16/4201/2022/ @@ -754,10 +753,10 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter else !LMW thk(c,j) = 0.138 - 1.01*(bw(c,j)/1000) +(3.233*((bw(c,j)/1000)*(bw(c,j)/1000))) ! LMW Sturm I think end if -! case default -! write(iulog,*) subname//' ERROR: unknown thermal_cond_snow value: ', thermal_cond_snow -! call endrun(msg=errMsg(sourcefile, __LINE__)) -! end select + case default + write(iulog,*) subname//' ERROR: unknown snow_thermal_cond_method value: ', snow_thermal_cond_method + call endrun(msg=errMsg(sourcefile, __LINE__)) + end select end if end do diff --git a/src/main/clm_varctl.F90 b/src/main/clm_varctl.F90 index 95c800504b..a5201a53cf 100644 --- a/src/main/clm_varctl.F90 +++ b/src/main/clm_varctl.F90 @@ -221,6 +221,8 @@ module clm_varctl ! which snow cover fraction parameterization to use character(len=64), public :: snow_cover_fraction_method + ! which snow thermal conductivity parameterization to use + character(len=25), public :: snow_thermal_cond_method ! atmospheric CO2 molar ratio (by volume) (umol/mol) real(r8), public :: co2_ppmv = 355._r8 ! diff --git a/src/main/controlMod.F90 b/src/main/controlMod.F90 index 5937e55b04..deb8c044d8 100644 --- a/src/main/controlMod.F90 +++ b/src/main/controlMod.F90 @@ -199,7 +199,8 @@ subroutine control_init(dtime) clump_pproc, & create_crop_landunit, nsegspc, co2_ppmv, & albice, soil_layerstruct_predefined, soil_layerstruct_userdefined, & - soil_layerstruct_userdefined_nlevsoi, use_subgrid_fluxes, snow_cover_fraction_method, & + soil_layerstruct_userdefined_nlevsoi, use_subgrid_fluxes, & + snow_thermal_cond_method, snow_cover_fraction_method, & irrigate, run_zero_weight_urban, all_active, & crop_fsat_equals_zero, for_testing_run_ncdiopio_tests, & for_testing_use_second_grain_pool, for_testing_use_repr_structure_pool, & @@ -850,6 +851,7 @@ subroutine control_spmd() ! physics variables call mpi_bcast (nsegspc, 1, MPI_INTEGER, 0, mpicom, ier) call mpi_bcast (use_subgrid_fluxes , 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (snow_thermal_cond_method, len(snow_thermal_cond_method), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (snow_cover_fraction_method , len(snow_cover_fraction_method), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (z0param_method , len(z0param_method), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (use_z0m_snowmelt, 1, MPI_LOGICAL, 0, mpicom, ier) From 0b066bee71e80cbc7b7d164c0d862ed5cb9b9af1 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 2 Nov 2023 12:28:15 -0600 Subject: [PATCH 047/173] cleaning up tests and implementing path fix with chdir to tempdir --- python/ctsm/test/test_sys_modify_singlept_site_neon.py | 7 ++----- python/ctsm/test/test_sys_run_neon.py | 7 +++---- python/ctsm/test/test_unit_neon_surf_wrapper.py | 3 --- python/ctsm/test/test_unit_run_neon.py | 3 --- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index ab285f1a8b..3f2dc6d7f2 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -10,10 +10,6 @@ import shutil import sys -# THESE LINES ARE JUST HERE FOR TESTING -_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) -sys.path.insert(1, _CTSM_PYTHON) - from ctsm.path_utils import path_to_ctsm_root from ctsm import unit_testing from ctsm.site_and_regional.modify_singlept_site_neon import main, get_parser @@ -59,7 +55,8 @@ def test_modify_site(self): "--neon_site", path_to_ctsm_root() + "/ctsm/cime_config/usermods_dirs/NEON/ABBY", ] - # TODO: the above requires a full path instead of site name because of how run_neon is configured. + # TODO: the above requires a full path instead of site name + # because of how run_neon is configured. # This needs to be fixed/generalized. parser = get_parser() with self.assertRaises(SystemExit): diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index d14b2ce312..74bee17b35 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -11,10 +11,6 @@ import shutil import sys -# THESE LINES ARE JUST HERE FOR TESTING -_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) -sys.path.insert(1, _CTSM_PYTHON) - from ctsm import unit_testing from ctsm.site_and_regional.run_neon import main, get_parser @@ -32,6 +28,7 @@ def setUp(self): Check tempdir for history files """ self._tempdir = tempfile.mkdtemp() + os.chdir(self._tempdir) # cd to tempdir def tearDown(self): """ @@ -52,6 +49,7 @@ def test_one_site(self): "BART", "--setup-only", "--output-root", + #"FAKE_DIR" self._tempdir, ] valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] @@ -60,6 +58,7 @@ def test_one_site(self): # assert that BART directories were created during setup self.assertTrue("BART" in glob.glob(self._tempdir + "/BART*")[0]) + #self.assertTrue("BART" in glob.glob("BART*")[0]) # TODO: Would also be useful to test the following items: # It might be good to ensure the log files are working as expected? diff --git a/python/ctsm/test/test_unit_neon_surf_wrapper.py b/python/ctsm/test/test_unit_neon_surf_wrapper.py index 0092bdb91c..443af2079b 100755 --- a/python/ctsm/test/test_unit_neon_surf_wrapper.py +++ b/python/ctsm/test/test_unit_neon_surf_wrapper.py @@ -7,8 +7,6 @@ """ import unittest -import configparser -import argparse import os import sys @@ -19,7 +17,6 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing from ctsm.site_and_regional.neon_surf_wrapper import get_parser -from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index 72c1b67a31..314b1e8b89 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -7,8 +7,6 @@ """ import unittest -import configparser -import argparse import os import sys @@ -19,7 +17,6 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing from ctsm.site_and_regional.run_neon import check_neon_listing -from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name From f477d610887b793f32dfd6bb91da7bab4451e959 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 2 Nov 2023 13:29:57 -0600 Subject: [PATCH 048/173] fix for listing.csv file generated in python dir --- python/ctsm/test/test_sys_run_neon.py | 4 ++-- .../test_unit_modify_singlept_site_neon.py | 4 ++++ python/ctsm/test/test_unit_run_neon.py | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index 74bee17b35..4dd5aca896 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -49,7 +49,7 @@ def test_one_site(self): "BART", "--setup-only", "--output-root", - #"FAKE_DIR" + # "FAKE_DIR" self._tempdir, ] valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] @@ -58,7 +58,7 @@ def test_one_site(self): # assert that BART directories were created during setup self.assertTrue("BART" in glob.glob(self._tempdir + "/BART*")[0]) - #self.assertTrue("BART" in glob.glob("BART*")[0]) + # self.assertTrue("BART" in glob.glob("BART*")[0]) # TODO: Would also be useful to test the following items: # It might be good to ensure the log files are working as expected? diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index 7817dd535e..4c3a473394 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -124,6 +124,8 @@ def test_check_neon_time(self): """ Test that dictionary containing last modified information is correctly downloaded """ + previous_dir = os.getcwd() + os.chdir(self._tempdir) # cd to tempdir last_abby_download = check_neon_time()[ "https://storage.neonscience.org/neon-ncar/NEON/surf_files/v1/ABBY_surfaceData.csv" ] @@ -138,6 +140,8 @@ def test_check_neon_time(self): self.assertGreater( int(last_abby_download[:4]), 2021, "ABBY download is older than expected" ) + # change back to previous dir once listing.csv file has been created in tempdir and test complete + os.chdir(previous_dir) if __name__ == "__main__": diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index 314b1e8b89..2cdcc915ad 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -7,6 +7,8 @@ """ import unittest +import tempfile +import shutil import os import sys @@ -26,11 +28,25 @@ class TestRunNeon(unittest.TestCase): Basic class for testing run_neon.py. """ + def setUp(self): + """ + Make /_tempdir for use by these tests. + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + def test_check_neon_listing(self): """ Test that neon listing is available for valid sites """ valid_neon_sites = ["ABBY", "BART"] + previous_dir = os.getcwd() + os.chdir(self._tempdir) # cd to tempdir available_list = check_neon_listing(valid_neon_sites) self.assertEqual( available_list[0].name, "ABBY", "available list of actual sites not as expected" @@ -38,16 +54,22 @@ def test_check_neon_listing(self): self.assertEqual( available_list[1].name, "BART", "available list of actual sites not as expected" ) + # change back to previous dir once listing.csv file has been created in tempdir and test complete + os.chdir(previous_dir) def test_check_neon_listing_misspelled(self): """ Test that neon listing is not available for invalid sites """ valid_neon_sites = ["INVALID_SITE1", "INVALID_SITE2"] + previous_dir = os.getcwd() + os.chdir(self._tempdir) # cd to tempdir available_list = check_neon_listing(valid_neon_sites) self.assertEqual( available_list, [], "available list of incorrect dummy site not as expected" ) + # change back to previous dir once listing.csv file has been created in tempdir and test complete + os.chdir(previous_dir) if __name__ == "__main__": From f2312244149b37694f87a73be2012ca9fbccd734 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 2 Nov 2023 16:16:15 -0600 Subject: [PATCH 049/173] cleanup; changelog; some pylint fixes --- doc/ChangeLog | 138 ++++++++++++++++++ doc/ChangeSum | 1 + python/README.md | 3 +- python/ctsm/run_ctsm_py_tests.py | 5 +- .../site_and_regional/neon_surf_wrapper.py | 10 +- python/ctsm/site_and_regional/run_neon.py | 2 + .../test_sys_modify_singlept_site_neon.py | 2 +- python/ctsm/test/test_sys_run_neon.py | 7 +- .../test_unit_modify_singlept_site_neon.py | 5 +- python/ctsm/test/test_unit_run_neon.py | 4 +- 10 files changed, 160 insertions(+), 17 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 955f071933..d787c8d36c 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,142 @@ =============================================================== +Tag name: ctsm5.1.dev151 +Originator(s): tking (Teagan King,UCAR/CCR) +Date: Thu Nov 2 16:03:44 MDT 2023 +One-line Summary: Move tools to python directory and add tests + +Purpose and description of changes +---------------------------------- + +Move the following scripts to /python/ctsm/site_and_regional +and make wrapper scripts for them in /tools/site_and_regional: +- run_neon.py +- neon_surf_wrapper.py +- modify_singlept_site_neon.py + +Add unit testing for: +- iso_utils +- modify_singlept_site_neon +- neon_surf_wrapper +- run_neon + +Add system testing for: +- modify_singlept_site_neon +- run_neon + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +#1441 + +Known Issues: +There were some previous pylint errors that still remain in this tag. + +Notes of particular relevance for users +--------------------------------------- +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): +wrapper scripts don't have .py suffixes + +Notes of particular relevance for developers: +--------------------------------------------- +Changes to tests or testing: +Numerous changes were made to include new tests. +README.md for testing was also updated to clarify that arguments should be used. + +Testing summary: +---------------- +[... Remove before making master tag. + +Nearly all CTSM tags should undergo 'regular' (aux_clm) testing. +However, it occasionally makes sense to do more or less system testing; +here is guidance on different available levels of system testing: + a) no system testing (for use when the only changes are ones that + have absolutely no impact on system runs; this + includes documentation-only tags, tags that + just change the tools or some python code that + does not impact system runs, etc.) + b) minimal (for use in rare cases where only a small change with + known behavior is added ... eg. a minor bug fix. This + might be to just run the "short" test list, or to run + a single test. Whatever makes sense for the particular case.) + c) python only (for use where the only changes are in the python directory: + run the python testing listed below) + d) regular (regular tests on normal machines if CTSM source is modified) + e) release (regular tests plus the fates, ctsm_sci, mosart and rtm test lists + and normally all of the ancillary tests (build-namelist, python, ptclm, etc.) + would be run as well) + +In addition, various other tests of the tools, python and perl +infrastructure should be run when appropriate, as described below. + +...] + +[Remove any lines that don't apply.] + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + (any machine) - + + [If python code has changed and you are NOT running aux_clm (e.g., because the only changes are in python + code) then also run the clm_pymods test suite; this is a small subset of aux_clm that runs the system + tests impacted by python changes. The best way to do this, if you expect no changes from the last tag in + either model output or namelists, is: create sym links pointing to the last tag's baseline directory, + named with the upcoming tag; then run the clm_pymods test suite comparing against these baselines but NOT + doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to + separately run the clm_pymods test suite, and you can remove the following line.] + + clm_pymods test suite on cheyenne - + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- + izumi ------- + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- + izumi ------- + + any other testing (give details below): + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + +Answer changes +-------------- +Changes answers relative to baseline: None + +Other details +------------- +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull/2156) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev140 Originator(s): afoster (Adrianna Foster) Date: Tue Sep 12 14:47:06 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 5ca9ceab70..b5d5170267 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev151 tking 11/02/2023 Move tools to python directory and add tests ctsm5.1.dev140 afoster 09/12/2023 add lai_streams capability for FATES ctsm5.1.dev139 slevis 08/28/2023 Fix problems uncovered by nag -nan tests ctsm5.1.dev138 slevis 08/25/2023 Refactor max_patch_per_col and maxsoil_patches loops diff --git a/python/README.md b/python/README.md index cf3b893084..c40f55c6c7 100644 --- a/python/README.md +++ b/python/README.md @@ -47,7 +47,8 @@ thing, but support different options: 2. via `./run_ctsm_py_tests` You can specify various arguments to this; run `./run_ctsm_py_tests - -h` for details + -h` for details. Please specify either --unit or --sys rather than + not including any arguments. In any configuration where you run the system tests, you need to first execute `module load nco`. diff --git a/python/ctsm/run_ctsm_py_tests.py b/python/ctsm/run_ctsm_py_tests.py index 0542dc41cb..8b39d69afa 100644 --- a/python/ctsm/run_ctsm_py_tests.py +++ b/python/ctsm/run_ctsm_py_tests.py @@ -45,7 +45,10 @@ def main(description): def _commandline_args(description): - """Parse and return command-line arguments""" + """Parse and return command-line arguments + Note that run_ctsm_py_tests is not intended to be + used without argument specifications + """ parser = argparse.ArgumentParser( description=description, formatter_class=argparse.RawTextHelpFormatter ) diff --git a/python/ctsm/site_and_regional/neon_surf_wrapper.py b/python/ctsm/site_and_regional/neon_surf_wrapper.py index c2e5ac61b8..a2cc619a29 100755 --- a/python/ctsm/site_and_regional/neon_surf_wrapper.py +++ b/python/ctsm/site_and_regional/neon_surf_wrapper.py @@ -26,12 +26,10 @@ from __future__ import print_function import os -import sys -import tqdm import logging import argparse import subprocess - +import tqdm import pandas as pd @@ -89,13 +87,13 @@ def execute(command): subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - # raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) - # print (e.ouput) print(e) def main(): - + """ + Loop through neon sites and execute subset and modify commands + """ args = get_parser().parse_args() if args.verbose: diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index e8706e4cce..104e325617 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -74,6 +74,8 @@ from ctsm.utils import parse_isoduration from ctsm.download_utils import download_file +from ctsm import add_cime_to_path + from standard_script_setup import * logger = logging.getLogger(__name__) diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 3f2dc6d7f2..b5ded30399 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -57,7 +57,7 @@ def test_modify_site(self): ] # TODO: the above requires a full path instead of site name # because of how run_neon is configured. - # This needs to be fixed/generalized. + # This needs to be fixed in run_neon. parser = get_parser() with self.assertRaises(SystemExit): print( diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index 4dd5aca896..30d996b817 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -13,6 +13,7 @@ from ctsm import unit_testing from ctsm.site_and_regional.run_neon import main, get_parser +from ctsm.path_utils import path_to_ctsm_root # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -44,21 +45,19 @@ def test_one_site(self): # run the run_neon tool sys.argv = [ - "run_neon", + os.path.join( path_to_ctsm_root(), "tools", "site_and_regional", "run_neon" ), "--neon-sites", "BART", "--setup-only", "--output-root", - # "FAKE_DIR" self._tempdir, ] - valid_neon_sites = ["ABBY", "OSBS", "BART"] # ["all"] + valid_neon_sites = ["ABBY", "OSBS", "BART"] parser = get_parser(sys.argv, "description_for_parser", valid_neon_sites) main("") # assert that BART directories were created during setup self.assertTrue("BART" in glob.glob(self._tempdir + "/BART*")[0]) - # self.assertTrue("BART" in glob.glob("BART*")[0]) # TODO: Would also be useful to test the following items: # It might be good to ensure the log files are working as expected? diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index 4c3a473394..98d630447e 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -84,9 +84,10 @@ def test_find_soil_structure(self): Test to ensure that correct attributes are found for find_soil_structure. soil_texture_raw_data_file_name should be found, and test should go through sysexit. """ + surf_file_name = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" surf_file = os.path.join( path_to_ctsm_root(), - "python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc", + "python/ctsm/test/testinputs/", surf_file_name, ) f1 = xr.open_dataset(surf_file) self.assertEqual( @@ -140,7 +141,7 @@ def test_check_neon_time(self): self.assertGreater( int(last_abby_download[:4]), 2021, "ABBY download is older than expected" ) - # change back to previous dir once listing.csv file has been created in tempdir and test complete + # change back to previous dir once listing.csv file is created in tempdir and test complete os.chdir(previous_dir) diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index 2cdcc915ad..a35608e249 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -54,7 +54,7 @@ def test_check_neon_listing(self): self.assertEqual( available_list[1].name, "BART", "available list of actual sites not as expected" ) - # change back to previous dir once listing.csv file has been created in tempdir and test complete + # change to previous dir once listing.csv file is created in tempdir and test complete os.chdir(previous_dir) def test_check_neon_listing_misspelled(self): @@ -68,7 +68,7 @@ def test_check_neon_listing_misspelled(self): self.assertEqual( available_list, [], "available list of incorrect dummy site not as expected" ) - # change back to previous dir once listing.csv file has been created in tempdir and test complete + # change to previous dir once listing.csv file is created in tempdir and test complete os.chdir(previous_dir) From e4d38681df23ccca0ae29581a45f8362574e0630 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 2 Nov 2023 16:18:12 -0600 Subject: [PATCH 050/173] black reformat --- python/ctsm/test/test_sys_run_neon.py | 2 +- python/ctsm/test/test_unit_modify_singlept_site_neon.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_run_neon.py b/python/ctsm/test/test_sys_run_neon.py index 30d996b817..b6814ee2bc 100755 --- a/python/ctsm/test/test_sys_run_neon.py +++ b/python/ctsm/test/test_sys_run_neon.py @@ -45,7 +45,7 @@ def test_one_site(self): # run the run_neon tool sys.argv = [ - os.path.join( path_to_ctsm_root(), "tools", "site_and_regional", "run_neon" ), + os.path.join(path_to_ctsm_root(), "tools", "site_and_regional", "run_neon"), "--neon-sites", "BART", "--setup-only", diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index 98d630447e..ecd96357b3 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -87,7 +87,8 @@ def test_find_soil_structure(self): surf_file_name = "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" surf_file = os.path.join( path_to_ctsm_root(), - "python/ctsm/test/testinputs/", surf_file_name, + "python/ctsm/test/testinputs/", + surf_file_name, ) f1 = xr.open_dataset(surf_file) self.assertEqual( From 239a5524e697e2274f10d206f79c2b91e8d4cdcc Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 3 Nov 2023 10:59:00 -0600 Subject: [PATCH 051/173] Change snow_thermal_cond_method to char*25 --- bld/namelist_files/namelist_definition_ctsm.xml | 2 +- src/biogeophys/SoilTemperatureMod.F90 | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 23f6063a97..58c1980232 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -2835,7 +2835,7 @@ NiuYang2007: Niu and Yang 2007 SwensonLawrence2012: Swenson and Lawrence 2012 - Parameterization to use for snow thermal conductivity diff --git a/src/biogeophys/SoilTemperatureMod.F90 b/src/biogeophys/SoilTemperatureMod.F90 index 69f9958efa..bd7efc0788 100644 --- a/src/biogeophys/SoilTemperatureMod.F90 +++ b/src/biogeophys/SoilTemperatureMod.F90 @@ -624,7 +624,6 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter use landunit_varcon , only : istice, istwet use column_varcon , only : icol_roof, icol_sunwall, icol_shadewall, icol_road_perv, icol_road_imperv use clm_varctl , only : iulog, snow_thermal_cond_method - ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds From 1f5e0428692434b5b3cc9838e992ed3cd0dc5d1c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 3 Nov 2023 11:27:49 -0600 Subject: [PATCH 052/173] ignore black revision --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c520df4357..e15f21d8ca 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -12,6 +12,7 @@ b771971e3299c4fa56534b93421f7a2b9c7282fd 8bc4688e52ea23ef688e283698f70a44388373eb 4ee49e3e516ca7dee5df378f65664f93a7db4415 0207bc98dd5c75cd69a0e788bc53e41093712f5c +e4d38681df23ccca0ae29581a45f8362574e0630 # Ran SystemTests and python/ctsm through black python formatter 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c 8056ae649c1b37f5e10aaaac79005d6e3a8b2380 From 389bca9186bb852b11b18ce9df7736f6cb90272c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 2 Nov 2023 14:52:19 -0600 Subject: [PATCH 053/173] Make output_dir if needed. --- tools/contrib/ssp_anomaly_forcing_smooth | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index a0b6419744..7bfb0bd384 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -120,10 +120,9 @@ if os.path.exists(datapath): print("Input data directory:" + datapath) else: sys.exit("Could not find input directory: " + datapath) -if os.path.exists(args.output_dir): - print("Output data directory:" + args.output_dir) -else: - sys.exit("Could not find output directory: " + args.output_dir) +if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir) +print("Output data directory:" + args.output_dir) # Settings to run with today = datetime.date.today() From 247305996c17dabdf150fd8d771e9108882aeeb6 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 2 Nov 2023 16:27:07 -0600 Subject: [PATCH 054/173] Refactor output file setup. --- tools/contrib/ssp_anomaly_forcing_smooth | 183 +++++++++++++---------- 1 file changed, 104 insertions(+), 79 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 7bfb0bd384..774d4e5b78 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -21,6 +21,104 @@ import numpy as np import netCDF4 as netcdf4 +# Adds global attributes, returning hdir and fdir +def add_global_attributes(ds, historydate, histdir, sspdir, num_ens, climo_year, climo_base_nyrs, dpath, dfile, hist_yrstart, hist_yrend, ssp_yrstart, ssp_yrend, timetag): + ds.Created_on = timetag + + ds.title = "anomaly forcing data" + ds.note1 = ( + "Anomaly/scale factors calculated relative to " + + str(climo_year - (climo_base_nyrs - 1) / 2) + + "-" + + str(climo_year + (climo_base_nyrs - 1) / 2) + ) + ds.history = historydate + ": created by " + sys.argv[0] + stdout = os.popen("git describe") + ds.gitdescribe = stdout.read().rstrip() + ds.Source = "CMIP6 CESM simulations" + ds.Conventions = "CF-1.0" + ds.comment = ( + "Monthly scale factors for given SSP scenario compared to a climatology based on" + + " data centered on " + + str(climo_year) + + " over the range given in note1" + ) + ds.number_of_ensemble_members = str(num_ens) + ds.Created_by = getuser() + + for nens in range(num_ens): + hdir = dpath + histdir[nens] + dfile + fdir = dpath + sspdir[nens] + dfile + if nens == 0: + ds.Created_from_historical_dirs = hdir + ds.Created_from_scenario_dirs = fdir + else: + ds.Created_from_historical_dirs += ", " + hdir + ds.Created_from_scenario_dirs += ", " + fdir + + ds.History_years = str(hist_yrstart) + "," + str(hist_yrend) + ds.Scenario_years = str(ssp_yrstart) + "," + str(ssp_yrend) + ds.institution = "National Center for Atmospheric Research" + return hdir,fdir + + +def create_fill_dims(ds, nlat, nlon, ssp_time_units, ssp_time_longname, lon, lat, ssp_time): + ds.createDimension("lat", size=int(nlat)) + ds.createDimension("lon", size=int(nlon)) + ds.createDimension("time", None) + + wtime = ds.createVariable("time", np.float64, ("time",)) + wlat = ds.createVariable("lat", np.float64, ("lat",)) + wlon = ds.createVariable("lon", np.float64, ("lon",)) + + wtime.units = ssp_time_units + wlon.units = "degrees_east" + wlat.units = "degrees_north" + + wtime.long_name = ssp_time_longname + wlon.long_name = "Longitude" + wlat.long_name = "Latitude" + + wlon.mode = "time-invariant" + wlat.mode = "time-invariant" + + # adjust time to middle of month + wtime_offset = 15 - ssp_time[0] + wtime[:] = ssp_time + wtime_offset + + wtime.calendar = "noleap" + wlon[:] = lon + wlat[:] = lat + + return ds + + +def create_fill_ancillary_vars(ds, landfrac, landmask, area): + + wmask = ds.createVariable("landmask", np.int32, ("lat", "lon")) + warea = ds.createVariable("area", np.float64, ("lat", "lon")) + wfrac = ds.createVariable("landfrac", np.float64, ("lat", "lon")) + + warea.units = "km2" + wfrac.units = "unitless" + wmask.units = "unitless" + + warea.long_name = "Grid cell area" + wfrac.long_name = "Grid cell land fraction" + wmask.long_name = "Grid cell land mask" + + warea.mode = "time-invariant" + wfrac.mode = "time-invariant" + wmask.mode = "time-invariant" + + # write to file -------------------------------------------- + wmask[:, :] = landmask + wfrac[:, :] = landfrac + warea[:, :] = area + + return ds + + parser = argparse.ArgumentParser(description="Create anomaly forcing") parser.add_argument( "sspnum", @@ -560,87 +658,14 @@ for f in range(nfields): x = x2.communicate() timetag = x[0].decode("utf-8").strip() - outfile.Created_on = timetag + # Add global attributes and get hdir/fdir + hdir, fdir = add_global_attributes(outfile, historydate, histdir, sspdir, num_ens, climo_year, climo_base_nyrs, dpath, dfile, hist_yrstart, hist_yrend, ssp_yrstart, ssp_yrend, timetag) - outfile.title = "anomaly forcing data" - outfile.note1 = ( - "Anomaly/scale factors calculated relative to " - + str(climo_year - (climo_base_nyrs - 1) / 2) - + "-" - + str(climo_year + (climo_base_nyrs - 1) / 2) - ) - outfile.history = historydate + ": created by " + sys.argv[0] - stdout = os.popen("git describe") - outfile.gitdescribe = stdout.read().rstrip() - outfile.Source = "CMIP6 CESM simulations" - outfile.Conventions = "CF-1.0" - outfile.comment = ( - "Monthly scale factors for given SSP scenario compared to a climatology based on" - + " data centered on " - + str(climo_year) - + " over the range given in note1" - ) - outfile.number_of_ensemble_members = str(num_ens) - outfile.Created_by = getuser() - - for nens in range(num_ens): - hdir = dpath + histdir[nens] + dfile - fdir = dpath + sspdir[nens] + dfile - if nens == 0: - outfile.Created_from_historical_dirs = hdir - outfile.Created_from_scenario_dirs = fdir - else: - outfile.Created_from_historical_dirs += ", " + hdir - outfile.Created_from_scenario_dirs += ", " + fdir - - outfile.History_years = str(hist_yrstart) + "," + str(hist_yrend) - outfile.Scenario_years = str(ssp_yrstart) + "," + str(ssp_yrend) - outfile.institution = "National Center for Atmospheric Research" - - outfile.createDimension("lat", size=int(nlat)) - outfile.createDimension("lon", size=int(nlon)) - outfile.createDimension("time", None) - - wtime = outfile.createVariable("time", np.float64, ("time",)) - wlat = outfile.createVariable("lat", np.float64, ("lat",)) - wlon = outfile.createVariable("lon", np.float64, ("lon",)) - wmask = outfile.createVariable("landmask", np.int32, ("lat", "lon")) - warea = outfile.createVariable("area", np.float64, ("lat", "lon")) - wfrac = outfile.createVariable("landfrac", np.float64, ("lat", "lon")) - wtime.units = ssp_time_units - wlon.units = "degrees_east" - wlat.units = "degrees_north" - warea.units = "km2" - wfrac.units = "unitless" - wmask.units = "unitless" - - # wtime.long_name = 'Months since January '+str(fut_yrstart) - wtime.long_name = ssp_time_longname - wlon.long_name = "Longitude" - wlat.long_name = "Latitude" - warea.long_name = "Grid cell area" - wfrac.long_name = "Grid cell land fraction" - wmask.long_name = "Grid cell land mask" - wlon.mode = "time-invariant" - wlat.mode = "time-invariant" - warea.mode = "time-invariant" - wfrac.mode = "time-invariant" - wmask.mode = "time-invariant" - - wtime.calendar = "noleap" + # Create dimensions + outfile = create_fill_dims(outfile, nlat, nlon, ssp_time_units, ssp_time_longname, lon, lat, ssp_time) - # write to file -------------------------------------------- - # wtime_offset = 0 - # adjust time to middle of month - # wtime_offset = -15 - wtime_offset = 15 - ssp_time[0] - wtime[:] = ssp_time + wtime_offset - wtime.calendar = "noleap" - wlon[:] = lon - wlat[:] = lat - wmask[:, :] = landmask - wfrac[:, :] = landfrac - warea[:, :] = area + # Create and fill ancillary variables () + outfile = create_fill_ancillary_vars(outfile, landfrac, landmask, area) # -- End if on open file if field_out[f] == "sfcWind": From dda9163d52e7930ce863ff4bfd76bd05741f6048 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 3 Nov 2023 13:59:47 -0600 Subject: [PATCH 055/173] Refactor creation and filling of forcing variables. --- tools/contrib/ssp_anomaly_forcing_smooth | 112 ++++++++++++----------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 774d4e5b78..ae92020981 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -119,6 +119,61 @@ def create_fill_ancillary_vars(ds, landfrac, landmask, area): return ds +def create_fill_forcing(ds, field_out, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld): + if field_out[f] == "sfcWind": + wvar = ds.createVariable( + field_out_wind[0], + np.float64, + ("time", "lat", "lon"), + fill_value=np.float64(1.0e36), + ) + else: + wvar = ds.createVariable( + field_out[f], + np.float64, + ("time", "lat", "lon"), + fill_value=np.float64(1.0e36), + ) + wvar.units = units[f] + wvar.mode = "time-dependent" + if field_out[f] == "sfcWind": + wvar.long_name = str(long_name) + " U component " + anomsf[f] + else: + wvar.long_name = str(long_name) + " " + anomsf[f] + # List of source files + wvar.historical_source_files = "".join(histfiles).replace(hdir, "") + wvar.scenario_source_files = "".join(sspfiles).replace(fdir, "") + + # write to file -------------------------------------------- + if field_out[f] == "sfcWind": + wvar[:, :, :] = anom_fld / np.sqrt(2) + else: + wvar[:, :, :] = anom_fld + + return ds + + +def create_fill_windv(ds, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld): + + wvar = ds.createVariable( + field_out_wind[1], + np.float64, + ("time", "lat", "lon"), + fill_value=np.float64(1.0e36), + ) + wvar.units = units[f] + wvar.cell_methods = "time: mean" + wvar.long_name = str(long_name) + " V component " + anomsf[f] + # List of source files + wvar.historical_source_files = "".join(histfiles).replace(hdir, "") + wvar.scenario_source_files = "".join(sspfiles).replace(fdir, "") + + # write to file -------------------------------------------- + wvar[:, :, :] = anom_fld / np.sqrt(2) + + return ds + + parser = argparse.ArgumentParser(description="Create anomaly forcing") parser.add_argument( "sspnum", @@ -668,63 +723,14 @@ for f in range(nfields): outfile = create_fill_ancillary_vars(outfile, landfrac, landmask, area) # -- End if on open file - if field_out[f] == "sfcWind": - wvar = outfile.createVariable( - field_out_wind[0], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - else: - wvar = outfile.createVariable( - field_out[f], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar.units = units[f] - wvar.mode = "time-dependent" - - # write to file -------------------------------------------- - if field_out[f] == "sfcWind": - wvar.long_name = str(long_name) + " U component " + anomsf[f] - else: - wvar.long_name = str(long_name) + " " + anomsf[f] - - if field_out[f] == "sfcWind": - wvar[:, :, :] = anom_fld / np.sqrt(2) - else: - wvar[:, :, :] = anom_fld + outfile = create_fill_forcing(outfile, field_out, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld) - # List of source files - wvar.historical_source_files = "".join(histfiles).replace(hdir, "") - wvar.scenario_source_files = "".join(sspfiles).replace(fdir, "") - - # create second wind field for V component + # create second wind field for V component if field_out[f] == "sfcWind": - command = 'date "+%y%m%d"' - x2 = subprocess.Popen(command, stdout=subprocess.PIPE, shell="True") - x = x2.communicate() - timetag = x[0].decode("utf-8").strip() - - wvar = outfile.createVariable( - field_out_wind[1], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar.units = units[f] - wvar.cell_methods = "time: mean" - wvar.long_name = str(long_name) + " V component " + anomsf[f] - - # write to file -------------------------------------------- - wvar[:, :, :] = anom_fld / np.sqrt(2) - - # List of source files - wvar.historical_source_files = "".join(histfiles).replace(hdir, "") - wvar.scenario_source_files = "".join(sspfiles).replace(fdir, "") + create_fill_windv(outfile, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld) # -- end if statement for write for V field -------- + break # -- End Loop over forcing fields ------------------------------------ outfile.close() From ca394867da41ed3159eb5e70569062d456969d5c Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 3 Nov 2023 14:37:51 -0600 Subject: [PATCH 056/173] bug fix --- src/cpl/share_esmf/cropcalStreamMod.F90 | 59 ++++++++++++++----------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index 46696eeba9..19fc11d136 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -7,7 +7,8 @@ module cropcalStreamMod ! Read crop calendars from streams ! ! !USES: - use ESMF + use ESMF , only : ESMF_LogFoundError, ESMF_LOGERR_PASSTHRU, ESMF_Finalize + use ESMF , only : ESMF_END_ABORT use shr_kind_mod , only : r8 => shr_kind_r8, CL => shr_kind_CL, CS => shr_kind_CS use dshr_strdata_mod , only : shr_strdata_type use decompMod , only : bounds_type @@ -54,7 +55,7 @@ subroutine cropcal_init(bounds) ! ! Initialize data stream information for crop calendars. ! - ! !USES: + !USES: use shr_mpi_mod , only : shr_mpi_bcast use clm_nlUtilsMod , only : find_nlgroup_name use lnd_comp_shr , only : mesh, model_clock @@ -62,7 +63,7 @@ subroutine cropcal_init(bounds) use controlMod , only : NLFilename ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds ! bounds + type(bounds_type), intent(in) :: bounds ! bounds ! ! !LOCAL VARIABLES: integer :: i,n,ivt ! index @@ -248,12 +249,12 @@ subroutine cropcal_advance( bounds ) ! ! Advance crop calendar streams ! - ! !USES: + !USES: use clm_time_manager , only : get_curr_date use dshr_strdata_mod , only : shr_strdata_advance ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds + type(bounds_type), intent(in) :: bounds ! ! !LOCAL VARIABLES: integer :: g, ig ! Indices @@ -298,22 +299,22 @@ end subroutine cropcal_advance !================================================================ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) - ! - ! Interpolate data stream information for crop calendars. - ! - ! !USES: - use CropType , only : crop_type - use PatchType , only : patch - use clm_time_manager, only : get_curr_days_per_year - use pftconMod , only : pftname - use dshr_methods_mod , only : dshr_fldbun_getfldptr - ! - ! !ARGUMENTS: - implicit none - type(bounds_type) , intent(in) :: bounds - integer , intent(in) :: num_pcropp ! number of prog. crop patches in filter - integer , intent(in) :: filter_pcropp(:) ! filter for prognostic crop patches - type(crop_type) , intent(inout) :: crop_inst + ! + ! Interpolate data stream information for crop calendars. + ! + ! !USES: + use CropType , only : crop_type + use PatchType , only : patch + use clm_time_manager, only : get_curr_days_per_year + use pftconMod , only : pftname + use dshr_methods_mod , only : dshr_fldbun_getfldptr + ! + ! !ARGUMENTS: + implicit none + type(bounds_type) , intent(in) :: bounds + integer , intent(in) :: num_pcropp ! number of prog. crop patches in filter + integer , intent(in) :: filter_pcropp(:) ! filter for prognostic crop patches + type(crop_type) , intent(inout) :: crop_inst ! ! !LOCAL VARIABLES: integer :: ivt, p, ip, ig @@ -322,6 +323,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) integer :: n, g integer :: lsize integer :: rc + integer :: begp, endp real(r8), pointer :: dataptr1d_swindow_start(:) real(r8), pointer :: dataptr1d_swindow_end (:) real(r8), pointer :: dataptr1d_cultivar_gdds(:) @@ -342,6 +344,9 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) ! Place all data from each type into a temporary 2d array lsize = bounds%endg - bounds%begg + 1 + begp = bounds%begp + endp= bounds%endp + dayspyr = get_curr_days_per_year() ! Read prescribed sowing window start dates from input files @@ -397,17 +402,17 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) ! Ensure that, if mxsowings > 1, sowing windows are ordered such that ENDS are monotonically increasing. This is necessary because of how get_swindow() works. if (mxsowings > 1) then - if (any(ends(:,2:mxsowings) <= ends(:,1:mxsowings-1) .and. & - ends(:,2:mxsowings) >= 1)) then + if (any(ends(begp:endp,2:mxsowings) <= ends(begp:endp,1:mxsowings-1) .and. & + ends(begp:endp,2:mxsowings) >= 1)) then write(iulog, *) 'Sowing window inputs must be ordered such that end dates are monotonically increasing.' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if end if ! Handle invalid sowing window values - if (any(starts < 1 .or. ends < 1)) then + if (any(starts(begp:endp,:) < 1 .or. ends(begp:endp,:) < 1)) then ! Fail if not allowing fallback to paramfile sowing windows - if ((.not. allow_invalid_swindow_inputs) .and. any(all(starts < 1, dim=2) .and. patch%wtgcell > 0._r8 .and. patch%itype >= npcropmin)) then + if ((.not. allow_invalid_swindow_inputs) .and. any(all(starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell > 0._r8 .and. patch%itype >= npcropmin)) then write(iulog, *) 'At least one crop in one gridcell has invalid prescribed sowing window start date(s). To ignore and fall back to paramfile sowing windows, set allow_invalid_swindow_inputs to .true.' write(iulog, *) 'Affected crops:' do ivt = npcropmin, mxpft @@ -422,11 +427,11 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Fail if a sowing window start date is prescribed without an end date (or vice versa) - else if (any((starts >= 1 .and. ends < 1) .or. (starts < 1 .and. ends >= 1))) then + else if (any((starts(begp:endp,:) >= 1 .and. ends(begp:endp,:) < 1) .or. (starts(begp:endp,:) < 1 .and. ends(begp:endp,:) >= 1))) then write(iulog, *) 'Every prescribed sowing window start date must have a corresponding end date.' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if - end if + end if end if ! use_cropcal_rx_swindows deallocate(dataptr2d_swindow_start) From 8b55281d1b1ee54df621a40cf749f317c8fcdded Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 3 Nov 2023 14:24:51 -0600 Subject: [PATCH 057/173] Refactor creation and filling of time. --- tools/contrib/ssp_anomaly_forcing_smooth | 83 ++++++++++-------------- 1 file changed, 36 insertions(+), 47 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index ae92020981..f7acd6a13b 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -62,37 +62,48 @@ def add_global_attributes(ds, historydate, histdir, sspdir, num_ens, climo_year, return hdir,fdir -def create_fill_dims(ds, nlat, nlon, ssp_time_units, ssp_time_longname, lon, lat, ssp_time): - ds.createDimension("lat", size=int(nlat)) - ds.createDimension("lon", size=int(nlon)) - ds.createDimension("time", None) - - wtime = ds.createVariable("time", np.float64, ("time",)) +def create_fill_latlon(ds, lon, lat): + ds.createDimension("lat", int(lat.size)) + ds.createDimension("lon", int(lon.size)) wlat = ds.createVariable("lat", np.float64, ("lat",)) wlon = ds.createVariable("lon", np.float64, ("lon",)) - wtime.units = ssp_time_units wlon.units = "degrees_east" wlat.units = "degrees_north" - - wtime.long_name = ssp_time_longname wlon.long_name = "Longitude" wlat.long_name = "Latitude" - wlon.mode = "time-invariant" wlat.mode = "time-invariant" - # adjust time to middle of month - wtime_offset = 15 - ssp_time[0] - wtime[:] = ssp_time + wtime_offset - - wtime.calendar = "noleap" wlon[:] = lon wlat[:] = lat return ds +def create_fill_time(ds, time, ntime, ssp_time_units=None, ssp_time_longname=None, adj_time=False): + if ntime is not None: + ntime = int(ntime) + ds.createDimension("time", ntime) + + wtime = ds.createVariable("time", np.float64, ("time",)) + + if ssp_time_units is not None: + wtime.units = ssp_time_units + if ssp_time_longname is not None: + wtime.long_name = ssp_time_longname + wtime.calendar = "noleap" + + # adjust time to middle of month + if adj_time: + wtime_offset = 15 - time[0] + wtime[:] = time + wtime_offset + else: + wtime[:] = time + + return ds + + def create_fill_ancillary_vars(ds, landfrac, landmask, area): wmask = ds.createVariable("landmask", np.int32, ("lat", "lon")) @@ -392,6 +403,8 @@ nfields = len(field_in) output_format = "NETCDF3_64BIT_DATA" # -- Loop over forcing fields ------------------------------------ + + for f in range(nfields): # -- Loop over ensemble members ------------------------------ @@ -633,28 +646,15 @@ for f in range(nfields): "w", format=output_format, ) - w.createDimension("lat", int(nlat)) - w.createDimension("lon", int(nlon)) - w.createDimension("time", int(nmo)) - - wtime = w.createVariable("time", np.float64, ("time",)) - wlat = w.createVariable("lat", np.float64, ("lat",)) - wlon = w.createVariable("lon", np.float64, ("lon",)) + w = create_fill_latlon(w, lon, lat) + w = create_fill_time(w, time[0:12], nmo) + wvar = w.createVariable( field_out[f], np.float64, ("time", "lat", "lon"), fill_value=np.float64(1.0e36), ) - wtime[ - :, - ] = time[0:12] - wlon[ - :, - ] = lon - wlat[ - :, - ] = lat wvar[:, :, :] = climo w.close() @@ -664,13 +664,9 @@ for f in range(nfields): "w", format=output_format, ) - w.createDimension("lat", int(nlat)) - w.createDimension("lon", int(nlon)) - w.createDimension("time", int(tm)) - - wtime = w.createVariable("time", np.float64, ("time",)) - wlat = w.createVariable("lat", np.float64, ("lat",)) - wlon = w.createVariable("lon", np.float64, ("lon",)) + w = create_fill_latlon(w, lon, lat) + w = create_fill_time(w, time, tm) + wvar = w.createVariable( field_out[f], np.float64, @@ -683,14 +679,6 @@ for f in range(nfields): ("time", "lat", "lon"), fill_value=np.float64(1.0e36), ) - - wtime[:] = time - wlon[ - :, - ] = lon - wlat[ - :, - ] = lat wvar[:, :, :] = temp_fld wvar2[:, :, :] = stemp_fld w.close() @@ -717,7 +705,8 @@ for f in range(nfields): hdir, fdir = add_global_attributes(outfile, historydate, histdir, sspdir, num_ens, climo_year, climo_base_nyrs, dpath, dfile, hist_yrstart, hist_yrend, ssp_yrstart, ssp_yrend, timetag) # Create dimensions - outfile = create_fill_dims(outfile, nlat, nlon, ssp_time_units, ssp_time_longname, lon, lat, ssp_time) + outfile = create_fill_latlon(outfile, lon, lat) + outfile = create_fill_time(outfile, ssp_time, None, ssp_time_units=ssp_time_units, ssp_time_longname=ssp_time_longname, adj_time=True) # Create and fill ancillary variables () outfile = create_fill_ancillary_vars(outfile, landfrac, landmask, area) From 8c994b32a4d8fce59c7cfa95bb1078dabbd75133 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 3 Nov 2023 15:11:00 -0600 Subject: [PATCH 058/173] Allow alternate flags with hyphens instead of underscores. --- tools/contrib/ssp_anomaly_forcing_smooth | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index f7acd6a13b..647f96bf4f 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -195,12 +195,14 @@ parser.add_argument( ) parser.add_argument( "--write_climo", + "--write-climo", help="write out climatology files and exit", action="store_true", default=False, ) parser.add_argument( "--print_ssps", + "--print-ssps", help="Just print out directory names and exit", action="store_true", default=False, From 904266ad28a957ef93a829f0e195f949f1f67ea5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 3 Nov 2023 15:26:19 -0600 Subject: [PATCH 059/173] Create and fill lat/lon in separate calls of function. --- tools/contrib/ssp_anomaly_forcing_smooth | 34 +++++++++++++----------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 647f96bf4f..fadebbf89a 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -62,22 +62,21 @@ def add_global_attributes(ds, historydate, histdir, sspdir, num_ens, climo_year, return hdir,fdir -def create_fill_latlon(ds, lon, lat): - ds.createDimension("lat", int(lat.size)) - ds.createDimension("lon", int(lon.size)) - wlat = ds.createVariable("lat", np.float64, ("lat",)) - wlon = ds.createVariable("lon", np.float64, ("lon",)) +def create_fill_latlon(ds, data, var_name): - wlon.units = "degrees_east" - wlat.units = "degrees_north" - wlon.long_name = "Longitude" - wlat.long_name = "Latitude" - wlon.mode = "time-invariant" - wlat.mode = "time-invariant" + ds.createDimension(var_name, int(data.size)) + wl = ds.createVariable(var_name, np.float64, (var_name,)) - wlon[:] = lon - wlat[:] = lat + if var_name == "lat": + wl.units = "degrees_north" + wl.long_name = "Latitude" + elif var_name == "lon": + wl.units = "degrees_east" + wl.long_name = "Longitude" + wl.mode = "time-invariant" + wl[:] = data + return ds @@ -648,7 +647,8 @@ for f in range(nfields): "w", format=output_format, ) - w = create_fill_latlon(w, lon, lat) + w = create_fill_latlon(w, lat, "lat") + w = create_fill_latlon(w, lon, "lon") w = create_fill_time(w, time[0:12], nmo) wvar = w.createVariable( @@ -666,7 +666,8 @@ for f in range(nfields): "w", format=output_format, ) - w = create_fill_latlon(w, lon, lat) + w = create_fill_latlon(w, lat, "lat") + w = create_fill_latlon(w, lon, "lon") w = create_fill_time(w, time, tm) wvar = w.createVariable( @@ -707,7 +708,8 @@ for f in range(nfields): hdir, fdir = add_global_attributes(outfile, historydate, histdir, sspdir, num_ens, climo_year, climo_base_nyrs, dpath, dfile, hist_yrstart, hist_yrend, ssp_yrstart, ssp_yrend, timetag) # Create dimensions - outfile = create_fill_latlon(outfile, lon, lat) + outfile = create_fill_latlon(outfile, lat, "lat") + outfile = create_fill_latlon(outfile, lon, "lon") outfile = create_fill_time(outfile, ssp_time, None, ssp_time_units=ssp_time_units, ssp_time_longname=ssp_time_longname, adj_time=True) # Create and fill ancillary variables () From b626e88a2e930e61e440783cbbadaccae06ed81a Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 3 Nov 2023 15:36:01 -0600 Subject: [PATCH 060/173] Small corrections for tests to run --- bld/CLMBuildNamelist.pm | 2 +- src/biogeophys/SoilTemperatureMod.F90 | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 9a039b1cfc..116a585270 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -2034,7 +2034,7 @@ sub setup_logic_snow { add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'snow_thermal_cond_method' ); my $var = $nl->get_value('snow_thermal_cond_method'); - if ( $var ne 'Jordan1991' && $var ne 'Sturm1997' ) { + if ( $var ne "'Jordan1991'" && $var ne "'Sturm1997'" ) { $log->fatal_error("$var is incorrect entry for the namelist variable snow_thermal_cond_method; expected Jordan1991 or Sturm1997"); } diff --git a/src/biogeophys/SoilTemperatureMod.F90 b/src/biogeophys/SoilTemperatureMod.F90 index bd7efc0788..0dc8876d24 100644 --- a/src/biogeophys/SoilTemperatureMod.F90 +++ b/src/biogeophys/SoilTemperatureMod.F90 @@ -619,6 +619,7 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter ! flux from the interface to the node j+1. ! ! !USES: + use shr_log_mod , only : errMsg => shr_log_errMsg use clm_varpar , only : nlevsno, nlevgrnd, nlevurb, nlevsoi, nlevmaxurbgrnd use clm_varcon , only : denh2o, denice, tfrz, tkwat, tkice, tkair, cpice, cpliq, thk_bedrock, csol_bedrock use landunit_varcon , only : istice, istwet @@ -648,6 +649,8 @@ subroutine SoilThermProp (bounds, num_urbanc, filter_urbanc, num_nolakec, filter real(r8) :: fl ! volume fraction of liquid or unfrozen water to total water real(r8) :: satw ! relative total water content of soil. real(r8) :: zh2osfc + + character(len=*),parameter :: subname = 'SoilThermProp' !----------------------------------------------------------------------- call t_startf( 'SoilThermProp' ) From 2b45ef4bd559a2d76ab15ff66cbbfc9021523a3f Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sun, 5 Nov 2023 20:57:24 -0700 Subject: [PATCH 061/173] Refactor add_to_dataset() out of create_fill_forcing(). --- tools/contrib/ssp_anomaly_forcing_smooth | 128 +++++++++-------------- 1 file changed, 52 insertions(+), 76 deletions(-) diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index fadebbf89a..362e47c67d 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -129,58 +129,58 @@ def create_fill_ancillary_vars(ds, landfrac, landmask, area): return ds -def create_fill_forcing(ds, field_out, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld): - if field_out[f] == "sfcWind": - wvar = ds.createVariable( - field_out_wind[0], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - else: - wvar = ds.createVariable( - field_out[f], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar.units = units[f] - wvar.mode = "time-dependent" - if field_out[f] == "sfcWind": - wvar.long_name = str(long_name) + " U component " + anomsf[f] - else: - wvar.long_name = str(long_name) + " " + anomsf[f] - # List of source files - wvar.historical_source_files = "".join(histfiles).replace(hdir, "") - wvar.scenario_source_files = "".join(sspfiles).replace(fdir, "") - - # write to file -------------------------------------------- - if field_out[f] == "sfcWind": - wvar[:, :, :] = anom_fld / np.sqrt(2) - else: - wvar[:, :, :] = anom_fld +def add_to_dataset(ds, var_name, data, units=None, mode=None, historical_source_files=None, scenario_source_files=None, long_name=None, cell_methods=None): + dims = ("time", "lat", "lon") + data_type = np.float64 + + wvar = ds.createVariable( + var_name, + data_type, + dims, + fill_value=data_type(1.0e36), + ) + + wvar[:, :, :] = data + + if units is not None: + wvar.units = units + if mode is not None: + wvar.mode = mode + if historical_source_files is not None: + wvar.historical_source_files = historical_source_files + if scenario_source_files is not None: + wvar.scenario_source_files = scenario_source_files + if long_name is not None: + wvar.long_name = long_name + if cell_methods is not None: + wvar.cell_methods = cell_methods return ds -def create_fill_windv(ds, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld): - - wvar = ds.createVariable( - field_out_wind[1], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar.units = units[f] - wvar.cell_methods = "time: mean" - wvar.long_name = str(long_name) + " V component " + anomsf[f] - # List of source files - wvar.historical_source_files = "".join(histfiles).replace(hdir, "") - wvar.scenario_source_files = "".join(sspfiles).replace(fdir, "") - - # write to file -------------------------------------------- - wvar[:, :, :] = anom_fld / np.sqrt(2) +def create_fill_forcing(ds, field_out, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld): + + historical_source_files = "".join(histfiles).replace(hdir, "") + scenario_source_files = "".join(sspfiles).replace(fdir, "") + mode = "time-dependent" + + if field_out[f] == "sfcWind": + long_name = str(long_name) + " U component " + anomsf[f] + var_name = field_out_wind[0] + data = anom_fld / np.sqrt(2) + else: + long_name = str(long_name) + " " + anomsf[f] + var_name = field_out[f] + data = anom_fld + # Was missing cell_methods attribute in original + ds = add_to_dataset(ds, var_name, data, units=units[f], mode=mode, historical_source_files=historical_source_files, scenario_source_files=scenario_source_files, long_name=long_name) + if field_out[f] == "sfcWind": + long_name = long_name.replace("U component", "V component") + var_name = field_out_wind[1] + # Was missing mode attribute in original + ds = add_to_dataset(ds, var_name, data, units=units[f], historical_source_files=historical_source_files, scenario_source_files=scenario_source_files, long_name=long_name, cell_methods="time: mean") + return ds @@ -651,13 +651,7 @@ for f in range(nfields): w = create_fill_latlon(w, lon, "lon") w = create_fill_time(w, time[0:12], nmo) - wvar = w.createVariable( - field_out[f], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar[:, :, :] = climo + add_to_dataset(w, field_out[f], climo) w.close() # Use NetCDF4 format, because using older NetCDF formats are too slow @@ -670,21 +664,10 @@ for f in range(nfields): w = create_fill_latlon(w, lon, "lon") w = create_fill_time(w, time, tm) - wvar = w.createVariable( - field_out[f], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar2 = w.createVariable( - "smooth_" + field_out[f], - np.float64, - ("time", "lat", "lon"), - fill_value=np.float64(1.0e36), - ) - wvar[:, :, :] = temp_fld - wvar2[:, :, :] = stemp_fld + add_to_dataset(w, field_out[f], temp_fld) + add_to_dataset(w, "smooth_" + field_out[f], stemp_fld) w.close() + print("Exit early after writing out climatology\n\n") sys.exit() @@ -712,19 +695,12 @@ for f in range(nfields): outfile = create_fill_latlon(outfile, lon, "lon") outfile = create_fill_time(outfile, ssp_time, None, ssp_time_units=ssp_time_units, ssp_time_longname=ssp_time_longname, adj_time=True) - # Create and fill ancillary variables () + # Create and fill ancillary variables outfile = create_fill_ancillary_vars(outfile, landfrac, landmask, area) # -- End if on open file outfile = create_fill_forcing(outfile, field_out, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld) - # create second wind field for V component - if field_out[f] == "sfcWind": - create_fill_windv(outfile, units, anomsf, field_out_wind, f, hdir, fdir, histfiles, sspfiles, long_name, anom_fld) - - # -- end if statement for write for V field -------- - break - # -- End Loop over forcing fields ------------------------------------ outfile.close() From 61d746faf51a855dbd7ef46d41a4f85d4757d892 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 6 Nov 2023 11:04:48 -0700 Subject: [PATCH 062/173] Move comments and add whitespace. --- src/biogeochem/CNVegCarbonFluxType.F90 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index c7aa3469e2..1c5fb60d31 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -3729,8 +3729,8 @@ subroutine RestartBulkOnly ( this, bounds, ncid, flag ) ! BACKWARDS_COMPATIBILITY(wjs/ssr, 2022-06-10) See note in CallRestartvarDimOK() if (CallRestartvarDimOK(ncid, flag, 'mxharvests')) then do k = repr_grain_min, repr_grain_max - data2dptr => this%repr_grainc_to_food_perharv_patch(:,:,k) ! e.g., grainc_to_food_perharv + data2dptr => this%repr_grainc_to_food_perharv_patch(:,:,k) varname = get_repr_rest_fname(k)//'c_to_food_perharv' call restartvar(ncid=ncid, flag=flag, varname=varname, & xtype=ncd_double, & @@ -3746,8 +3746,8 @@ subroutine RestartBulkOnly ( this, bounds, ncid, flag ) end if do k = repr_grain_min, repr_grain_max - data1dptr => this%repr_grainc_to_food_thisyr_patch(:,k) ! e.g., grainc_to_food_thisyr + data1dptr => this%repr_grainc_to_food_thisyr_patch(:,k) varname = get_repr_rest_fname(k)//'c_to_food_thisyr' call restartvar(ncid=ncid, flag=flag, varname=varname, & xtype=ncd_double, & @@ -3755,8 +3755,9 @@ subroutine RestartBulkOnly ( this, bounds, ncid, flag ) long_name=get_repr_longname(k)//' C to food per calendar year; should only be output annually', & units='gC/m2', & interpinic_flag='interp', readvar=readvar, data=data1dptr) - data1dptr => this%repr_grainc_to_seed_thisyr_patch(:,k) + ! e.g., grainc_to_seed_thisyr + data1dptr => this%repr_grainc_to_seed_thisyr_patch(:,k) varname = get_repr_rest_fname(k)//'c_to_seed_thisyr' call restartvar(ncid=ncid, flag=flag, varname=varname, & xtype=ncd_double, & From 32f2907e6de8d4870623b2e75235420802f67688 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 6 Nov 2023 11:08:29 -0700 Subject: [PATCH 063/173] Save repr_grainc_to_seed_perharv_patch for restart. --- src/biogeochem/CNVegCarbonFluxType.F90 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index 1c5fb60d31..298e7b3053 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -3742,6 +3742,20 @@ subroutine RestartBulkOnly ( this, bounds, ncid, flag ) readvar=readvar, & scale_by_thickness=.false., & interpinic_flag='interp', data=data2dptr) + + ! e.g., grainc_to_seed_perharv + data2dptr => this%repr_grainc_to_seed_perharv_patch(:,:,k) + varname = get_repr_rest_fname(k)//'c_to_seed_perharv' + call restartvar(ncid=ncid, flag=flag, varname=varname, & + xtype=ncd_double, & + dim1name='pft', & + dim2name='mxharvests', & + switchdim=.true., & + long_name=get_repr_longname(k)//' C to seed per harvest; should only be output annually', & + units='gC/m2', & + readvar=readvar, & + scale_by_thickness=.false., & + interpinic_flag='interp', data=data2dptr) end do end if From cc43a1a73bf8dd12995bfdd28f137ac09e9680d4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 6 Nov 2023 11:12:01 -0700 Subject: [PATCH 064/173] Save sowing window dates for restart. --- src/biogeochem/CropType.F90 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 29c4717ab3..e8de7059c0 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -655,6 +655,16 @@ subroutine Restart(this, bounds, ncid, cnveg_state_inst, flag) long_name='crop sowing dates for this patch this year', units='day of year', & scale_by_thickness=.false., & interpinic_flag='interp', readvar=readvar, data=this%sdates_thisyr_patch) + call restartvar(ncid=ncid, flag=flag, varname='swindow_starts_thisyr_patch', xtype=ncd_double, & + dim1name='pft', dim2name='mxsowings', switchdim=.true., & + long_name='sowing window start dates for this patch this year', units='day of year', & + scale_by_thickness=.false., & + interpinic_flag='interp', readvar=readvar, data=this%swindow_starts_thisyr_patch) + call restartvar(ncid=ncid, flag=flag, varname='swindow_ends_thisyr_patch', xtype=ncd_double, & + dim1name='pft', dim2name='mxsowings', switchdim=.true., & + long_name='sowing window end dates for this patch this year', units='day of year', & + scale_by_thickness=.false., & + interpinic_flag='interp', readvar=readvar, data=this%swindow_ends_thisyr_patch) ! Fill variable(s) derived from read-in variable(s) if (flag == 'read' .and. readvar) then do p = bounds%begp,bounds%endp From 82ac5d64cab397043bc4dd320b7d00c4e4cdb263 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 6 Nov 2023 14:16:40 -0700 Subject: [PATCH 065/173] remove extraneous ws --- src/cpl/share_esmf/cropcalStreamMod.F90 | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index 19fc11d136..0ea63f2c6d 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -55,7 +55,7 @@ subroutine cropcal_init(bounds) ! ! Initialize data stream information for crop calendars. ! - !USES: + ! !USES: use shr_mpi_mod , only : shr_mpi_bcast use clm_nlUtilsMod , only : find_nlgroup_name use lnd_comp_shr , only : mesh, model_clock @@ -63,7 +63,7 @@ subroutine cropcal_init(bounds) use controlMod , only : NLFilename ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds ! bounds + type(bounds_type), intent(in) :: bounds ! bounds ! ! !LOCAL VARIABLES: integer :: i,n,ivt ! index @@ -249,12 +249,12 @@ subroutine cropcal_advance( bounds ) ! ! Advance crop calendar streams ! - !USES: + ! !USES: use clm_time_manager , only : get_curr_date use dshr_strdata_mod , only : shr_strdata_advance ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds + type(bounds_type), intent(in) :: bounds ! ! !LOCAL VARIABLES: integer :: g, ig ! Indices @@ -299,22 +299,22 @@ end subroutine cropcal_advance !================================================================ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) - ! - ! Interpolate data stream information for crop calendars. - ! - ! !USES: - use CropType , only : crop_type - use PatchType , only : patch - use clm_time_manager, only : get_curr_days_per_year - use pftconMod , only : pftname - use dshr_methods_mod , only : dshr_fldbun_getfldptr - ! - ! !ARGUMENTS: - implicit none - type(bounds_type) , intent(in) :: bounds - integer , intent(in) :: num_pcropp ! number of prog. crop patches in filter - integer , intent(in) :: filter_pcropp(:) ! filter for prognostic crop patches - type(crop_type) , intent(inout) :: crop_inst + ! + ! Interpolate data stream information for crop calendars. + ! + ! !USES: + use CropType , only : crop_type + use PatchType , only : patch + use clm_time_manager, only : get_curr_days_per_year + use pftconMod , only : pftname + use dshr_methods_mod , only : dshr_fldbun_getfldptr + ! + ! !ARGUMENTS: + implicit none + type(bounds_type) , intent(in) :: bounds + integer , intent(in) :: num_pcropp ! number of prog. crop patches in filter + integer , intent(in) :: filter_pcropp(:) ! filter for prognostic crop patches + type(crop_type) , intent(inout) :: crop_inst ! ! !LOCAL VARIABLES: integer :: ivt, p, ip, ig @@ -410,7 +410,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) end if ! Handle invalid sowing window values - if (any(starts(begp:endp,:) < 1 .or. ends(begp:endp,:) < 1)) then + if (any(starts(begp:endp,:) < 1 .or. ends(begp:endp,:) < 1)) then ! Fail if not allowing fallback to paramfile sowing windows if ((.not. allow_invalid_swindow_inputs) .and. any(all(starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell > 0._r8 .and. patch%itype >= npcropmin)) then write(iulog, *) 'At least one crop in one gridcell has invalid prescribed sowing window start date(s). To ignore and fall back to paramfile sowing windows, set allow_invalid_swindow_inputs to .true.' @@ -431,7 +431,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, crop_inst) write(iulog, *) 'Every prescribed sowing window start date must have a corresponding end date.' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if - end if + end if end if ! use_cropcal_rx_swindows deallocate(dataptr2d_swindow_start) From 2a6b4acf60717f805ed8ed887112b8d3c5edb63a Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:03:50 -0700 Subject: [PATCH 066/173] Update ChangeSum --- doc/ChangeSum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeSum b/doc/ChangeSum index b5d5170267..66d064f72e 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev151 tking 11/02/2023 Move tools to python directory and add tests + ctsm5.1.dev153 tking 11/02/2023 Move tools to python directory and add tests ctsm5.1.dev140 afoster 09/12/2023 add lai_streams capability for FATES ctsm5.1.dev139 slevis 08/28/2023 Fix problems uncovered by nag -nan tests ctsm5.1.dev138 slevis 08/25/2023 Refactor max_patch_per_col and maxsoil_patches loops From eaf488d3bd882f2ed42e14243dd710752df9beb4 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:04:14 -0700 Subject: [PATCH 067/173] Update ChangeLog --- doc/ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index d787c8d36c..40fd18a89c 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,5 @@ =============================================================== -Tag name: ctsm5.1.dev151 +Tag name: ctsm5.1.dev153 Originator(s): tking (Teagan King,UCAR/CCR) Date: Thu Nov 2 16:03:44 MDT 2023 One-line Summary: Move tools to python directory and add tests From 80d47d0c2d15dd0ec7e4b5be0722b0e9866184ce Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 8 Nov 2023 11:58:30 -0700 Subject: [PATCH 068/173] Functionize and unit-test latbaset calculation. --- src/biogeochem/CropType.F90 | 25 ++-- src/biogeochem/test/CMakeLists.txt | 1 + .../test/Latbaset_test/CMakeLists.txt | 6 + .../test/Latbaset_test/test_Latbaset.pf | 112 ++++++++++++++++++ 4 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 src/biogeochem/test/Latbaset_test/CMakeLists.txt create mode 100644 src/biogeochem/test/Latbaset_test/test_Latbaset.pf diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 29c4717ab3..7e4dec6c1f 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -23,6 +23,7 @@ module CropType private ! ! !PUBLIC DATA TYPES: + public :: latbaset ! ! Possible values of cphase @@ -383,13 +384,7 @@ subroutine InitCold(this, bounds) this%nyrs_crop_active_patch(p) = 0 - if ( grc%latdeg(g) >= 0.0_r8 .and. grc%latdeg(g) <= 30.0_r8) then - this%latbaset_patch(p)=pftcon%baset(ivt)+12._r8-0.4_r8*grc%latdeg(g) - else if (grc%latdeg(g) < 0.0_r8 .and. grc%latdeg(g) >= -30.0_r8) then - this%latbaset_patch(p)=pftcon%baset(ivt)+12._r8+0.4_r8*grc%latdeg(g) - else - this%latbaset_patch(p)=pftcon%baset(ivt) - end if + this%latbaset_patch(p) = latbaset(pftcon%baset(ivt), grc%latdeg(g), this%baset_latvary_intercept, this%baset_latvary_slope) if ( trim(this%baset_mapping) == baset_map_constant ) then this%latbaset_patch(p) = nan end if @@ -972,4 +967,20 @@ subroutine checkDates( ) end subroutine checkDates + real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope) + ! !ARGUMENTS: + real(r8), intent(in) :: baset + real(r8), intent(in) :: latdeg + real(r8), intent(in) :: baset_latvary_intercept + real(r8), intent(in) :: baset_latvary_slope + + if ( latdeg >= 0.0_r8 .and. latdeg <= 30.0_r8) then + latbaset = baset + 12._r8 - 0.4_r8*latdeg + else if (latdeg < 0.0_r8 .and. latdeg >= -30.0_r8) then + latbaset = baset + 12._r8 + 0.4_r8*latdeg + else + latbaset = baset + end if + end function latbaset + end module CropType diff --git a/src/biogeochem/test/CMakeLists.txt b/src/biogeochem/test/CMakeLists.txt index ad91c7c995..81fe9bbaf0 100644 --- a/src/biogeochem/test/CMakeLists.txt +++ b/src/biogeochem/test/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(Species_test) add_subdirectory(CNVegComputeSeed_test) add_subdirectory(CNPhenology_test) +add_subdirectory(Latbaset_test) diff --git a/src/biogeochem/test/Latbaset_test/CMakeLists.txt b/src/biogeochem/test/Latbaset_test/CMakeLists.txt new file mode 100644 index 0000000000..217fc7233c --- /dev/null +++ b/src/biogeochem/test/Latbaset_test/CMakeLists.txt @@ -0,0 +1,6 @@ +set (pfunit_sources + test_Latbaset.pf) + +add_pfunit_ctest(CropTypeLatbaset + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeochem/test/Latbaset_test/test_Latbaset.pf b/src/biogeochem/test/Latbaset_test/test_Latbaset.pf new file mode 100644 index 0000000000..1b164d5d69 --- /dev/null +++ b/src/biogeochem/test/Latbaset_test/test_Latbaset.pf @@ -0,0 +1,112 @@ +module test_Latbaset + + ! Tests of CropType module: latbaset + + use funit + use shr_kind_mod , only : r8 => shr_kind_r8 + use unittestSubgridMod + use unittestSimpleSubgridSetupsMod + use unittestFilterBuilderMod + use CropType, only : latbaset + + implicit none + + @TestCase + type, extends(TestCase) :: TestLatbaset + contains + procedure :: setUp + procedure :: tearDown + end type TestLatbaset + + real(r8) :: baset + real(r8) :: latdeg + real(r8) :: baset_latvary_intercept + real(r8) :: baset_latvary_slope + real(r8) :: expected + +contains + + subroutine setUp(this) + class(TestLatbaset), intent(inout) :: this + end subroutine setUp + + subroutine tearDown(this) + class(TestLatbaset), intent(inout) :: this + + call unittest_subgrid_teardown() + end subroutine tearDown + + @Test + subroutine too_far_north(this) + class(TestLatbaset), intent(inout) :: this + + baset = 5._r8 + latdeg = 80._r8 + baset_latvary_intercept = 8.7_r8 + baset_latvary_slope = 0.5_r8 + + @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) + end subroutine too_far_north + + @Test + subroutine too_far_south(this) + class(TestLatbaset), intent(inout) :: this + + baset = 5._r8 + latdeg = -80._r8 + baset_latvary_intercept = 8.7_r8 + baset_latvary_slope = 0.5_r8 + + @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) + end subroutine too_far_south + + @Test + subroutine at_northern_limit(this) + class(TestLatbaset), intent(inout) :: this + + baset = 5._r8 + latdeg = 30._r8 + baset_latvary_intercept = 12._r8 + baset_latvary_slope = 0.4_r8 + + @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) + end subroutine at_northern_limit + + @Test + subroutine at_southern_limit(this) + class(TestLatbaset), intent(inout) :: this + + baset = 5._r8 + latdeg = -30._r8 + baset_latvary_intercept = 12._r8 + baset_latvary_slope = 0.4_r8 + + @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) + end subroutine at_southern_limit + + @Test + subroutine in_nh(this) + class(TestLatbaset), intent(inout) :: this + + baset = 5._r8 + latdeg = 10._r8 + baset_latvary_intercept = 13._r8 + baset_latvary_slope = 0.3_r8 + + @assertEqual(15._r8, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) + end subroutine in_nh + + @Test + subroutine in_sh(this) + class(TestLatbaset), intent(inout) :: this + + baset = 5._r8 + latdeg = -10._r8 + baset_latvary_intercept = 13._r8 + baset_latvary_slope = 0.3_r8 + + @assertEqual(15._r8, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) + end subroutine in_sh + +end module test_Latbaset + From 151cf482e3a31f548916c4fd794825aec0f3cc84 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 8 Nov 2023 12:21:46 -0700 Subject: [PATCH 069/173] latbaset() now uses baset_latvary_intercept and _slope. --- src/biogeochem/CropType.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 7e4dec6c1f..b3e09b3adf 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -975,9 +975,9 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary real(r8), intent(in) :: baset_latvary_slope if ( latdeg >= 0.0_r8 .and. latdeg <= 30.0_r8) then - latbaset = baset + 12._r8 - 0.4_r8*latdeg + latbaset = baset + baset_latvary_intercept - baset_latvary_slope*latdeg else if (latdeg < 0.0_r8 .and. latdeg >= -30.0_r8) then - latbaset = baset + 12._r8 + 0.4_r8*latdeg + latbaset = baset + baset_latvary_intercept + baset_latvary_slope*latdeg else latbaset = baset end if From 054f2a6f3d81f008ac21a5a885a708494862d2f9 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 8 Nov 2023 12:27:44 -0700 Subject: [PATCH 070/173] Simplify latbaset(). --- src/biogeochem/CropType.F90 | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index b3e09b3adf..8068fed4d9 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -973,14 +973,18 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary real(r8), intent(in) :: latdeg real(r8), intent(in) :: baset_latvary_intercept real(r8), intent(in) :: baset_latvary_slope + ! + ! !LOCAL VARIABLES + real(r8) :: maxlat ! Outside latitude range defined by ±maxlat, use baset - if ( latdeg >= 0.0_r8 .and. latdeg <= 30.0_r8) then - latbaset = baset + baset_latvary_intercept - baset_latvary_slope*latdeg - else if (latdeg < 0.0_r8 .and. latdeg >= -30.0_r8) then - latbaset = baset + baset_latvary_intercept + baset_latvary_slope*latdeg - else + maxlat = 30._r8 + + if (abs(latdeg) > maxlat) then latbaset = baset + else + latbaset = baset + baset_latvary_intercept - baset_latvary_slope*abs(latdeg) end if + end function latbaset end module CropType From f2d30ed3da8129921be56e7897d39acbb57be5b9 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 8 Nov 2023 13:23:06 -0700 Subject: [PATCH 071/173] Do not hard-code maxlat. --- src/biogeochem/CropType.F90 | 10 +++++++++- src/biogeochem/test/Latbaset_test/test_Latbaset.pf | 10 +++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 8068fed4d9..7e80d4a9ea 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -24,6 +24,7 @@ module CropType ! ! !PUBLIC DATA TYPES: public :: latbaset + public :: latbaset_max_lat ! ! Possible values of cphase @@ -977,7 +978,7 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary ! !LOCAL VARIABLES real(r8) :: maxlat ! Outside latitude range defined by ±maxlat, use baset - maxlat = 30._r8 + maxlat = latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) if (abs(latdeg) > maxlat) then latbaset = baset @@ -987,4 +988,11 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary end function latbaset + real(r8) function latbaset_max_lat(intercept, slope) + real(r8), intent(in) :: intercept + real(r8), intent(in) :: slope + + latbaset_max_lat = intercept / slope + end function latbaset_max_lat + end module CropType diff --git a/src/biogeochem/test/Latbaset_test/test_Latbaset.pf b/src/biogeochem/test/Latbaset_test/test_Latbaset.pf index 1b164d5d69..8fcc4de1bb 100644 --- a/src/biogeochem/test/Latbaset_test/test_Latbaset.pf +++ b/src/biogeochem/test/Latbaset_test/test_Latbaset.pf @@ -7,7 +7,7 @@ module test_Latbaset use unittestSubgridMod use unittestSimpleSubgridSetupsMod use unittestFilterBuilderMod - use CropType, only : latbaset + use CropType, only : latbaset, latbaset_max_lat implicit none @@ -41,9 +41,9 @@ contains class(TestLatbaset), intent(inout) :: this baset = 5._r8 - latdeg = 80._r8 baset_latvary_intercept = 8.7_r8 baset_latvary_slope = 0.5_r8 + latdeg = 10._r8 + latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) end subroutine too_far_north @@ -53,9 +53,9 @@ contains class(TestLatbaset), intent(inout) :: this baset = 5._r8 - latdeg = -80._r8 baset_latvary_intercept = 8.7_r8 baset_latvary_slope = 0.5_r8 + latdeg = -10._r8 - latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) end subroutine too_far_south @@ -65,9 +65,9 @@ contains class(TestLatbaset), intent(inout) :: this baset = 5._r8 - latdeg = 30._r8 baset_latvary_intercept = 12._r8 baset_latvary_slope = 0.4_r8 + latdeg = latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) end subroutine at_northern_limit @@ -77,9 +77,9 @@ contains class(TestLatbaset), intent(inout) :: this baset = 5._r8 - latdeg = -30._r8 baset_latvary_intercept = 12._r8 baset_latvary_slope = 0.4_r8 + latdeg = -latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) @assertEqual(baset, latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary_slope)) end subroutine at_southern_limit From aa61010e2700854ab01feec060870c5358f26e45 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 8 Nov 2023 13:40:47 -0700 Subject: [PATCH 072/173] Convert latbaset() to a one-liner. --- src/biogeochem/CropType.F90 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 7e80d4a9ea..11972d3286 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -974,17 +974,17 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary real(r8), intent(in) :: latdeg real(r8), intent(in) :: baset_latvary_intercept real(r8), intent(in) :: baset_latvary_slope - ! - ! !LOCAL VARIABLES - real(r8) :: maxlat ! Outside latitude range defined by ±maxlat, use baset - maxlat = latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) + ! Was originally + ! maxlat = latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) + ! if (abs(latdeg) > maxlat) then + ! latbaset = baset + ! else + ! latbaset = baset + baset_latvary_intercept - baset_latvary_slope*abs(latdeg) + ! end if + ! But the one-liner below should improve efficiency, at least marginally. - if (abs(latdeg) > maxlat) then - latbaset = baset - else - latbaset = baset + baset_latvary_intercept - baset_latvary_slope*abs(latdeg) - end if + latbaset = baset + baset_latvary_intercept - min(baset_latvary_intercept, baset_latvary_slope * abs(latdeg)) end function latbaset From 35478a38d5e5448444cf7486d3c82cdf2c769240 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 9 Nov 2023 13:23:37 -0700 Subject: [PATCH 073/173] Improve CropType%InitCold() efficiency. --- src/biogeochem/CropType.F90 | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 11972d3286..03e25135e6 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -373,37 +373,32 @@ subroutine InitCold(this, bounds) type(bounds_type), intent(in) :: bounds ! ! !LOCAL VARIABLES: - integer :: c, l, g, p, m, ivt ! indices + integer :: l, g, p, ivt ! indices + logical :: latvary_baset character(len=*), parameter :: subname = 'InitCold' !----------------------------------------------------------------------- -!DLL - added wheat & sugarcane restrictions to base T vary by lat + latvary_baset = trim(this%baset_mapping) == baset_map_latvary + if (.not. latvary_baset) then + this%latbaset_patch(bounds%begp:bounds%endp) = nan + end if + do p= bounds%begp,bounds%endp - g = patch%gridcell(p) - ivt = patch%itype(p) + l = patch%landunit(p) this%nyrs_crop_active_patch(p) = 0 - this%latbaset_patch(p) = latbaset(pftcon%baset(ivt), grc%latdeg(g), this%baset_latvary_intercept, this%baset_latvary_slope) - if ( trim(this%baset_mapping) == baset_map_constant ) then - this%latbaset_patch(p) = nan - end if - end do -!DLL -- end of mods - - if (use_crop) then - do p= bounds%begp,bounds%endp + if (lun%itype(l) == istcrop) then g = patch%gridcell(p) - l = patch%landunit(p) - c = patch%column(p) + ivt = patch%itype(p) + this%fertnitro_patch(p) = fert_cft(g,ivt) - if (lun%itype(l) == istcrop) then - m = patch%itype(p) - this%fertnitro_patch(p) = fert_cft(g,m) + if (latvary_baset) then + this%latbaset_patch(p) = latbaset(pftcon%baset(ivt), grc%latdeg(g), this%baset_latvary_intercept, this%baset_latvary_slope) end if - end do - end if + end if + end do end subroutine InitCold From 239aa8715b278c8a1925d157cb0facbee5bc34b0 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sun, 12 Nov 2023 13:01:11 -0700 Subject: [PATCH 074/173] Move latbaset_max_lat() to test_Latbaset. --- src/biogeochem/CropType.F90 | 10 +--------- src/biogeochem/test/Latbaset_test/test_Latbaset.pf | 9 ++++++++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 03e25135e6..2e059c5ac1 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -24,7 +24,6 @@ module CropType ! ! !PUBLIC DATA TYPES: public :: latbaset - public :: latbaset_max_lat ! ! Possible values of cphase @@ -971,7 +970,7 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary real(r8), intent(in) :: baset_latvary_slope ! Was originally - ! maxlat = latbaset_max_lat(baset_latvary_intercept, baset_latvary_slope) + ! maxlat = baset_latvary_intercept / baset_latvary_slope ! if (abs(latdeg) > maxlat) then ! latbaset = baset ! else @@ -983,11 +982,4 @@ real(r8) function latbaset(baset, latdeg, baset_latvary_intercept, baset_latvary end function latbaset - real(r8) function latbaset_max_lat(intercept, slope) - real(r8), intent(in) :: intercept - real(r8), intent(in) :: slope - - latbaset_max_lat = intercept / slope - end function latbaset_max_lat - end module CropType diff --git a/src/biogeochem/test/Latbaset_test/test_Latbaset.pf b/src/biogeochem/test/Latbaset_test/test_Latbaset.pf index 8fcc4de1bb..ebb5bfa5e4 100644 --- a/src/biogeochem/test/Latbaset_test/test_Latbaset.pf +++ b/src/biogeochem/test/Latbaset_test/test_Latbaset.pf @@ -7,7 +7,7 @@ module test_Latbaset use unittestSubgridMod use unittestSimpleSubgridSetupsMod use unittestFilterBuilderMod - use CropType, only : latbaset, latbaset_max_lat + use CropType, only : latbaset implicit none @@ -36,6 +36,13 @@ contains call unittest_subgrid_teardown() end subroutine tearDown + real(r8) function latbaset_max_lat(intercept, slope) + real(r8), intent(in) :: intercept + real(r8), intent(in) :: slope + + latbaset_max_lat = intercept / slope + end function latbaset_max_lat + @Test subroutine too_far_north(this) class(TestLatbaset), intent(inout) :: this From df110db54f2fdea4ae2a314dd65c48b716ac1ce3 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Tue, 14 Nov 2023 12:51:09 -0700 Subject: [PATCH 075/173] Update ChangeLog/ChangeSum --- doc/ChangeLog | 108 ++++++++++++++++++++++++++++---------------------- doc/ChangeSum | 2 +- 2 files changed, 61 insertions(+), 49 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 374793cd34..50636efe5a 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,12 +1,13 @@ =============================================================== Tag name: ctsm5.1.dev152 -Originator(s): multiple (Teagan King,UCAR/CCR; slevis TSS) -Date: Thu Nov 2 16:03:44 MDT 2023 -One-line Summary: Move tools to python directory and add tests; also... +Originator(s): multiple (tking (Teagan King); slevis (Sam Levis); AdrienDams (Adrien Damseaux); afoster (Adrianna Foster); samrabin (Sam Rabin); ekluzek (Erik Kluzek); wwieder (Will Wieder); sacks (Bill Sacks); a few others listed below) +Date: Tue Nov 14 10:54:43 MST 2023 +One-line Summary: Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors Purpose and description of changes ---------------------------------- +#2156 tking, slevis Move the following scripts to /python/ctsm/site_and_regional and make wrapper scripts for them in /tools/site_and_regional: - run_neon.py @@ -23,6 +24,30 @@ Add system testing for: - modify_singlept_site_neon - run_neon +#2148 Adrien Damseaux, Victoria Dutch, Leanne Wake +Add namelist option snow_thermal_cond_method to select between Jordan (1991) (default) and +Sturm et al. (1997). Sturm option described for single point runs by Dutch et al. (2022). + +#2233 afoster, sacks +Fix a compiler error (for GNU 13.2) within cropcalStreamMod. +Simple fix was to change whole-array assignments/references for the starts and ends arrays to specifically +reference bounds (begp and endp). + +#2235 srabin, wwieder +Refactor ssp_anomaly_forcing script to make it easier to read and more amenable to future development. +- Adds --output-dir option; default ./anomaly_forcing reproduces previous behavior +- Makes synonyms for options with hyphens replacing underscores + +#2237 srabin +Add the following fields to restart files: +- repr_grainc_to_seed_perharv_patch +- swindow_starts_thisyr_patch +- swindow_ends_thisyr_patch + +#2044 ekluzek +More confined regular expression for NEON and a few simple fixes. + + Significant changes to scientifically-supported configurations -------------------------------------------------------------- @@ -43,50 +68,35 @@ Does this tag change answers significantly for any of the following physics conf Bugs fixed or introduced ------------------------ CTSM issues fixed (include CTSM Issue #): -#1441 +Closes #2156 Fixes #1441 +Closes #2148 +Closes #2233 Fixes #2232 +Closes #2235 +Closes #2237 Fixes #2236 +Closes #2044 Fixes #2039 Fixes #2103 Fixes #2028 Fixes #1506 Fixes #1499 Known Issues: -There were some previous pylint errors that still remain in this tag. +pylint errors from previous work remain in this tag. Notes of particular relevance for users --------------------------------------- Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): -wrapper scripts don't have .py suffixes +#2156 New wrapper scripts don't have .py suffixes. +#2148 New namelist option snow_thermal_cond_method as described above. +#2133 None +#2135 New --output-dir option; default ./anomaly_forcing reproduces previous behavior. +Also makes synonyms for options with hyphens replacing underscores. +#2137 None +#2044 None Notes of particular relevance for developers: --------------------------------------------- Changes to tests or testing: -Numerous changes were made to include new tests. -README.md for testing was also updated to clarify that arguments should be used. +#2156 Numerous changes were made to include new tests. +README.md for testing was updated to clarify that arguments should be used. Testing summary: ---------------- -[... Remove before making master tag. - -Nearly all CTSM tags should undergo 'regular' (aux_clm) testing. -However, it occasionally makes sense to do more or less system testing; -here is guidance on different available levels of system testing: - a) no system testing (for use when the only changes are ones that - have absolutely no impact on system runs; this - includes documentation-only tags, tags that - just change the tools or some python code that - does not impact system runs, etc.) - b) minimal (for use in rare cases where only a small change with - known behavior is added ... eg. a minor bug fix. This - might be to just run the "short" test list, or to run - a single test. Whatever makes sense for the particular case.) - c) python only (for use where the only changes are in the python directory: - run the python testing listed below) - d) regular (regular tests on normal machines if CTSM source is modified) - e) release (regular tests plus the fates, ctsm_sci, mosart and rtm test lists - and normally all of the ancillary tests (build-namelist, python, ptclm, etc.) - would be run as well) - -In addition, various other tests of the tools, python and perl -infrastructure should be run when appropriate, as described below. - -...] - [Remove any lines that don't apply.] [PASS means all tests PASS; OK means tests PASS other than expected fails.] @@ -101,21 +111,11 @@ infrastructure should be run when appropriate, as described below. python testing (if python code has changed; see instructions in python/README.md; document testing done): - (any machine) - - - [If python code has changed and you are NOT running aux_clm (e.g., because the only changes are in python - code) then also run the clm_pymods test suite; this is a small subset of aux_clm that runs the system - tests impacted by python changes. The best way to do this, if you expect no changes from the last tag in - either model output or namelists, is: create sym links pointing to the last tag's baseline directory, - named with the upcoming tag; then run the clm_pymods test suite comparing against these baselines but NOT - doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to - separately run the clm_pymods test suite, and you can remove the following line.] - - clm_pymods test suite on cheyenne - + (any machine) - cheyenne OK (pylint suggestions from previous work remain) regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - cheyenne ---- + cheyenne ---- IN PROGRESS izumi ------- fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) @@ -128,12 +128,24 @@ If the tag used for baseline comparisons was NOT the previous tag, note that her Answer changes -------------- -Changes answers relative to baseline: None +Changes answers relative to baseline: +#2156 NO +#2148 NO +#2233 NO +#2235 Adds attributes to write_climo files' dimension variables +#2237 ONLY Smallville "no initial" restarts; specifically, this previously failing (COMPARE_base_rest) +aux_clm test ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropQianRs.izumi_gnu.clm-cropMonthlyNoinitial +#2044 For NEON cases? Other details ------------- Pull Requests that document the changes (include PR ids): -(https://github.com/ESCOMP/ctsm/pull/2156) + https://github.com/ESCOMP/ctsm/pull/2156 + https://github.com/ESCOMP/ctsm/pull/2148 + https://github.com/ESCOMP/ctsm/pull/2233 + https://github.com/ESCOMP/ctsm/pull/2235 + https://github.com/ESCOMP/ctsm/pull/2237 + https://github.com/ESCOMP/ctsm/pull/2044 =============================================================== =============================================================== diff --git a/doc/ChangeSum b/doc/ChangeSum index 8668cd240a..6af88be0e4 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev152 multiple 11/13/2023 Move tools to python directory and add tests and... + ctsm5.1.dev152 multiple 11/14/2023 Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors ctsm5.1.dev151 rgknox 11/11/2023 Fixes to FATES long run restarts ctsm5.1.dev150 rgknox 11/06/2023 FATES API fix to support future fates npp-fixation coupling, and urgent coupling fixes with E3SM. ctsm5.1.dev149 samrabin 11/03/2023 Rearrange leaf/stem "harvest" and fix soil gas diffusivity From ed57371cb5d6d9f9461472c747d6ab02030e4893 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Tue, 14 Nov 2023 13:09:03 -0700 Subject: [PATCH 076/173] Minor update to ChangeLog --- doc/ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 50636efe5a..c198863971 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -24,7 +24,7 @@ Add system testing for: - modify_singlept_site_neon - run_neon -#2148 Adrien Damseaux, Victoria Dutch, Leanne Wake +#2148 Adrien Damseaux (AWI, Germany), Victoria Dutch, Leanne Wake Add namelist option snow_thermal_cond_method to select between Jordan (1991) (default) and Sturm et al. (1997). Sturm option described for single point runs by Dutch et al. (2022). From db5806ab4cfebfc104e71eb3804e7d5c91631a89 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Tue, 14 Nov 2023 17:15:04 -0700 Subject: [PATCH 077/173] Remove test from ExpectedTestFails.xml and update ChangeLog --- cime_config/testdefs/ExpectedTestFails.xml | 7 ------ doc/ChangeLog | 28 ++++++++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index b696fadcf2..8e72abb8eb 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -57,13 +57,6 @@ - - - FAIL - #2236 - - - diff --git a/doc/ChangeLog b/doc/ChangeLog index c198863971..9405e19922 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev152 Originator(s): multiple (tking (Teagan King); slevis (Sam Levis); AdrienDams (Adrien Damseaux); afoster (Adrianna Foster); samrabin (Sam Rabin); ekluzek (Erik Kluzek); wwieder (Will Wieder); sacks (Bill Sacks); a few others listed below) -Date: Tue Nov 14 10:54:43 MST 2023 +Date: Tue Nov 14 17:09:43 MST 2023 One-line Summary: Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors Purpose and description of changes @@ -115,8 +115,14 @@ Testing summary: regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - cheyenne ---- IN PROGRESS - izumi ------- + cheyenne ---- OK + izumi ------- OK, the following PASS/FAILs are expected: + +PASS ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropQianRs.izumi_gnu.clm-cropMonthlyNoinitial COMPARE_base_rest (UNEXPECTED: expected FAIL) +FAIL ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropQianRs.izumi_gnu.clm-cropMonthlyNoinitial BASELINE ctsm5.1.dev151: DIFF + +FAIL SMS_Ld10_D_Mmpi-serial.CLM_USRDAT.I1PtClm51Bgc.izumi_nag.clm-default--clm-NEON-NIWO BASELINE ctsm5.1.dev151: DIFF +FAIL SMS_Ld10_D_Mmpi-serial.CLM_USRDAT.I1PtClm51Bgc.izumi_nag.clm-NEON-MOAB--clm-PRISM BASELINE ctsm5.1.dev151: DIFF fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) cheyenne ---- @@ -132,10 +138,18 @@ Changes answers relative to baseline: #2156 NO #2148 NO #2233 NO -#2235 Adds attributes to write_climo files' dimension variables -#2237 ONLY Smallville "no initial" restarts; specifically, this previously failing (COMPARE_base_rest) -aux_clm test ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropQianRs.izumi_gnu.clm-cropMonthlyNoinitial -#2044 For NEON cases? +#2235 NO, adds attributes to write_climo files' dimension variables +#2237 ONLY Smallville "no initial" restarts; specifically, this previously +failing (COMPARE_base_rest) aux_clm test +ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropQianRs.izumi_gnu.clm-cropMonthlyNoinitial +now differs from the baseline as follows: + SUMMARY of cprnc: + A total number of 76 fields were compared + and 3 had differences in fill patterns + A total number of 2 fields could not be analyzed + diff_test: the two files seem to be DIFFERENT +#2044 ONLY the NEON tests listed above due to the one-line change in +cime_config/usermods_dirs/NEON/defaults/shell_commands in #2044 Other details ------------- From f9978db4a7cbc924bf1bc8a848ecbccee1be5236 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 1 Nov 2023 20:59:53 -0600 Subject: [PATCH 078/173] Changes to fix #2219 --- src/biogeophys/BareGroundFluxesMod.F90 | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/biogeophys/BareGroundFluxesMod.F90 b/src/biogeophys/BareGroundFluxesMod.F90 index 7dfa83820d..dcc2b8f5b5 100644 --- a/src/biogeophys/BareGroundFluxesMod.F90 +++ b/src/biogeophys/BareGroundFluxesMod.F90 @@ -190,8 +190,12 @@ subroutine BareGroundFluxes(bounds, num_noexposedvegp, filter_noexposedvegp, & forc_th => atm2lnd_inst%forc_th_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric potential temperature (Kelvin) forc_t => atm2lnd_inst%forc_t_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric temperature (Kelvin) forc_pbot => atm2lnd_inst%forc_pbot_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric pressure (Pa) - forc_rho => atm2lnd_inst%forc_rho_downscaled_col , & ! Input: [real(r8) (:) ] density (kg/m**3) - forc_q => wateratm2lndbulk_inst%forc_q_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric specific humidity (kg/kg) + forc_rho => atm2lnd_inst%forc_rho_downscaled_col , & ! Input: [real(r8) (:) ] density (kg/m**3) + forc_hgt_t => atm2lnd_inst%forc_hgt_t_grc , & ! Input: [real(r8) (:) ] observational height of temperature [m] + forc_hgt_u => atm2lnd_inst%forc_hgt_u_grc , & ! Input: [real(r8) (:) ] observational height of wind [m] + forc_hgt_q => atm2lnd_inst%forc_hgt_q_grc , & ! Input: [real(r8) (:) ] observational height of specific humidity [m] + + forc_q => wateratm2lndbulk_inst%forc_q_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric specific humidity (kg/kg) watsat => soilstate_inst%watsat_col , & ! Input: [real(r8) (:,:) ] volumetric soil water at saturation (porosity) soilbeta => soilstate_inst%soilbeta_col , & ! Input: [real(r8) (:) ] soil wetness relative to field capacity @@ -362,9 +366,9 @@ subroutine BareGroundFluxes(bounds, num_noexposedvegp, filter_noexposedvegp, & ! Update the forcing heights for new roughness lengths ! TODO(KWO, 2022-03-15) Only for Meier2022 for now to maintain bfb with ZengWang2007 if (z0param_method == 'Meier2022') then - forc_hgt_u_patch(p) = forc_hgt_u_patch(g) + z0mg_patch(p) + displa(p) - forc_hgt_t_patch(p) = forc_hgt_t_patch(g) + z0hg_patch(p) + displa(p) - forc_hgt_q_patch(p) = forc_hgt_q_patch(g) + z0qg_patch(p) + displa(p) + forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg_patch(p) + displa(p) + forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg_patch(p) + displa(p) + forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg_patch(p) + displa(p) end if thvstar = tstar*(1._r8+0.61_r8*forc_q(c)) + 0.61_r8*forc_th(c)*qstar zeta(p) = zldis(p)*vkc*grav*thvstar/(ustar(p)**2*thv(c)) From b8c71fa58119b5ef164746660123b3f9cbaaadc2 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 15 Nov 2023 16:09:22 -0700 Subject: [PATCH 079/173] Change namelist_defaults_ctsm.xml to use ctsm51_params.RMz0.c231011.nc --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index ec3bc2f2b4..21746f5411 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -488,7 +488,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/ctsm51_params.c211112.nc +lnd/clm2/paramdata/ctsm51_params.RMz0.c231011.nc lnd/clm2/paramdata/clm50_params.c211112.nc lnd/clm2/paramdata/clm45_params.c211112.nc From 626f520a0c90d3fa5d11273591cdbf87a9d776d7 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 15 Nov 2023 16:10:52 -0700 Subject: [PATCH 080/173] Update forcing heights per #2071 --- src/biogeophys/BareGroundFluxesMod.F90 | 10 ++++------ src/biogeophys/CanopyFluxesMod.F90 | 9 +++------ src/biogeophys/LakeFluxesMod.F90 | 22 ++++++---------------- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/biogeophys/BareGroundFluxesMod.F90 b/src/biogeophys/BareGroundFluxesMod.F90 index dcc2b8f5b5..2ad943f1ca 100644 --- a/src/biogeophys/BareGroundFluxesMod.F90 +++ b/src/biogeophys/BareGroundFluxesMod.F90 @@ -364,12 +364,10 @@ subroutine BareGroundFluxes(bounds, num_noexposedvegp, filter_noexposedvegp, & z0qg_patch(p) = z0hg_patch(p) ! Update the forcing heights for new roughness lengths - ! TODO(KWO, 2022-03-15) Only for Meier2022 for now to maintain bfb with ZengWang2007 - if (z0param_method == 'Meier2022') then - forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg_patch(p) + displa(p) - forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg_patch(p) + displa(p) - forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg_patch(p) + displa(p) - end if + forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg_patch(p) + displa(p) + forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg_patch(p) + displa(p) + forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg_patch(p) + displa(p) + thvstar = tstar*(1._r8+0.61_r8*forc_q(c)) + 0.61_r8*forc_th(c)*qstar zeta(p) = zldis(p)*vkc*grav*thvstar/(ustar(p)**2*thv(c)) diff --git a/src/biogeophys/CanopyFluxesMod.F90 b/src/biogeophys/CanopyFluxesMod.F90 index ae0832a2b2..f152e761eb 100644 --- a/src/biogeophys/CanopyFluxesMod.F90 +++ b/src/biogeophys/CanopyFluxesMod.F90 @@ -927,12 +927,9 @@ subroutine CanopyFluxes(bounds, num_exposedvegp, filter_exposedvegp, z0qv(p) = z0mv(p) ! Update the forcing heights - ! TODO(KWO, 2022-03-15) Only for Meier2022 for now to maintain bfb with ZengWang2007 - if (z0param_method == 'Meier2022') then - forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mv(p) + displa(p) - forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hv(p) + displa(p) - forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qv(p) + displa(p) - end if + forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mv(p) + displa(p) + forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hv(p) + displa(p) + forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qv(p) + displa(p) end do diff --git a/src/biogeophys/LakeFluxesMod.F90 b/src/biogeophys/LakeFluxesMod.F90 index fdd105ad8f..fb5f723839 100644 --- a/src/biogeophys/LakeFluxesMod.F90 +++ b/src/biogeophys/LakeFluxesMod.F90 @@ -373,16 +373,9 @@ subroutine LakeFluxes(bounds, num_lakec, filter_lakec, num_lakep, filter_lakep, ! Surface temperature and fluxes ! Update forcing heights for updated roughness lengths - ! TODO(KWO, 2022-03-15) Only for Meier2022 for now to maintain bfb with ZengWang2007 - if (z0param_method == 'Meier2022') then - forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg(p) - forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg(p) - forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg(p) - else - forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg(p) - forc_hgt_t_patch(p) = forc_hgt_t(g) + z0mg(p) - forc_hgt_q_patch(p) = forc_hgt_q(g) + z0mg(p) - end if + forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg(p) + forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg(p) + forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg(p) ! Find top layer jtop(c) = snl(c) + 1 @@ -626,12 +619,9 @@ subroutine LakeFluxes(bounds, num_lakec, filter_lakec, num_lakep, filter_lakep, end if ! Update forcing heights for updated roughness lengths - ! TODO(KWO, 2022-03-15) Only for Meier2022 for now to maintain bfb with ZengWang2007 - if (z0param_method == 'Meier2022') then - forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg(p) - forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg(p) - forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg(p) - end if + forc_hgt_u_patch(p) = forc_hgt_u(g) + z0mg(p) + forc_hgt_t_patch(p) = forc_hgt_t(g) + z0hg(p) + forc_hgt_q_patch(p) = forc_hgt_q(g) + z0qg(p) end do ! end of filtered pft loop From 319d194d6962ec7fc4496b4e2d90e64d3277ffef Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 15 Nov 2023 17:57:52 -0700 Subject: [PATCH 081/173] Update code and params file with MIMICS changes per #1845 --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- .../SoilBiogeochemDecompCascadeMIMICSMod.F90 | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 21746f5411..7d8a3e0342 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -488,7 +488,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/ctsm51_params.RMz0.c231011.nc +/glade/u/home/slevis/paramfiles/ctsm51_params.c231115.nc lnd/clm2/paramdata/clm50_params.c211112.nc lnd/clm2/paramdata/clm45_params.c211112.nc diff --git a/src/soilbiogeochem/SoilBiogeochemDecompCascadeMIMICSMod.F90 b/src/soilbiogeochem/SoilBiogeochemDecompCascadeMIMICSMod.F90 index b8e7137a32..7185febca6 100644 --- a/src/soilbiogeochem/SoilBiogeochemDecompCascadeMIMICSMod.F90 +++ b/src/soilbiogeochem/SoilBiogeochemDecompCascadeMIMICSMod.F90 @@ -251,12 +251,12 @@ subroutine readParams ( ncid ) call ncd_io(trim(tString), params_inst%mimics_fmet(:), 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(sourcefile, __LINE__)) - allocate(params_inst%mimics_fchem_r(4)) + allocate(params_inst%mimics_fchem_r(2)) tString='mimics_fchem_r' call ncd_io(trim(tString), params_inst%mimics_fchem_r(:), 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(sourcefile, __LINE__)) - allocate(params_inst%mimics_fchem_k(4)) + allocate(params_inst%mimics_fchem_k(2)) tString='mimics_fchem_k' call ncd_io(trim(tString), params_inst%mimics_fchem_k(:), 'read', ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(sourcefile, __LINE__)) @@ -828,10 +828,8 @@ subroutine decomp_rates_mimics(bounds, num_bgc_soilc, filter_bgc_soilc, & real(r8):: mimics_fmet_p4 real(r8):: mimics_fchem_r_p1 real(r8):: mimics_fchem_r_p2 - real(r8):: mimics_fchem_r_p3 real(r8):: mimics_fchem_k_p1 real(r8):: mimics_fchem_k_p2 - real(r8):: mimics_fchem_k_p3 real(r8):: mimics_tau_mod_min real(r8):: mimics_tau_mod_max real(r8):: mimics_tau_mod_factor @@ -1092,10 +1090,8 @@ subroutine decomp_rates_mimics(bounds, num_bgc_soilc, filter_bgc_soilc, & mimics_fmet_p4 = params_inst%mimics_fmet(4) mimics_fchem_r_p1 = params_inst%mimics_fchem_r(1) mimics_fchem_r_p2 = params_inst%mimics_fchem_r(2) - mimics_fchem_r_p3 = params_inst%mimics_fchem_r(3) mimics_fchem_k_p1 = params_inst%mimics_fchem_k(1) mimics_fchem_k_p2 = params_inst%mimics_fchem_k(2) - mimics_fchem_k_p3 = params_inst%mimics_fchem_k(3) mimics_tau_mod_min = params_inst%mimics_tau_mod_min mimics_tau_mod_max = params_inst%mimics_tau_mod_max mimics_tau_mod_factor = params_inst%mimics_tau_mod_factor @@ -1186,9 +1182,9 @@ subroutine decomp_rates_mimics(bounds, num_bgc_soilc, filter_bgc_soilc, & ! Used in the update of certain pathfrac terms that vary with time ! in the next loop fchem_m1 = min(1._r8, max(0._r8, mimics_fchem_r_p1 * & - exp(mimics_fchem_r_p2 * fmet) * mimics_fchem_r_p3)) + exp(mimics_fchem_r_p2 * fmet))) fchem_m2 = min(1._r8, max(0._r8, mimics_fchem_k_p1 * & - exp(mimics_fchem_k_p2 * fmet) * mimics_fchem_k_p3)) + exp(mimics_fchem_k_p2 * fmet))) do j = 1,nlevdecomp ! vmax ends up in units of per hour but is expected @@ -1283,6 +1279,7 @@ subroutine decomp_rates_mimics(bounds, num_bgc_soilc, filter_bgc_soilc, & ! The right hand side is OXIDAT in the testbed (line 1145) decomp_k(c,j,i_chem_som) = (term_1 + term_2) * w_d_o_scalars + ! Currently, mimics_densdep = 1 so as to have no effect decomp_k(c,j,i_cop_mic) = tau_m1 * & m1_conc**(mimics_densdep - 1.0_r8) * w_d_o_scalars favl = min(1.0_r8, max(0.0_r8, 1.0_r8 - fphys_m1(c,j) - fchem_m1)) From 28ebdd4668afe27ddb4615b624b4cee9ea0e5938 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Fri, 17 Nov 2023 07:52:17 -0700 Subject: [PATCH 082/173] remove expected fails, update neon testdef to use v2 data --- cime_config/testdefs/ExpectedTestFails.xml | 20 ------------------- .../NEON/FATES/defaults/shell_commands | 2 ++ .../NEON/defaults/shell_commands | 2 ++ 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 8e72abb8eb..047d8ee952 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -59,26 +59,6 @@ - - - FAIL - #667 - - - - - - FAIL - FATES#897 - - - - - - FAIL - FATES#897 - - diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands index 798abcae84..5bec7332b0 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands @@ -39,3 +39,5 @@ fi # Explicitly set PIO Type to NETCDF since this is a single processor case (should already be set this way) ./xmlchange PIO_TYPENAME=netcdf + +./xmlchange NEONVERSION="v2" diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 1f5427ca98..39810dbc70 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -43,3 +43,5 @@ fi # Explicitly set PIO Type to NETCDF since this is a single processor case (should already be set this way) ./xmlchange PIO_TYPENAME=netcdf + +./xmlchange NEONVERSION="v2" From af5c36c5ac116b27b3613a394275f76cf5cf90af Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 17 Nov 2023 11:53:29 -0700 Subject: [PATCH 083/173] update externals, update changelog --- Externals_CLM.cfg | 2 +- doc/ChangeLog | 62 +++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg index cab31b6b7f..5f8e8d2441 100644 --- a/Externals_CLM.cfg +++ b/Externals_CLM.cfg @@ -2,7 +2,7 @@ local_path = src/fates protocol = git repo_url = https://github.com/NGEET/fates -tag = sci.1.68.2_api.30.0.0 +tag = sci.1.68.2_api.31.0.0 required = True [externals_description] diff --git a/doc/ChangeLog b/doc/ChangeLog index 9405e19922..65926f9164 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,66 @@ =============================================================== +Tag name: ctsm5.1.dev153 +Originator(s): afoster (Adrianna Foster) and johnpaulalex (John Paul Alex) +Date: Fri Nov 17 11:53:14 MST 2023 +One-line Summary: Call new FATES-side FatesReadParameters + +Purpose and description of changes +---------------------------------- + +Have CTSM use the new code path in FATES that allows passing in a `fates_param_reader_type`, which does the actual work reading the parameter files, in lieu of calling CTSM methods. + +Also updated NEON usermods to use version 2 data by default, rather than latest. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + + +Testing summary: +---------------- + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- OK + izumi ------- OK + + fates baseline: `fates-sci.1.68.2_api.31.0.0-ctsm5.1.dev153` + + +Answer changes +-------------- + +None + + +Other details +-------------- + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): fates + +Pull Requests that document the changes (include PR ids): +https://github.com/NGEET/fates/pull/1096 +https://github.com/ESCOMP/CTSM/pull/2198 + + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev152 Originator(s): multiple (tking (Teagan King); slevis (Sam Levis); AdrienDams (Adrien Damseaux); afoster (Adrianna Foster); samrabin (Sam Rabin); ekluzek (Erik Kluzek); wwieder (Will Wieder); sacks (Bill Sacks); a few others listed below) Date: Tue Nov 14 17:09:43 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 6af88be0e4..953884bab1 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev153 afoster 11/17/2023 Call new FATES-side FatesReadParameters ctsm5.1.dev152 multiple 11/14/2023 Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors ctsm5.1.dev151 rgknox 11/11/2023 Fixes to FATES long run restarts ctsm5.1.dev150 rgknox 11/06/2023 FATES API fix to support future fates npp-fixation coupling, and urgent coupling fixes with E3SM. From 2ee6943d11f72919d2550e1f1760090465683257 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 17 Nov 2023 12:13:28 -0700 Subject: [PATCH 084/173] Change namelist_defaults_ctsm.xml to use ctsm51_params.c231115b.nc --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 7d8a3e0342..f3c8728294 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -488,7 +488,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -/glade/u/home/slevis/paramfiles/ctsm51_params.c231115.nc +/glade/u/home/slevis/paramfiles/ctsm51_params.c231115b.nc lnd/clm2/paramdata/clm50_params.c211112.nc lnd/clm2/paramdata/clm45_params.c211112.nc From f185a31aa7d9b5bcf4744f722de7c99523307226 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 17 Nov 2023 12:16:33 -0700 Subject: [PATCH 085/173] Revised indentation based on code review --- src/biogeophys/BareGroundFluxesMod.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/biogeophys/BareGroundFluxesMod.F90 b/src/biogeophys/BareGroundFluxesMod.F90 index 2ad943f1ca..7db214065d 100644 --- a/src/biogeophys/BareGroundFluxesMod.F90 +++ b/src/biogeophys/BareGroundFluxesMod.F90 @@ -191,9 +191,9 @@ subroutine BareGroundFluxes(bounds, num_noexposedvegp, filter_noexposedvegp, & forc_t => atm2lnd_inst%forc_t_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric temperature (Kelvin) forc_pbot => atm2lnd_inst%forc_pbot_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric pressure (Pa) forc_rho => atm2lnd_inst%forc_rho_downscaled_col , & ! Input: [real(r8) (:) ] density (kg/m**3) - forc_hgt_t => atm2lnd_inst%forc_hgt_t_grc , & ! Input: [real(r8) (:) ] observational height of temperature [m] - forc_hgt_u => atm2lnd_inst%forc_hgt_u_grc , & ! Input: [real(r8) (:) ] observational height of wind [m] - forc_hgt_q => atm2lnd_inst%forc_hgt_q_grc , & ! Input: [real(r8) (:) ] observational height of specific humidity [m] + forc_hgt_t => atm2lnd_inst%forc_hgt_t_grc , & ! Input: [real(r8) (:) ] observational height of temperature [m] + forc_hgt_u => atm2lnd_inst%forc_hgt_u_grc , & ! Input: [real(r8) (:) ] observational height of wind [m] + forc_hgt_q => atm2lnd_inst%forc_hgt_q_grc , & ! Input: [real(r8) (:) ] observational height of specific humidity [m] forc_q => wateratm2lndbulk_inst%forc_q_downscaled_col , & ! Input: [real(r8) (:) ] atmospheric specific humidity (kg/kg) From bd08985459befa7070b40ae903f75ec32192ab6c Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 17 Nov 2023 13:24:42 -0700 Subject: [PATCH 086/173] add issues to changelog --- doc/ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 65926f9164..af74421906 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -26,7 +26,10 @@ Does this tag change answers significantly for any of the following physics conf [ ] clm4_5 +Bugs fixed or introduced +------------------------ +Some progress towards CTSM#2006 and FATES#1076 Testing summary: ---------------- From b8d1508286afe7963357b0b12bf0481d01df5dc6 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 17 Nov 2023 13:31:31 -0700 Subject: [PATCH 087/173] update to pend --- cime_config/testdefs/ExpectedTestFails.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 047d8ee952..44876ba818 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -83,7 +83,7 @@ - FAIL + PEND FATES#983 This job should time out on izumi, seems to be hanging on history output. From 29ca5ada72b03d374b93c0a06e21af2c8e977a3a Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 17 Nov 2023 18:45:07 -0700 Subject: [PATCH 088/173] Move SNICAR params to params files per #2247 --- bld/CLMBuildNamelist.pm | 1 - bld/namelist_files/namelist_defaults_ctsm.xml | 10 +--- .../namelist_definition_ctsm.xml | 5 -- src/biogeophys/AerosolMod.F90 | 57 ------------------- src/biogeophys/SnowHydrologyMod.F90 | 44 +++++++++----- src/biogeophys/SnowSnicarMod.F90 | 31 +++++----- src/biogeophys/WaterDiagnosticBulkType.F90 | 14 ++--- 7 files changed, 57 insertions(+), 105 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 5769af0ea2..17d64cd59f 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -4212,7 +4212,6 @@ sub setup_logic_snowpack { add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'snow_overburden_compaction_method'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'lotmp_snowdensity_method'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'upplim_destruct_metamorph'); - add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fresh_snw_rds_max'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'reset_snow'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'reset_snow_glc'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'reset_snow_glc_ela'); diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index f3c8728294..07b440c37b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -431,10 +431,6 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 175.d00 175.d00 -54.526d00 -204.526d00 -204.526d00 - 0.08d00 .false. @@ -488,9 +484,9 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -/glade/u/home/slevis/paramfiles/ctsm51_params.c231115b.nc -lnd/clm2/paramdata/clm50_params.c211112.nc -lnd/clm2/paramdata/clm45_params.c211112.nc +/glade/u/home/slevis/paramfiles/ctsm51_params.c231117.nc +/glade/u/home/slevis/paramfiles/clm50_params.c231117.nc +/glade/u/home/slevis/paramfiles/clm45_params.c231117.nc diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 93306b32d5..2cb95d5c68 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -2783,11 +2783,6 @@ Snow compaction overburden exponential factor (1/K) Not used for snow_overburden_compaction_method=Vionnet2012 - -maximum warm (at freezing) fresh snow effective radius [microns] - - If set to .true., then reset the snow pack over non-glacier columns to a small value. diff --git a/src/biogeophys/AerosolMod.F90 b/src/biogeophys/AerosolMod.F90 index bce2b6f9ab..39ade89fb0 100644 --- a/src/biogeophys/AerosolMod.F90 +++ b/src/biogeophys/AerosolMod.F90 @@ -26,9 +26,6 @@ module AerosolMod public :: AerosolFluxes ! ! !PUBLIC DATA MEMBERS: - real(r8), public, parameter :: snw_rds_min = 54.526_r8 ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] - real(r8), public :: fresh_snw_rds_max = 204.526_r8 ! maximum warm fresh snow effective radius [microns] - ! type, public :: aerosol_type real(r8), pointer, public :: mss_bcpho_col(:,:) ! mass of hydrophobic BC in snow (col,lyr) [kg] real(r8), pointer, public :: mss_bcphi_col(:,:) ! mass of hydrophillic BC in snow (col,lyr) [kg] @@ -93,7 +90,6 @@ module AerosolMod procedure, private :: InitAllocate procedure, private :: InitHistory procedure, private :: InitCold - procedure, private :: InitReadNML end type aerosol_type @@ -113,7 +109,6 @@ subroutine Init(this, bounds, NLFilename) call this%InitAllocate(bounds) call this%InitHistory(bounds) call this%InitCold(bounds) - call this%InitReadNML(NLFilename) end subroutine Init @@ -294,58 +289,6 @@ subroutine InitCold(this, bounds) end subroutine InitCold !----------------------------------------------------------------------- - subroutine InitReadNML(this, NLFilename) - ! - ! !USES: - ! !USES: - use fileutils , only : getavu, relavu, opnfil - use shr_nl_mod , only : shr_nl_find_group_name - use spmdMod , only : masterproc, mpicom - use shr_mpi_mod , only : shr_mpi_bcast - use clm_varctl , only : iulog - ! - ! !ARGUMENTS: - class(aerosol_type) :: this - character(len=*), intent(in) :: NLFilename ! Input namelist filename - ! - ! !LOCAL VARIABLES: - !----------------------------------------------------------------------- - integer :: ierr ! error code - integer :: unitn ! unit for namelist file - - character(len=*), parameter :: subname = 'Aerosol::InitReadNML' - character(len=*), parameter :: nmlname = 'aerosol' - !----------------------------------------------------------------------- - namelist/aerosol/ fresh_snw_rds_max - - if (masterproc) then - unitn = getavu() - write(iulog,*) 'Read in '//nmlname//' namelist' - call opnfil (NLFilename, unitn, 'F') - call shr_nl_find_group_name(unitn, nmlname, status=ierr) - if (ierr == 0) then - read(unitn, nml=aerosol, iostat=ierr) - if (ierr /= 0) then - call endrun(msg="ERROR reading "//nmlname//" namelist "//errmsg(sourcefile, __LINE__)) - end if - else - call endrun(msg="ERROR could NOT find "//nmlname//" namelist "//errmsg(sourcefile, __LINE__)) - end if - call relavu( unitn ) - end if - - call shr_mpi_bcast (fresh_snw_rds_max , mpicom) - - if (masterproc) then - write(iulog,*) ' ' - write(iulog,*) nmlname//' settings:' - write(iulog,nml=aerosol) - write(iulog,*) ' ' - end if - - end subroutine InitReadNML - - !------------------------------------------------------------------------ subroutine Restart(this, bounds, ncid, flag, & h2osoi_ice_col, h2osoi_liq_col) ! diff --git a/src/biogeophys/SnowHydrologyMod.F90 b/src/biogeophys/SnowHydrologyMod.F90 index 4698e1136d..9fb1a52dbc 100644 --- a/src/biogeophys/SnowHydrologyMod.F90 +++ b/src/biogeophys/SnowHydrologyMod.F90 @@ -79,7 +79,14 @@ module SnowHydrologyMod real(r8) :: rho_max ! Wind drift compaction / maximum density (kg/m3) real(r8) :: tau_ref ! Wind drift compaction / reference time (48*3600) (s) real(r8) :: scvng_fct_mlt_sf ! Scaling factor modifying scavenging factors for BC, OC, and dust species inclusion in meltwater (-) + real(r8) :: scvng_fct_mlt_bcphi ! scavenging factor for hydrophillic BC inclusion in meltwater [frc] + real(r8) :: scvng_fct_mlt_bcpho ! scavenging factor for hydrophobic BC inclusion in meltwater [frc] + real(r8) :: scvng_fct_mlt_dst1 ! scavenging factor for dust species 1 inclusion in meltwater [frc] + real(r8) :: scvng_fct_mlt_dst2 ! scavenging factor for dust species 2 inclusion in meltwater [frc] + real(r8) :: scvng_fct_mlt_dst3 ! scavenging factor for dust species 3 inclusion in meltwater [frc] + real(r8) :: scvng_fct_mlt_dst4 ! scavenging factor for dust species 4 inclusion in meltwater [frc] real(r8) :: ceta ! Overburden compaction constant (kg/m3) + real(r8) :: snw_rds_min ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] end type params_type type(params_type), private :: params_inst @@ -120,14 +127,8 @@ module SnowHydrologyMod ! 7= dust species 3 ! 8= dust species 4 ! - real(r8), public, parameter :: scvng_fct_mlt_bcphi = 0.20_r8 ! scavenging factor for hydrophillic BC inclusion in meltwater [frc] - real(r8), public, parameter :: scvng_fct_mlt_bcpho = 0.03_r8 ! scavenging factor for hydrophobic BC inclusion in meltwater [frc] real(r8), public, parameter :: scvng_fct_mlt_ocphi = 0.20_r8 ! scavenging factor for hydrophillic OC inclusion in meltwater [frc] real(r8), public, parameter :: scvng_fct_mlt_ocpho = 0.03_r8 ! scavenging factor for hydrophobic OC inclusion in meltwater [frc] - real(r8), public, parameter :: scvng_fct_mlt_dst1 = 0.02_r8 ! scavenging factor for dust species 1 inclusion in meltwater [frc] - real(r8), public, parameter :: scvng_fct_mlt_dst2 = 0.02_r8 ! scavenging factor for dust species 2 inclusion in meltwater [frc] - real(r8), public, parameter :: scvng_fct_mlt_dst3 = 0.01_r8 ! scavenging factor for dust species 3 inclusion in meltwater [frc] - real(r8), public, parameter :: scvng_fct_mlt_dst4 = 0.01_r8 ! scavenging factor for dust species 4 inclusion in meltwater [frc] ! The following are public for the sake of unit testing integer, parameter, public :: LoTmpDnsSlater2017 = 2 ! For temperature below -15C use equation from Slater 2017 @@ -316,8 +317,22 @@ subroutine readParams( ncid ) call readNcdioScalar(ncid, 'tau_ref', subname, params_inst%tau_ref) ! Scaling factor modifying scavenging factors for BC, OC, and dust species inclusion in meltwater (-) call readNcdioScalar(ncid, 'scvng_fct_mlt_sf', subname, params_inst%scvng_fct_mlt_sf) + ! scavenging factor for hydrophillic BC inclusion in meltwater [frc] + call readNcdioScalar(ncid, 'scvng_fct_mlt_bcphi', subname, params_inst%scvng_fct_mlt_bcphi) + ! scavenging factor for hydrophobic BC inclusion in meltwater [frc] + call readNcdioScalar(ncid, 'scvng_fct_mlt_bcpho', subname, params_inst%scvng_fct_mlt_bcpho) + ! scavenging factor for dust species 1 inclusion in meltwater [frc] + call readNcdioScalar(ncid, 'scvng_fct_mlt_dst1', subname, params_inst%scvng_fct_mlt_dst1) + ! scavenging factor for dust species 2 inclusion in meltwater [frc] + call readNcdioScalar(ncid, 'scvng_fct_mlt_dst2', subname, params_inst%scvng_fct_mlt_dst2) + ! scavenging factor for dust species 3 inclusion in meltwater [frc] + call readNcdioScalar(ncid, 'scvng_fct_mlt_dst3', subname, params_inst%scvng_fct_mlt_dst3) + ! scavenging factor for dust species 4 inclusion in meltwater [frc] + call readNcdioScalar(ncid, 'scvng_fct_mlt_dst4', subname, params_inst%scvng_fct_mlt_dst4) ! Overburden compaction constant (kg/m3) call readNcdioScalar(ncid, 'ceta', subname, params_inst%ceta) + ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] + call readNcdioScalar(ncid, 'snw_rds_min', subname, params_inst%snw_rds_min) end subroutine readParams @@ -1586,7 +1601,7 @@ subroutine CalcAndApplyAerosolFluxes(bounds, num_snowc, filter_snowc, & ! BCPHI: ! 1. flux with meltwater: qout_bc_phi(c) = qflx_snow_percolation(c,j)*params_inst%scvng_fct_mlt_sf* & - scvng_fct_mlt_bcphi*(mss_bcphi(c,j)/mss_liqice) + params_inst%scvng_fct_mlt_bcphi*(mss_bcphi(c,j)/mss_liqice) if (qout_bc_phi(c)*dtime > mss_bcphi(c,j)) then qout_bc_phi(c) = mss_bcphi(c,j)/dtime mss_bcphi(c,j) = 0._r8 @@ -1598,7 +1613,7 @@ subroutine CalcAndApplyAerosolFluxes(bounds, num_snowc, filter_snowc, & ! BCPHO: ! 1. flux with meltwater: qout_bc_pho(c) = qflx_snow_percolation(c,j)*params_inst%scvng_fct_mlt_sf* & - scvng_fct_mlt_bcpho*(mss_bcpho(c,j)/mss_liqice) + params_inst%scvng_fct_mlt_bcpho*(mss_bcpho(c,j)/mss_liqice) if (qout_bc_pho(c)*dtime > mss_bcpho(c,j)) then qout_bc_pho(c) = mss_bcpho(c,j)/dtime mss_bcpho(c,j) = 0._r8 @@ -1634,7 +1649,7 @@ subroutine CalcAndApplyAerosolFluxes(bounds, num_snowc, filter_snowc, & ! DUST 1: ! 1. flux with meltwater: qout_dst1(c) = qflx_snow_percolation(c,j)*params_inst%scvng_fct_mlt_sf* & - scvng_fct_mlt_dst1*(mss_dst1(c,j)/mss_liqice) + params_inst%scvng_fct_mlt_dst1*(mss_dst1(c,j)/mss_liqice) if (qout_dst1(c)*dtime > mss_dst1(c,j)) then qout_dst1(c) = mss_dst1(c,j)/dtime mss_dst1(c,j) = 0._r8 @@ -1646,7 +1661,7 @@ subroutine CalcAndApplyAerosolFluxes(bounds, num_snowc, filter_snowc, & ! DUST 2: ! 1. flux with meltwater: qout_dst2(c) = qflx_snow_percolation(c,j)*params_inst%scvng_fct_mlt_sf* & - scvng_fct_mlt_dst2*(mss_dst2(c,j)/mss_liqice) + params_inst%scvng_fct_mlt_dst2*(mss_dst2(c,j)/mss_liqice) if (qout_dst2(c)*dtime > mss_dst2(c,j)) then qout_dst2(c) = mss_dst2(c,j)/dtime mss_dst2(c,j) = 0._r8 @@ -1658,7 +1673,7 @@ subroutine CalcAndApplyAerosolFluxes(bounds, num_snowc, filter_snowc, & ! DUST 3: ! 1. flux with meltwater: qout_dst3(c) = qflx_snow_percolation(c,j)*params_inst%scvng_fct_mlt_sf* & - scvng_fct_mlt_dst3*(mss_dst3(c,j)/mss_liqice) + params_inst%scvng_fct_mlt_dst3*(mss_dst3(c,j)/mss_liqice) if (qout_dst3(c)*dtime > mss_dst3(c,j)) then qout_dst3(c) = mss_dst3(c,j)/dtime mss_dst3(c,j) = 0._r8 @@ -1670,7 +1685,7 @@ subroutine CalcAndApplyAerosolFluxes(bounds, num_snowc, filter_snowc, & ! DUST 4: ! 1. flux with meltwater: qout_dst4(c) = qflx_snow_percolation(c,j)*params_inst%scvng_fct_mlt_sf* & - scvng_fct_mlt_dst4*(mss_dst4(c,j)/mss_liqice) + params_inst%scvng_fct_mlt_dst4*(mss_dst4(c,j)/mss_liqice) if (qout_dst4(c)*dtime > mss_dst4(c,j)) then qout_dst4(c) = mss_dst4(c,j)/dtime mss_dst4(c,j) = 0._r8 @@ -3936,7 +3951,6 @@ function MassWeightedSnowRadius( rds1, rds2, swtot, zwtot ) result(mass_weighted ! Calculate the mass weighted snow radius when two layers are combined ! ! !USES: - use AerosolMod , only : snw_rds_min use SnowSnicarMod, only : snw_rds_max implicit none ! !ARGUMENTS: @@ -3951,8 +3965,8 @@ function MassWeightedSnowRadius( rds1, rds2, swtot, zwtot ) result(mass_weighted if ( mass_weighted_snowradius > snw_rds_max ) then mass_weighted_snowradius = snw_rds_max - else if ( mass_weighted_snowradius < snw_rds_min ) then - mass_weighted_snowradius = snw_rds_min + else if ( mass_weighted_snowradius < params_inst%snw_rds_min ) then + mass_weighted_snowradius = params_inst%snw_rds_min end if end function MassWeightedSnowRadius diff --git a/src/biogeophys/SnowSnicarMod.F90 b/src/biogeophys/SnowSnicarMod.F90 index 9a95188232..5bc1c61edb 100644 --- a/src/biogeophys/SnowSnicarMod.F90 +++ b/src/biogeophys/SnowSnicarMod.F90 @@ -18,7 +18,6 @@ module SnowSnicarMod use shr_const_mod , only : SHR_CONST_RHOICE use abortutils , only : endrun use decompMod , only : bounds_type, subgrid_level_column - use AerosolMod , only : snw_rds_min use atm2lndType , only : atm2lnd_type use WaterStateBulkType , only : waterstatebulk_type use WaterDiagnosticBulkType , only : waterdiagnosticbulk_type @@ -42,6 +41,8 @@ module SnowSnicarMod real(r8) :: snw_rds_refrz ! Effective radius of re-frozen snow (microns) real(r8) :: C2_liq_Brun89 ! Constant for liquid water grain growth [m3 s-1], ! from Brun89: corrected for LWC in units of percent + real(r8) :: fresh_snw_rds_max ! maximum warm fresh snow effective radius [microns] + real(r8) :: snw_rds_min ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] end type params_type type(params_type), private :: params_inst ! @@ -66,7 +67,6 @@ module SnowSnicarMod integer, parameter :: snw_rds_max_tbl = 1500 ! maximum effective radius defined in Mie lookup table [microns] integer, parameter :: snw_rds_min_tbl = 30 ! minimium effective radius defined in Mie lookup table [microns] - integer, parameter :: snw_rds_min_int = nint(snw_rds_min) ! minimum allowed snow effective radius as integer [microns] real(r8), parameter :: snw_rds_max = 1500._r8 ! maximum allowed snow effective radius [microns] real(r8), parameter :: min_snw = 1.0E-30_r8 ! minimum snow mass required for SNICAR RT calculation [kg m-2] @@ -182,6 +182,10 @@ subroutine readParams( ncid ) call readNcdioScalar(ncid, 'snw_rds_refrz', subname, params_inst%snw_rds_refrz) ! constant for liquid water grain growth [m3 s-1], from Brun89: corrected for LWC in units of percent call readNcdioScalar(ncid, 'C2_liq_Brun89', subname, params_inst%C2_liq_Brun89) + ! maximum warm fresh snow effective radius [microns] + call readNcdioScalar(ncid, 'fresh_snw_rds_max', subname, params_inst%fresh_snw_rds_max) + ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] + call readNcdioScalar(ncid, 'snw_rds_min', subname, params_inst%snw_rds_min) end subroutine readParams @@ -677,7 +681,7 @@ subroutine SNICAR_RT (bounds, num_nourbanc, filter_nourbanc, & snl_lcl = -1 h2osno_ice_lcl(0) = h2osno_lcl h2osno_liq_lcl(0) = 0._r8 - snw_rds_lcl(0) = snw_rds_min_int + snw_rds_lcl(0) = nint(params_inst%snw_rds_min) else flg_nosnl = 0 snl_lcl = snl(c_idx) @@ -1628,10 +1632,10 @@ subroutine SnowAge_grain(bounds, & !LvK extra boundary check, to prevent when using old restart file with lower snw_rds_min than current run - snw_rds(c_idx,i) = max(snw_rds(c_idx,i), snw_rds_min) + snw_rds(c_idx,i) = max(snw_rds(c_idx,i), params_inst%snw_rds_min) ! change in snow effective radius, using best-fit parameters - dr_fresh = snw_rds(c_idx,i)-snw_rds_min + dr_fresh = snw_rds(c_idx,i) - params_inst%snw_rds_min dr = (bst_drdt0 * (bst_tau / (dr_fresh + bst_tau))**(1._r8 / bst_kappa)) * (dtime / secsphr) ! @@ -1701,7 +1705,7 @@ subroutine SnowAge_grain(bounds, & !********** 5. CHECK BOUNDARIES *********** ! ! boundary check - snw_rds(c_idx,i) = max(snw_rds(c_idx,i), snw_rds_min) + snw_rds(c_idx,i) = max(snw_rds(c_idx,i), params_inst%snw_rds_min) snw_rds(c_idx,i) = min(snw_rds(c_idx,i), snw_rds_max) ! set top layer variables for history files @@ -1720,7 +1724,7 @@ subroutine SnowAge_grain(bounds, & do fc = 1, num_nosnowc c_idx = filter_nosnowc(fc) if (h2osno_no_layers(c_idx) > 0._r8) then - snw_rds(c_idx,0) = snw_rds_min + snw_rds(c_idx,0) = params_inst%snw_rds_min endif enddo @@ -1744,7 +1748,7 @@ real(r8) function FreshSnowRadius(c_idx, atm2lnd_inst) ! Author: Leo VanKampenhout ! ! !USES: - use AerosolMod , only : fresh_snw_rds_max + ! ! !ARGUMENTS: integer, intent(in) :: c_idx ! column index type(atm2lnd_type) , intent(in) :: atm2lnd_inst ! Forcing from atmosphere @@ -1753,16 +1757,17 @@ real(r8) function FreshSnowRadius(c_idx, atm2lnd_inst) !----------------------------------------------------------------------- real(r8), parameter :: tmin = tfrz - 30._r8 ! start of linear ramp real(r8), parameter :: tmax = tfrz - 0._r8 ! end of linear ramp - real(r8), parameter :: gs_min = snw_rds_min ! minimum value - real(r8) :: gs_max ! maximum value + real(r8) :: gs_min ! minimum value + real(r8) :: gs_max ! maximum value associate( & forc_t => atm2lnd_inst%forc_t_downscaled_col & ! Input: [real(r8) (:) ] atmospheric temperature (Kelvin) ) - if ( fresh_snw_rds_max <= snw_rds_min )then - FreshSnowRadius = snw_rds_min + if ( params_inst%fresh_snw_rds_max <= params_inst%snw_rds_min )then + FreshSnowRadius = params_inst%snw_rds_min else - gs_max = fresh_snw_rds_max + gs_max = params_inst%fresh_snw_rds_max + gs_min = params_inst%snw_rds_min if (forc_t(c_idx) < tmin) then FreshSnowRadius = gs_min diff --git a/src/biogeophys/WaterDiagnosticBulkType.F90 b/src/biogeophys/WaterDiagnosticBulkType.F90 index 426ddb464c..057062777f 100644 --- a/src/biogeophys/WaterDiagnosticBulkType.F90 +++ b/src/biogeophys/WaterDiagnosticBulkType.F90 @@ -108,12 +108,10 @@ module WaterDiagnosticBulkType type, private :: params_type real(r8) :: zlnd ! Momentum roughness length for soil, glacier, wetland (m) + real(r8) :: snw_rds_min ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] end type params_type type(params_type), private :: params_inst - ! minimum allowed snow effective radius (also "fresh snow" value) [microns] - real(r8), public, parameter :: snw_rds_min = 54.526_r8 - character(len=*), parameter, private :: sourcefile = & __FILE__ !------------------------------------------------------------------------ @@ -136,6 +134,8 @@ subroutine readParams( ncid ) ! Momentum roughness length for soil, glacier, wetland (m) call readNcdioScalar(ncid, 'zlnd', subname, params_inst%zlnd) + ! minimum allowed snow effective radius (also cold "fresh snow" value) [microns] + call readNcdioScalar(ncid, 'snw_rds_min', subname, params_inst%snw_rds_min) end subroutine readParams @@ -750,11 +750,11 @@ subroutine InitBulkCold(this, bounds, & do c = bounds%begc,bounds%endc if (snl(c) < 0) then - this%snw_rds_col(c,snl(c)+1:0) = snw_rds_min + this%snw_rds_col(c,snl(c)+1:0) = params_inst%snw_rds_min this%snw_rds_col(c,-nlevsno+1:snl(c)) = 0._r8 - this%snw_rds_top_col(c) = snw_rds_min + this%snw_rds_top_col(c) = params_inst%snw_rds_min elseif (h2osno_input_col(c) > 0._r8) then - this%snw_rds_col(c,0) = snw_rds_min + this%snw_rds_col(c,0) = params_inst%snw_rds_min this%snw_rds_col(c,-nlevsno+1:-1) = 0._r8 this%snw_rds_top_col(c) = spval this%sno_liq_top_col(c) = spval @@ -1236,7 +1236,7 @@ subroutine ResetBulk(this, column) integer , intent(in) :: column ! column index !----------------------------------------------------------------------- - this%snw_rds_col(column,0) = snw_rds_min + this%snw_rds_col(column,0) = params_inst%snw_rds_min end subroutine ResetBulk From 71e174fa81170451cc5dd99098a89c0f1960dca9 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 11:09:18 -0700 Subject: [PATCH 089/173] Upd. paramfile / z0param_method in namelist_defaults_ctsm and two /testmod dirs --- bld/namelist_files/namelist_defaults_ctsm.xml | 10 ++++++---- .../testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm | 2 -- .../testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 07b440c37b..9fcea25784 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -484,9 +484,9 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -/glade/u/home/slevis/paramfiles/ctsm51_params.c231117.nc -/glade/u/home/slevis/paramfiles/clm50_params.c231117.nc -/glade/u/home/slevis/paramfiles/clm45_params.c231117.nc +lnd/clm2/paramdata/ctsm51_params.c231117.nc +lnd/clm2/paramdata/clm50_params.c231117.nc +lnd/clm2/paramdata/clm45_params.c231117.nc @@ -498,7 +498,9 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -ZengWang2007 +ZengWang2007 +ZengWang2007 +Meier2022 .true. .false. diff --git a/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm index 01df79ecba..6fa686b403 100644 --- a/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm @@ -1,4 +1,2 @@ z0param_method = 'Meier2022' use_z0m_snowmelt = .true. -paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/ctsm51_params.RMz0.c220304.nc' - diff --git a/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm index 7ae4a69aad..57ac8a6461 100644 --- a/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm @@ -1,2 +1,2 @@ -paramfile = '/glade/p/cesm/cseg/inputdata/lnd/clm2/paramdata/ctsm51_ciso_cwd_hr_params.c211112.nc' +paramfile = '/glade/p/cesm/cseg/inputdata/lnd/clm2/paramdata/ctsm51_ciso_cwd_hr_params.c231117.nc' hist_fincl1 = 'CWDC_HR','C13_CWDC_HR','C14_CWDC_HR','CWD_HR_L2','CWD_HR_L2_vr','CWD_HR_L3','CWD_HR_L3_vr' From 13ef0cd2e58a7a375cc0d3265cc6845a28d583c6 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 11:48:43 -0700 Subject: [PATCH 090/173] Draft ChangeLog/Sum --- doc/ChangeLog | 109 ++++++++++++++++++++++++++++++++++++++++++-------- doc/ChangeSum | 1 + 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index af74421906..62837f1f0a 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,97 @@ =============================================================== +Tag name: ctsm5.1.dev154 +Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310) +Date: Mon Nov 20 11:09:45 MST 2023 +One-line Summary: New params files: Changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc + +Purpose and description of changes +---------------------------------- + +This PR (#2258) addresses several issues: +1) Start using existing new params file for Meier roughness: +/glade/campaign/cesm/cesmdata/inputdata/lnd/clm2/paramdata/ctsm51_params.RMz0.c231011.nc +and include bug-fix #2219. +2) Update forcing heights per #2071. +3) Update params file for MIMICS per #1845. +4) Make leafcn for pfts 15 and 16 the same per #2184. +5) Switch the values of params kc_nonmyc and kn_nonmyc per #2120. +6) Move SNICAR parameters to ctsm51, clm50, and clm45 params files per #2247. + +See #2258 and the above issues for a list of contributors. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[x] clm5_1 + +[x] clm5_0 + +[x] ctsm5_0-nwp + +[x] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +Fixes #2219 +Fixes #2071 +Fixes #1845 +Fixes #2184 +Fixes #2120 +Fixes #2247 + + +Notes of particular relevance for users +--------------------------------------- +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): + SNICAR namelist variable fresh_snw_rds_max moved to the params file. + +Changes made to namelist defaults (e.g., changed parameter values): + SNICAR namelist variable fresh_snw_rds_max moved to the params file. + Pointing to new params files for clm4_5, clm5_0, clm5_1. + +Changes to the datasets (e.g., parameter, surface or initial files): + New clm5_1 params file with new parameters and with modified values of existing parameters. + New clm5_0 and clm4_5 params files with new parameters for SNICAR. + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- + izumi ------- + + +Answer changes +-------------- + +Changes answers relative to baseline: YES + + Summarize any changes to answers, i.e., + - what code configurations: ALL + - what platforms/compilers: ALL + - nature of change: larger than roundoff/same climate? + I will post this tag on the Answer changing tags wiki page as "SIGNIFICANT" + and will run a simulation and diagnostics to compare against dev145. + +Other details +------------- +Pull Requests that document the changes (include PR ids): + https://github.com/ESCOMP/ctsm/pull/2258 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev153 Originator(s): afoster (Adrianna Foster) and johnpaulalex (John Paul Alex) Date: Fri Nov 17 11:53:14 MST 2023 @@ -162,18 +255,9 @@ README.md for testing was updated to clarify that arguments should be used. Testing summary: ---------------- -[Remove any lines that don't apply.] [PASS means all tests PASS; OK means tests PASS other than expected fails.] - build-namelist tests (if CLMBuildNamelist.pm has changed): - - cheyenne - - - tools-tests (test/tools) (if tools have been changed): - - cheyenne - - python testing (if python code has changed; see instructions in python/README.md; document testing done): (any machine) - cheyenne OK (pylint suggestions from previous work remain) @@ -189,13 +273,6 @@ FAIL ERS_Lm20_Mmpi-serial.1x1_smallvilleIA.I2000Clm50BgcCropQianRs.izumi_gnu.clm FAIL SMS_Ld10_D_Mmpi-serial.CLM_USRDAT.I1PtClm51Bgc.izumi_nag.clm-default--clm-NEON-NIWO BASELINE ctsm5.1.dev151: DIFF FAIL SMS_Ld10_D_Mmpi-serial.CLM_USRDAT.I1PtClm51Bgc.izumi_nag.clm-NEON-MOAB--clm-PRISM BASELINE ctsm5.1.dev151: DIFF - fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) - cheyenne ---- - izumi ------- - - any other testing (give details below): - -If the tag used for baseline comparisons was NOT the previous tag, note that here: Answer changes -------------- diff --git a/doc/ChangeSum b/doc/ChangeSum index 953884bab1..d86a084cd9 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev154 slevis 11/20/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc ctsm5.1.dev153 afoster 11/17/2023 Call new FATES-side FatesReadParameters ctsm5.1.dev152 multiple 11/14/2023 Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors ctsm5.1.dev151 rgknox 11/11/2023 Fixes to FATES long run restarts From 4792efce2a8cf0e7db0217e0ae89ad2f7b4c954a Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 13:55:22 -0700 Subject: [PATCH 091/173] Remove Meier2022 testmods directory; update testlist accordingly --- cime_config/testdefs/testlist_clm.xml | 16 ++++++++-------- .../clm/Meier2022_surf_rough/include_user_mods | 1 - .../clm/Meier2022_surf_rough/user_nl_clm | 2 -- 3 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/include_user_mods delete mode 100644 cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 34c2cc33f9..d0e2418b56 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -17,13 +17,13 @@ - + - + @@ -1163,16 +1163,16 @@ - + - + - + @@ -1182,16 +1182,16 @@ - + - + - + diff --git a/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/include_user_mods deleted file mode 100644 index fe0e18cf88..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/include_user_mods +++ /dev/null @@ -1 +0,0 @@ -../default diff --git a/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm deleted file mode 100644 index 6fa686b403..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/Meier2022_surf_rough/user_nl_clm +++ /dev/null @@ -1,2 +0,0 @@ -z0param_method = 'Meier2022' -use_z0m_snowmelt = .true. From 6b231bfd278886085d47ff6017b0794f4ceb8608 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 13:56:59 -0700 Subject: [PATCH 092/173] Small revision based on code review --- bld/namelist_files/namelist_defaults_ctsm.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 9fcea25784..0516ecbc22 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -498,8 +498,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -ZengWang2007 -ZengWang2007 +ZengWang2007 Meier2022 .true. From 939bea766ee176b9e4b1a485c04cfba3972744cc Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 16:07:33 -0700 Subject: [PATCH 093/173] For Meier IHist...Crop tests to pass, permit htop < 1e-10 for istcrop --- src/biogeophys/BiogeophysPreFluxCalcsMod.F90 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/biogeophys/BiogeophysPreFluxCalcsMod.F90 b/src/biogeophys/BiogeophysPreFluxCalcsMod.F90 index 11842560ee..62643d680c 100644 --- a/src/biogeophys/BiogeophysPreFluxCalcsMod.F90 +++ b/src/biogeophys/BiogeophysPreFluxCalcsMod.F90 @@ -138,7 +138,7 @@ subroutine SetZ0mDisp(bounds, num_nolakep, filter_nolakep, & type(canopystate_type) , intent(inout) :: canopystate_inst ! ! !LOCAL VARIABLES: - integer :: fp, p + integer :: fp, p, l character(len=*), parameter :: subname = 'SetZ0mDisp' real(r8) :: U_ustar ! wind at canopy height divided by friction velocity (unitless) @@ -164,6 +164,7 @@ subroutine SetZ0mDisp(bounds, num_nolakep, filter_nolakep, & do fp = 1, num_nolakep p = filter_nolakep(fp) + l = patch%landunit(p) if( .not.(patch%is_fates(p))) then select case (z0param_method) @@ -199,8 +200,13 @@ subroutine SetZ0mDisp(bounds, num_nolakep, filter_nolakep, & / 2._r8)**(-0.5_r8) / (pftcon%z0v_LAImax(patch%itype(p))) / pftcon%z0v_c(patch%itype(p)) if ( htop(p) <= 1.e-10_r8 )then - write(iulog,*) ' nstep = ', get_nstep(), ' htop = ', htop(p) - call endrun(subgrid_index=p, subgrid_level=subgrid_level_patch, msg=errMsg(sourcefile, __LINE__)) + if (lun%itype(l) == istcrop) then + z0m(p) = 0._r8 + displa(p) = 0._r8 + else + write(iulog,*) ' nstep = ', get_nstep(), ' htop = ', htop(p) + call endrun(subgrid_index=p, subgrid_level=subgrid_level_patch, msg=errMsg(sourcefile, __LINE__)) + end if else z0m(p) = htop(p) * (1._r8 - displa(p) / htop(p)) * exp(-0.4_r8 * U_ustar + & log(pftcon%z0v_cw(patch%itype(p))) - 1._r8 + pftcon%z0v_cw(patch%itype(p))**(-1._r8)) From ae8d808e9c33ab6153cd9d83f0dcc9430054fc6d Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 17:18:05 -0700 Subject: [PATCH 094/173] Update testlists: mimics tests to clm51 and remove obsolete Meier2022 test --- cime_config/testdefs/testlist_clm.xml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index d0e2418b56..5f9d86c7ff 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -17,15 +17,6 @@ - - - - - - - - - @@ -532,7 +523,7 @@ - + @@ -2039,7 +2030,7 @@ - + From 5ed438fee20418f61ec015ce8ce0f9db89c809ed Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 20 Nov 2023 17:51:44 -0700 Subject: [PATCH 095/173] Update ChangeLog --- doc/ChangeLog | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 62837f1f0a..648c27e556 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -61,6 +61,11 @@ Changes to the datasets (e.g., parameter, surface or initial files): New clm5_1 params file with new parameters and with modified values of existing parameters. New clm5_0 and clm4_5 params files with new parameters for SNICAR. +Notes of particular relevance for developers: +--------------------------------------------- +Changes to tests or testing: +- Remove clm50 Meier test. Should be clm51 but the compset I1850Clm51BgcNoAnthro does not exist. +- Remove Meier testmod directory. Remove from corresponding tests. Testing summary: ---------------- @@ -69,8 +74,8 @@ Testing summary: regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - cheyenne ---- - izumi ------- + cheyenne ---- OK + izumi ------- OK Answer changes From fc631f4392ced2bad4206dab45e30d65c9db4ddf Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Tue, 21 Nov 2023 13:11:18 -0700 Subject: [PATCH 096/173] Update ExpectedTestFails and testlist --- cime_config/testdefs/ExpectedTestFails.xml | 2 +- cime_config/testdefs/testlist_clm.xml | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 44876ba818..0f8062da0a 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -74,7 +74,7 @@ - + PEND #1045 diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 5f9d86c7ff..85fdd0bb9c 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1167,9 +1167,6 @@ - - - @@ -1408,10 +1405,11 @@ - + + From e8aa4ebbf49b5966310b1b44daf6fbfce38bdf28 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Tue, 21 Nov 2023 13:20:38 -0700 Subject: [PATCH 097/173] Update ChangeLog/ChangeSum --- doc/ChangeLog | 6 ++++-- doc/ChangeSum | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 648c27e556..d9fd062d0d 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev154 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310) -Date: Mon Nov 20 11:09:45 MST 2023 +Date: Tue Nov 21 13:13:57 MST 2023 One-line Summary: New params files: Changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc Purpose and description of changes @@ -65,7 +65,9 @@ Notes of particular relevance for developers: --------------------------------------------- Changes to tests or testing: - Remove clm50 Meier test. Should be clm51 but the compset I1850Clm51BgcNoAnthro does not exist. -- Remove Meier testmod directory. Remove from corresponding tests. +- Remove Meier testmod directory and remove such reference from corresponding tests. +- Change mimics tests from clm50 to clm51. +- For details, see the updated testlist_clm.xml file. Testing summary: ---------------- diff --git a/doc/ChangeSum b/doc/ChangeSum index d86a084cd9..a057281dc5 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev154 slevis 11/20/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc + ctsm5.1.dev154 slevis 11/21/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc ctsm5.1.dev153 afoster 11/17/2023 Call new FATES-side FatesReadParameters ctsm5.1.dev152 multiple 11/14/2023 Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors ctsm5.1.dev151 rgknox 11/11/2023 Fixes to FATES long run restarts From d5785876d773fb63fadd13b2c26399b3df37053a Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 22 Nov 2023 12:46:42 -0700 Subject: [PATCH 098/173] Update ChangeLog/ChangeSum --- doc/ChangeLog | 22 ++++++++++++++++++++-- doc/ChangeSum | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index d9fd062d0d..a1626a9022 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev154 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310) -Date: Tue Nov 21 13:13:57 MST 2023 +Date: Wed Nov 22 09:53:01 MST 2023 One-line Summary: New params files: Changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc Purpose and description of changes @@ -60,6 +60,8 @@ Changes made to namelist defaults (e.g., changed parameter values): Changes to the datasets (e.g., parameter, surface or initial files): New clm5_1 params file with new parameters and with modified values of existing parameters. New clm5_0 and clm4_5 params files with new parameters for SNICAR. + ./rimport on the new params files fails with "No space left on device" but the 4 files are safe here: + /glade/u/home/slevis/paramfiles/*_params.c231117.nc Notes of particular relevance for developers: --------------------------------------------- @@ -88,10 +90,26 @@ Changes answers relative to baseline: YES Summarize any changes to answers, i.e., - what code configurations: ALL - what platforms/compilers: ALL - - nature of change: larger than roundoff/same climate? + - nature of change: + Larger than roundoff/same climate? I will post this tag on the Answer changing tags wiki page as "SIGNIFICANT" and will run a simulation and diagnostics to compare against dev145. + I used the izumi test-suite to perform one bfb sanity test: + I backed up my branch to 6dc1966 (before the snicar mods), then I put back the changes of the + commit right after snicar (71e174f). Comparing to dev154 (this tag's new baseline), + the izumi test-suite passed bfb (12 gnu, 18 intel, and 32 nag tests). + Other mods are quite confined and clear, so I will not pursue other sanity tests. + + Changes to answers commit-by-commit in this PR: + f9978db and b8c71fa: These two change answers for Meier2022 and, therefore, clm51 only + 626f520: Takes out if (z0param_method == 'Meier2022'), so changes answers for all three CLMs + 319d194: Changes answers for mimics and, therefore clm51 only (order of ops, so roundoff) + 2ee6943: Changes clm51 params file, so affects clm51 only (expect more than roundoff) + f185a31: bfb + 6dc1966: This git merge escomp/master probably does change answers from previous commit + 29ca5ad and 71e174f: Puts snicar params on the params files for all three CLMs; sanity test gave bfb + Other details ------------- Pull Requests that document the changes (include PR ids): diff --git a/doc/ChangeSum b/doc/ChangeSum index a057281dc5..2d2e82b420 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev154 slevis 11/21/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc + ctsm5.1.dev154 slevis 11/22/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc ctsm5.1.dev153 afoster 11/17/2023 Call new FATES-side FatesReadParameters ctsm5.1.dev152 multiple 11/14/2023 Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors ctsm5.1.dev151 rgknox 11/11/2023 Fixes to FATES long run restarts From 1d432db7d623da66101a9380a0476a710e796df7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 16 Oct 2023 18:19:34 -0600 Subject: [PATCH 099/173] turn excess ice and meier roughness on by default, add LND_TUNING_MODE for clm5_1_cam6.0, fix a syntax issue from the earlier merge (cherry picked from commit 2746a0d21289dd2a0226ff9727f8c5441f0fe7ee) Conflicts: bld/namelist_files/namelist_defaults_ctsm.xml --- bld/namelist_files/namelist_defaults_ctsm.xml | 4 ++-- bld/namelist_files/namelist_definition_ctsm.xml | 2 +- cime_config/config_component.xml | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 0516ecbc22..a3b8c093c6 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -502,7 +502,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). Meier2022 .true. -.false. +.true. @@ -2763,7 +2763,7 @@ use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa1 -.false. +.true. lnd/clm2/paramdata/exice_init_0.125x0.125_c20220516.nc lnd/clm2/paramdata/exice_init_0.125x0.125_ESMFmesh_cdf5_c20220802.nc diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 2cb95d5c68..97310ebe80 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -2156,7 +2156,7 @@ Land mask description + valid_values="clm4_5_CRUv7,clm4_5_GSWP3v1,clm4_5_cam6.0,clm5_0_cam6.0,clm5_0_CRUv7,clm5_0_GSWP3v1,clm5_1_GSWP3v1,clm5_1_cam6.0"> General configuration of model version and atmospheric forcing to tune the model to run under. This sets the model to run with constants and initial conditions that were set to run well under the configuration of model version and atmospheric forcing. To run well constants would need to be changed diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 743e9229b9..281a94a0b2 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -63,7 +63,7 @@ Tuning parameters and initial conditions should be optimized for what CLM model version and what meteorlogical forcing combination? UNSET - clm5_0_cam6.0,clm5_0_GSWP3v1,clm5_0_CRUv7,clm4_5_CRUv7,clm4_5_GSWP3v1,clm4_5_cam6.0,clm5_1_GSWP3v1 + clm5_0_cam6.0,clm5_0_GSWP3v1,clm5_0_CRUv7,clm4_5_CRUv7,clm4_5_GSWP3v1,clm4_5_cam6.0,clm5_1_GSWP3v1,clm5_1_cam6.0 clm4_5_CRUv7 clm4_5_CRUv7 @@ -76,6 +76,7 @@ clm5_0_cam6.0 clm5_0_cam6.0 clm5_1_GSWP3v1 + clm5_1_cam6.0 From 34368e0c3d03eeac7e002f1656aa6055996dae02 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 19 Oct 2023 00:51:09 -0600 Subject: [PATCH 100/173] Update externals to cesm2_3_beta16 (cherry picked from commit aba49b6f5373d998c1821bb101707d4d31a9ed27) --- Externals.cfg | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index e683f199ac..9e7b4acc31 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -8,7 +8,7 @@ required = True local_path = components/cism protocol = git repo_url = https://github.com/ESCOMP/CISM-wrapper -tag = cismwrap_2_1_95 +tag = cismwrap_2_1_96 externals = Externals_CISM.cfg required = True @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.65 +tag = ccs_config_cesm0.0.82 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -44,18 +44,18 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.125 +tag = cime6.0.175 required = True [cmeps] -tag = cmeps0.14.21 +tag = cmeps0.14.43 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.13 +tag = cdeps1.0.23 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -63,7 +63,7 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl77.0.5 +tag = cpl77.0.7 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 @@ -84,7 +84,7 @@ local_path = libraries/mct required = True [parallelio] -tag = pio2_5_10 +tag = pio2_6_2 protocol = git repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio From 53524623e4d114af4f4bc42fea4599a42e23ddc4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 19 Oct 2023 00:56:14 -0600 Subject: [PATCH 101/173] Add finidat for ne30np4.pg3 from @olyson /glade/p/cgd/tss/people/oleson/CLM5_restarts/ctsm51_cesm23a14a_ne30pg3ne30pg3mg17_CPLHIST_1850pAD.clm2.r.0561-01-01-00000.nc (cherry picked from commit b80808fc123ac5d623db0d4288f40e2aa0c46df8) --- bld/namelist_files/namelist_defaults_ctsm.xml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index a3b8c093c6..d1ab3fe8dc 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -678,6 +678,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). set up. If more finidat files are added you may need to add more of these. Or one specific file will be chosen over another. --> + hgrid=0.9x1.25 maxpft=17 mask=gx1v7 use_cn=.true. use_crop=.false. irrigate=.true. glc_nec=10 do_transient_pfts=.false. @@ -722,6 +723,12 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. + +hgrid=ne30np4.pg3 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. + + hgrid=1.9x2.5 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. @@ -878,7 +885,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). hgrid="ne0np4.ARCTICGRIS.ne30x8" use_cn=".false." maxpft="17" >hgrid=ne0np4.ARCTICGRIS.ne30x8 maxpft=17 mask=tx0.1v2 use_cn=.false. use_crop=.false. irrigate=.true. glc_nec=10 do_transient_pfts=.false. -p + lnd/clm2/initdata_map/clmi.I1850Clm50BgcCrop-ciso.1366-01-01.0.9x1.25_gx1v7_simyr1850_c200428.nc + +lnd/clm2/initdata_map/clmi.I1850Clm51CplhistBgcCrop-ciso.0561-01-01.ne30np4.pg3_gx1v7_simyr1850_c230717.nc + + hgrid=ne30np4.pg3 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. @@ -1283,6 +1282,8 @@ lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne16np4.pg3_hist_78pfts_CMIP6_simy lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg2_hist_78pfts_CMIP6_simyr1850_c200426.nc + +lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg3_hist_78pfts_CMIP6_simyr1850_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg3_hist_78pfts_CMIP6_simyr1850_c200426.nc From e40ca1c4d4b6708389eb5a0753e0782f67f58cf1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 20 Oct 2023 17:48:08 -0600 Subject: [PATCH 112/173] Add PE layout for ne30 on Cheyenne (cherry picked from commit da5beb8003c72eed6684edfaf1f783f2d8cadd15) --- cime_config/config_pes.xml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index 03cea52e7b..90b3fe9efe 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -631,6 +631,43 @@ + + + + none + + -1 + -20 + -20 + -20 + -20 + -20 + -20 + -20 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + From 01abaeea43313fd33ea5e73cc71f8658f26c2eba Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Oct 2023 15:01:31 -0600 Subject: [PATCH 113/173] Add more tests to deal with issue #2219, add a test that turns excess ice off, add tests to expected fails (cherry picked from commit c1cadb377c0458350691eba1426a3a56ae13aed6) --- cime_config/testdefs/ExpectedTestFails.xml | 15 +++++++++++ cime_config/testdefs/testlist_clm.xml | 27 +++++++++++++++++++ .../include_user_mods | 1 + .../clm51cam6LndTuningModeNoExIce/user_nl_clm | 3 +++ 4 files changed, 46 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods create mode 100644 cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 0f8062da0a..47c6ade336 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -28,6 +28,21 @@ --> + + + + FAIL + #2210 + + + + + FAIL + #2210 + + + + diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index f2a95596c3..5db428361a 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -53,6 +53,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods new file mode 100644 index 0000000000..a102434d88 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods @@ -0,0 +1 @@ +../clm51cam6LndTuningMode/ diff --git a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm new file mode 100644 index 0000000000..918670a060 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm @@ -0,0 +1,3 @@ +! Turn off excess ice +use_exice = .true. +use_exice_streams = .true. From 071cf722585be6a8e9989171b206316dfa2cc977 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 5 Nov 2023 01:48:13 -0600 Subject: [PATCH 114/173] Add an LII test using the spunup 1850 ne30pg3_g17 file (cherry picked from commit d6994f733f0bfe004e50a45e9d25ff7cb9a1b15e) --- cime_config/testdefs/testlist_clm.xml | 10 ++++++++++ .../clm/clm51cam6LndTuningModeCiso/include_user_mods | 1 + .../clm/clm51cam6LndTuningModeCiso/user_nl_clm | 5 +++++ 3 files changed, 16 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/include_user_mods create mode 100644 cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/user_nl_clm diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 5db428361a..984a439dd3 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1265,6 +1265,16 @@ + + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/include_user_mods new file mode 100644 index 0000000000..aa76c52034 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/include_user_mods @@ -0,0 +1 @@ +../clm51cam6LndTuningMode diff --git a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/user_nl_clm new file mode 100644 index 0000000000..e7627dea50 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeCiso/user_nl_clm @@ -0,0 +1,5 @@ +! Turn on Carbon isotopes +use_c13 = .true. +use_c14 = .true. +use_c13_timeseries = .true. +use_c14_bombspike = .true. From ab597dd271ca1d99ce3860c9899458043177ae76 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Nov 2023 02:47:16 -0700 Subject: [PATCH 115/173] Adjust tests with tests expected to pass (cherry picked from commit 9882289b28b3876d96d826989e09c9e31b8aa4fa) --- cime_config/testdefs/testlist_clm.xml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 984a439dd3..9d958d0d76 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -524,6 +524,15 @@ + + + + + + + + + @@ -1265,7 +1274,7 @@ - + @@ -1275,7 +1284,7 @@ - + From 6a7c0e7b23f893d589776dc0d64e9d6fedd4c040 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 27 Nov 2023 21:17:43 -0700 Subject: [PATCH 116/173] Update ChangeLog and ChangeSum. --- doc/ChangeLog | 46 ++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 47 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index a1626a9022..0b181a1543 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,50 @@ =============================================================== +Tag name: ctsm5.1.dev155 +Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) +Date: Mon Nov 27 21:16:51 MST 2023 +One-line Summary: Use baset_latvary parameters + +Purpose and description of changes +---------------------------------- + +Namelist parameters baset_latvary_slope and baset_latvary_intercept were never actually used, with values of 0.4 and 12 being hard-coded in the relevant subroutine instead. This PR fixes that, and also adds unit testing of a refactored function that uses them. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- PASS + + +Other details +------------- + +Pull Requests that document the changes (include PR ids): +* #2240: Use baset_latvary parameters (https://github.com/ESCOMP/CTSM/pull/2240) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev154 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310) Date: Wed Nov 22 09:53:01 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 2d2e82b420..65173a9cc1 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev155 samrabin 11/27/2023 Use baset_latvary parameters ctsm5.1.dev154 slevis 11/22/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc ctsm5.1.dev153 afoster 11/17/2023 Call new FATES-side FatesReadParameters ctsm5.1.dev152 multiple 11/14/2023 Mv tools to /python and add tests; add snow_thermal_cond_method; a few fixes / refactors From 16ba2019d8003b21a26d22e544d008eb4dad5485 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Nov 2023 00:47:31 -0700 Subject: [PATCH 117/173] Fix some of the filepaths (cherry picked from commit a4eff78bd07be1895176d13efecc99ff23a6624d) Conflicts: bld/namelist_files/namelist_defaults_ctsm.xml --- bld/namelist_files/namelist_defaults_ctsm.xml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2893784dac..67d934345d 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1282,8 +1282,6 @@ lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne16np4.pg3_hist_78pfts_CMIP6_simy lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg2_hist_78pfts_CMIP6_simyr1850_c200426.nc - -lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg3_hist_78pfts_CMIP6_simyr1850_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg3_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1431,9 +1429,9 @@ use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa1 lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4.pg2_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc + >lnd/clm2/surfdata_esmf/ctsm5.2.0/landuse.timeseries_ne30np4_SSP5-8.5_78_CMIP6_1850-2100_c230530.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTICGRIS.ne30x8_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc + >lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTICGRIS.ne30x8_SSP5-8.5_78_CMIP6_1850-2100_c230530.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTIC.ne30x4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc Date: Thu, 9 Nov 2023 00:49:15 -0700 Subject: [PATCH 118/173] Add issue number to expected fails (cherry picked from commit 4e091304e60250c8f9415619cbb04797162df2c0) Conflicts: cime_config/testdefs/ExpectedTestFails.xml --- cime_config/testdefs/ExpectedTestFails.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 47c6ade336..d30088c125 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -38,11 +38,10 @@ FAIL - #2210 + #2234 - From c48a334b7f6f63fca05062ce94f33402bb88cd87 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Nov 2023 15:16:29 -0700 Subject: [PATCH 119/173] Update baseline directory for Derecho (cherry picked from commit b916d3875da71ca48b8099133dbdfe358eec67d6) --- python/ctsm/machine_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index 305c86e03c..ea1a4db45b 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -69,7 +69,7 @@ "derecho": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, scratch_dir=os.path.join(os.path.sep, "glade", "derecho", "scratch", get_user()), - baseline_dir=os.path.join(os.path.sep, "glade", "p", "cgd", "tss", "ctsm_baselines"), + baseline_dir=os.path.join(os.path.sep, "glade", "campaign", "cgd", "tss", "ctsm_baselines"), account_required=True, create_test_retry=0, create_test_queue="regular", From 79f64df1f02cf5f5aeb8562269799e8f68e80325 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Nov 2023 17:31:18 -0700 Subject: [PATCH 120/173] Change queue and wallclock limit for Derecho (cherry picked from commit 863ed22b5a490bc63b83adf03b8f5bb788fa0e76) --- python/ctsm/machine_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index ea1a4db45b..18f4ff4752 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -75,8 +75,8 @@ create_test_queue="regular", job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( - queue="regular", - walltime="11:50:00", + queue="main", + walltime="03:50:00", extra_args="", # The following assumes a single node, with a single mpi proc; we may want # to add more flexibility in the future, making the node / proc counts From 0e8caee16565eaf2d4e610b65a582ed0b79ddeec Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Nov 2023 21:19:22 -0700 Subject: [PATCH 121/173] Add 2000 and HIST tests for ne30np4.pg3 to cesm3_dev testlist, remove the PEM test that is no longer needed and doesn't have a testmod for it (cherry picked from commit 04853a83c55ec77d954149555923a3c0dd5dcd77) --- cime_config/testdefs/testlist_clm.xml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 9d958d0d76..876c834d1e 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -53,22 +53,22 @@ - + - + - + - + @@ -98,6 +98,15 @@ + + + + + + + + + From 0fa6ebcb150bec66464e7100bf3c85c7583508a9 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Nov 2023 23:30:28 -0700 Subject: [PATCH 122/173] Get finidat file working for 2000 ne30np4.pg3 clm5_1_cam6.0 working (cherry picked from commit 88ac3ae9a7e01515bc202b9efc91ed7e6d515df8) --- bld/namelist_files/namelist_defaults_ctsm.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 67d934345d..a8e1f20221 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -826,6 +826,12 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + + lnd/clm2/initdata_map/clmi.BHIST.2000-01-01.0.9x1.25_gx1v7_simyr2000_c200728.nc +lnd/clm2/initdata_map/clmi.BHIST.2000-01-01.0.9x1.25_gx1v7_simyr2000_c200728.nc + + Date: Fri, 10 Nov 2023 00:20:22 -0700 Subject: [PATCH 123/173] Update ne30np4.pg3 1850 finidat file so LII tests work, and add in ne30np4.pg3 landuse.timeseries file (cherry picked from commit d3a82a568dded619a2ed2012a33048fbbe7372df) Conflicts: bld/namelist_files/namelist_defaults_ctsm.xml --- bld/namelist_files/namelist_defaults_ctsm.xml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index a8e1f20221..ac80b1b336 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -988,7 +988,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). ic_ymd="18500101" sim_year="1850" do_transient_pfts=".false." ic_tod="0" glc_nec="10" use_crop=".true." irrigate=".false." lnd_tuning_mode="clm5_1_cam6.0" ->lnd/clm2/initdata_map/clmi.I1850Clm51CplhistBgcCrop-ciso.0561-01-01.ne30np4.pg3_gx1v7_simyr1850_c230717.nc +>lnd/clm2/initdata_map/clmi.I1850Clm51CplhistBgcCrop-ciso.0561-01-01.ne30np4.pg3_gx1v7_simyr1850_c231109.nc @@ -1442,7 +1444,7 @@ use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa1 lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4.pg2_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc lnd/clm2/surfdata_esmf/ctsm5.2.0/landuse.timeseries_ne30np4_SSP5-8.5_78_CMIP6_1850-2100_c230530.nc + >lnd/clm2/surfdata_esmf/ctsm5.2.0/landuse.timeseries_ne30np4.pg3_SSP2-4.5_1850-2100_78pft_c231026.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTICGRIS.ne30x8_SSP5-8.5_78_CMIP6_1850-2100_c230530.nc Date: Fri, 10 Nov 2023 09:29:42 -0700 Subject: [PATCH 124/173] Actually turn Meier off (cherry picked from commit 1a09d07801e05c48aab90a30b1b1fe679fad8184) Conflicts: cime_config/testdefs/testmods_dirs/clm/ciso_decStartNoMeier/user_nl_clm --- cime_config/testdefs/testlist_clm.xml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 876c834d1e..03a4d0c91b 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -533,7 +533,7 @@ - + @@ -1283,17 +1283,25 @@ - + - - + - + + + + + + + + + + From 82800f05e04b8e61722fd7b3eb728a2b65019c54 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 09:15:54 -0700 Subject: [PATCH 125/173] Correct some namelist settings --- bld/namelist_files/namelist_defaults_ctsm.xml | 8 ++++---- .../clm/clm51cam6LndTuningModeNoExIce/include_user_mods | 1 - .../clm/clm51cam6LndTuningModeNoExIce/user_nl_clm | 3 --- 3 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods delete mode 100644 cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index ac80b1b336..7c5d19eeaa 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -502,7 +502,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). Meier2022 .true. -.true. +.false. @@ -1444,9 +1444,9 @@ use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa1 lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4.pg2_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc lnd/clm2/surfdata_esmf/ctsm5.2.0/landuse.timeseries_ne30np4.pg3_SSP2-4.5_1850-2100_78pft_c231026.nc + >lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTICGRIS.ne30x8_SSP5-8.5_78_CMIP6_1850-2100_c230530.nc + >lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTICGRIS.ne30x8_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne0np4.ARCTIC.ne30x4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa1 -.true. +.false. lnd/clm2/paramdata/exice_init_0.125x0.125_c20220516.nc lnd/clm2/paramdata/exice_init_0.125x0.125_ESMFmesh_cdf5_c20220802.nc diff --git a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods deleted file mode 100644 index a102434d88..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/include_user_mods +++ /dev/null @@ -1 +0,0 @@ -../clm51cam6LndTuningMode/ diff --git a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm deleted file mode 100644 index 918670a060..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/clm51cam6LndTuningModeNoExIce/user_nl_clm +++ /dev/null @@ -1,3 +0,0 @@ -! Turn off excess ice -use_exice = .true. -use_exice_streams = .true. From 8f3327cd5bc1c3d8a05d30c6477c9d8bf3ceb2db Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 10:20:15 -0700 Subject: [PATCH 126/173] The ne30pg3 finidat file needs to do interpolation to get started since it was created with ctsm5.2 datasets --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 7c5d19eeaa..60366149c0 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -987,7 +987,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/initdata_map/clmi.I1850Clm51CplhistBgcCrop-ciso.0561-01-01.ne30np4.pg3_gx1v7_simyr1850_c231109.nc From 30361835aaed1d6c6a828797d42f8b55af9efc1d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 11:54:16 -0700 Subject: [PATCH 127/173] Remove the ne30pg3 LII tests because they will fail as use_init_interp is required because the finidat file is based on ctsm5.2 datasets --- cime_config/testdefs/testlist_clm.xml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 03a4d0c91b..cbef667b93 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1283,15 +1283,6 @@ - - - - - - - - - @@ -1301,15 +1292,6 @@ - - - - - - - - - From ebb3e64e389c2d407147034070b7456879d327b2 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 11:55:08 -0700 Subject: [PATCH 128/173] Add use_init_interp to the ne30pg3 finidat file, because it's needed for non ctsm5.2 surface datasets --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 60366149c0..7c5d19eeaa 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -987,7 +987,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/initdata_map/clmi.I1850Clm51CplhistBgcCrop-ciso.0561-01-01.ne30np4.pg3_gx1v7_simyr1850_c231109.nc From 98cb379df0d6bb53f41455f11a74ee7ce70fb86c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 12:33:15 -0700 Subject: [PATCH 129/173] Set init_interp_fill_missing_with_natveg for the ne30np4.pg3 grid --- bld/CLMBuildNamelist.pm | 5 +++++ bld/namelist_files/namelist_defaults_ctsm.xml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 17d64cd59f..9a4dadfd6c 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -4356,6 +4356,11 @@ sub setup_logic_initinterp { $log->fatal_error("$var can only be set if use_init_interp is true"); } } + # This only seems to be important for this one grid... + if ( $nl_flags->{'res'} eq "ne30np4.pg3" ) { + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, "init_interp_fill_missing_with_natveg", + 'hgrid'=>$nl_flags->{'res'} ); + } } #------------------------------------------------------------------------------- diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 7c5d19eeaa..e6b35380f9 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -903,6 +903,8 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=ne0np4CONUS.ne30x8 maxpft=17 mask=tx0.1v2 use_cn=.false. use_crop=.false. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + +.true. - - - FAIL - #2210 - - - - + + FAIL - #2234 + #2268 From 9f457dbadc6a01d6b8ee98c9fa7c995941ad0585 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 16:10:31 -0700 Subject: [PATCH 131/173] Add a couple cesm3_dev tests to aux_clm --- cime_config/testdefs/testlist_clm.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index cbef667b93..8ffae44d6d 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -47,6 +47,7 @@ + @@ -77,12 +78,13 @@ - + + From 3a3a97eec36cd82d7b30c0e1ea17e35a8d69beb8 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 16:27:15 -0700 Subject: [PATCH 132/173] PE layouts for f45 and f10 for Derecho --- cime_config/config_pes.xml | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index 90b3fe9efe..60c7355825 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -113,6 +113,43 @@ + + + + none + + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + @@ -1336,6 +1373,44 @@ + + + + none + + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + From 651ff5c7b3c5426bb3cc2fc3b14445ef606c0463 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 19:14:57 -0700 Subject: [PATCH 133/173] Lengthen test time --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 8ffae44d6d..bb073be374 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -2032,7 +2032,7 @@ - + From a8c535a976daf87832ba4b72b91f0f67deea9ba5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 28 Nov 2023 19:28:24 -0700 Subject: [PATCH 134/173] Remove the nvhpc tests that are now passing --- cime_config/testdefs/ExpectedTestFails.xml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 067f54b8c0..f4b4ff6f5d 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -50,24 +50,9 @@ #1733 - - - - FAIL - #1733 - - - - - - FAIL - #1733 - - - + - FAIL From 10b3985ec55e07495c13a25692cbb86fa6529f66 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 29 Nov 2023 11:21:41 -0700 Subject: [PATCH 135/173] Update Externals.cfg to version from CESM3_dev aba49b6. --- Externals.cfg | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index e683f199ac..f5243f9166 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -8,7 +8,7 @@ required = True local_path = components/cism protocol = git repo_url = https://github.com/ESCOMP/CISM-wrapper -tag = cismwrap_2_1_95 +tag = cismwrap_2_1_96 externals = Externals_CISM.cfg required = True @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.65 +tag = ccs_config_cesm0.0.82 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -44,18 +44,18 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.125 +tag = cime6.0.175 required = True [cmeps] -tag = cmeps0.14.21 +tag = cmeps0.14.43 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.13 +tag = cdeps1.0.23 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -63,7 +63,7 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl77.0.5 +tag = cpl77.0.7 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 @@ -84,7 +84,7 @@ local_path = libraries/mct required = True [parallelio] -tag = pio2_5_10 +tag = pio2_6_2 protocol = git repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio @@ -98,4 +98,4 @@ tag = v1.0.8 required = False [externals_description] -schema_version = 1.0.0 +schema_version = 1.0.0 \ No newline at end of file From f85fea88f5ee68cf628640ad37592f472206e9bc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 30 Nov 2023 00:22:59 -0700 Subject: [PATCH 136/173] First pass at adding derecho tests, also change the syntax for ones with options per machine to options after machine lists, and add intel-oneapi and nvhpc tests to test lists with clm_short --- cime_config/testdefs/testlist_clm.xml | 931 ++++++++++++++++++++++---- 1 file changed, 807 insertions(+), 124 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index bb073be374..33e1f000fa 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -3,6 +3,7 @@ + @@ -12,14 +13,18 @@ + + + + @@ -29,6 +34,7 @@ + @@ -39,6 +45,8 @@ + + @@ -112,6 +120,7 @@ + @@ -121,6 +130,7 @@ + @@ -130,6 +140,7 @@ + @@ -146,6 +157,7 @@ + @@ -155,6 +167,7 @@ + @@ -164,6 +177,7 @@ + @@ -173,6 +187,7 @@ + @@ -182,6 +197,7 @@ + @@ -192,6 +208,8 @@ + + @@ -235,6 +253,7 @@ + @@ -250,6 +269,15 @@ + + + + + + + + + @@ -259,6 +287,15 @@ + + + + + + + + + @@ -267,6 +304,14 @@ + + + + + + + + @@ -275,9 +320,18 @@ + + + + + + + + + @@ -287,6 +341,8 @@ + + @@ -295,6 +351,7 @@ + @@ -303,6 +360,7 @@ + @@ -312,6 +370,7 @@ + @@ -320,6 +379,7 @@ + @@ -328,6 +388,7 @@ + @@ -337,6 +398,8 @@ + + @@ -351,9 +414,19 @@ + + + + + + + + + + @@ -363,6 +436,7 @@ + @@ -372,6 +446,7 @@ + @@ -395,9 +470,19 @@ + + + + + + + + + + @@ -407,6 +492,7 @@ + @@ -416,6 +502,7 @@ + @@ -425,6 +512,7 @@ + @@ -434,6 +522,7 @@ + @@ -443,6 +532,8 @@ + + @@ -451,6 +542,7 @@ + @@ -461,6 +553,8 @@ + + @@ -469,6 +563,7 @@ + @@ -477,6 +572,7 @@ + @@ -486,6 +582,8 @@ + + @@ -494,6 +592,7 @@ + @@ -503,6 +602,7 @@ + @@ -512,6 +612,7 @@ + @@ -535,6 +636,15 @@ + + + + + + + + + @@ -544,9 +654,19 @@ + + + + + + + + + + @@ -573,6 +693,8 @@ + + @@ -580,6 +702,19 @@ + + + + + + + + + + + + + @@ -588,6 +723,14 @@ + + + + + + + + @@ -597,6 +740,15 @@ + + + + + + + + + @@ -623,6 +775,14 @@ + + + + + + + + @@ -631,6 +791,14 @@ + + + + + + + + @@ -639,6 +807,14 @@ + + + + + + + + @@ -648,6 +824,15 @@ + + + + + + + + + @@ -657,6 +842,15 @@ + + + + + + + + + @@ -666,6 +860,15 @@ + + + + + + + + + @@ -675,10 +878,21 @@ + + + + + + + + + + + @@ -697,6 +911,7 @@ + @@ -706,6 +921,8 @@ + + @@ -714,6 +931,7 @@ + @@ -722,6 +940,7 @@ + @@ -738,6 +957,7 @@ + @@ -747,6 +967,7 @@ + @@ -786,9 +1007,19 @@ + + + + + + + + + + @@ -798,6 +1029,7 @@ + @@ -807,6 +1039,7 @@ + @@ -822,6 +1055,15 @@ + + + + + + + + + @@ -830,6 +1072,14 @@ + + + + + + + + @@ -839,6 +1089,15 @@ + + + + + + + + + @@ -848,9 +1107,19 @@ + + + + + + + + + + @@ -860,6 +1129,7 @@ + @@ -869,6 +1139,7 @@ + @@ -878,6 +1149,7 @@ + @@ -887,6 +1159,7 @@ + @@ -895,6 +1168,7 @@ + @@ -904,6 +1178,7 @@ + @@ -920,6 +1195,16 @@ + + + + + + + + + + @@ -930,6 +1215,16 @@ + + + + + + + + + + @@ -940,6 +1235,16 @@ + + + + + + + + + + @@ -949,6 +1254,15 @@ + + + + + + + + + @@ -957,6 +1271,14 @@ + + + + + + + + @@ -966,6 +1288,15 @@ + + + + + + + + + @@ -976,6 +1307,16 @@ + + + + + + + + + + @@ -984,6 +1325,14 @@ + + + + + + + + @@ -993,6 +1342,15 @@ + + + + + + + + + @@ -1003,6 +1361,16 @@ + + + + + + + + + + @@ -1013,9 +1381,20 @@ + + + + + + + + + + + @@ -1030,9 +1409,19 @@ + + + + + + + + + + @@ -1042,6 +1431,7 @@ + @@ -1051,6 +1441,7 @@ + @@ -1060,6 +1451,7 @@ + @@ -1075,12 +1467,27 @@ + + + + + + + + + + + + + + + @@ -1089,6 +1496,7 @@ + @@ -1098,6 +1506,7 @@ + @@ -1106,6 +1515,7 @@ + @@ -1115,6 +1525,8 @@ + + @@ -1132,6 +1544,7 @@ + @@ -1186,6 +1599,7 @@ + @@ -1210,6 +1624,15 @@ + + + + + + + + + @@ -1219,6 +1642,15 @@ + + + + + + + + + @@ -1228,9 +1660,19 @@ + + + + + + + + + + @@ -1241,6 +1683,8 @@ + + @@ -1250,6 +1694,7 @@ + @@ -1279,6 +1724,7 @@ + @@ -1294,9 +1740,19 @@ + + + + + + + + + + @@ -1307,6 +1763,8 @@ + + @@ -1317,6 +1775,8 @@ + + @@ -1326,6 +1786,8 @@ + + @@ -1350,10 +1812,21 @@ + + + + + + + + + + + @@ -1363,6 +1836,7 @@ + @@ -1373,6 +1847,8 @@ + + @@ -1381,6 +1857,7 @@ + @@ -1389,6 +1866,7 @@ + @@ -1399,6 +1877,7 @@ + @@ -1408,6 +1887,7 @@ + @@ -1417,6 +1897,7 @@ + @@ -1426,6 +1907,7 @@ + @@ -1435,6 +1917,7 @@ + @@ -1444,6 +1927,7 @@ + @@ -1453,6 +1937,7 @@ + @@ -1462,6 +1947,7 @@ + @@ -1471,6 +1957,7 @@ + @@ -1481,6 +1968,8 @@ + + @@ -1501,9 +1990,13 @@ + + + + @@ -1519,9 +2012,19 @@ - + + + + + + + + + + + - + @@ -1543,6 +2046,12 @@ + + + + + + @@ -1553,6 +2062,9 @@ + + + @@ -1561,6 +2073,7 @@ + @@ -1570,6 +2083,8 @@ + + @@ -1588,11 +2103,13 @@ + + - + @@ -1637,6 +2154,17 @@ + + + + + + + + + + + @@ -1663,6 +2191,7 @@ + @@ -1673,6 +2202,7 @@ + @@ -1690,23 +2220,23 @@ - - - - - - + + + + + + - - - - - - + + + + + + @@ -1721,6 +2251,7 @@ + @@ -1729,9 +2260,11 @@ + + @@ -1746,6 +2279,8 @@ + + @@ -1755,6 +2290,7 @@ + @@ -1781,6 +2317,8 @@ + + @@ -1790,6 +2328,7 @@ + @@ -1799,6 +2338,7 @@ + @@ -1808,6 +2348,7 @@ + @@ -1827,6 +2368,7 @@ + @@ -1837,6 +2379,8 @@ + + @@ -1863,6 +2407,7 @@ + @@ -1872,6 +2417,7 @@ + @@ -1880,6 +2426,7 @@ + @@ -1889,6 +2436,8 @@ + + @@ -1897,6 +2446,7 @@ + @@ -1906,6 +2456,7 @@ + @@ -1914,6 +2465,7 @@ + @@ -1923,6 +2475,7 @@ + @@ -1932,6 +2485,7 @@ + @@ -1941,6 +2495,7 @@ + @@ -1960,6 +2515,8 @@ + + @@ -1969,6 +2526,7 @@ + @@ -1977,6 +2535,7 @@ + @@ -1986,6 +2545,7 @@ + @@ -1995,6 +2555,8 @@ + + @@ -2012,6 +2574,7 @@ + @@ -2022,6 +2585,8 @@ + + @@ -2035,12 +2600,23 @@ + + + + + + + + + + + @@ -2049,6 +2625,7 @@ + @@ -2057,6 +2634,7 @@ + @@ -2066,6 +2644,7 @@ + @@ -2076,6 +2655,7 @@ + @@ -2086,6 +2666,7 @@ + @@ -2096,6 +2677,7 @@ + @@ -2112,6 +2694,16 @@ + + + + + + + + + + @@ -2146,15 +2738,24 @@ + + + + + + + + + - - - - - - + + + + + + @@ -2168,33 +2769,40 @@ - + + + + + + + - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + @@ -2210,9 +2818,19 @@ + + + + + + + + + + @@ -2223,6 +2841,7 @@ + @@ -2233,6 +2852,7 @@ + @@ -2244,6 +2864,7 @@ + @@ -2255,6 +2876,8 @@ + + @@ -2265,6 +2888,7 @@ + @@ -2275,6 +2899,7 @@ + @@ -2286,8 +2911,10 @@ + + @@ -2298,6 +2925,7 @@ + @@ -2308,6 +2936,7 @@ + @@ -2319,6 +2948,7 @@ + @@ -2329,6 +2959,7 @@ + @@ -2339,6 +2970,7 @@ + @@ -2349,6 +2981,7 @@ + @@ -2359,6 +2992,7 @@ + @@ -2368,6 +3002,7 @@ + @@ -2377,6 +3012,7 @@ + @@ -2386,6 +3022,7 @@ + @@ -2394,6 +3031,7 @@ + @@ -2403,6 +3041,7 @@ + @@ -2412,6 +3051,7 @@ + @@ -2422,6 +3062,7 @@ + @@ -2431,6 +3072,7 @@ + @@ -2440,6 +3082,7 @@ + @@ -2457,9 +3100,20 @@ + + + + + + + + + + + @@ -2469,6 +3123,7 @@ + @@ -2479,6 +3134,7 @@ + @@ -2490,6 +3146,8 @@ + + @@ -2499,6 +3157,7 @@ + @@ -2510,6 +3169,9 @@ + + + @@ -2519,6 +3181,7 @@ + @@ -2528,97 +3191,82 @@ - - - - - - + + + + + + - - - - - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + @@ -2631,6 +3279,15 @@ + + + + + + + + + @@ -2641,6 +3298,15 @@ + + + + + + + + + @@ -2651,47 +3317,64 @@ + + + + + + + + + - - - - - - + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + From 5c8f17b1a3badc4699882cae3c408c100a0e6a3f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 30 Nov 2023 00:29:36 -0700 Subject: [PATCH 137/173] Work on f09 f19 and nldas pe-layouts for Derecho --- cime_config/config_pes.xml | 233 +++++++++++++++++++++++++++++++++---- 1 file changed, 209 insertions(+), 24 deletions(-) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index 60c7355825..34fd593425 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -298,6 +298,80 @@ + + + + none + + -1 + -12 + -12 + -12 + -12 + -12 + -12 + -12 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + Much lower core count f19 layout, mainly for testing + + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + @@ -340,34 +414,34 @@ none - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -413,6 +487,43 @@ none + + -1 + -14 + -14 + -14 + -14 + -14 + -14 + -14 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + Much lower core count f09 layout, mainly for testing -1 -5 @@ -1560,5 +1671,79 @@ + + + + none + + -1 + -14 + -14 + -14 + -14 + -14 + -14 + -14 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + + + + + + Much lower core count nldas2 layout, mainly for testing + + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + From b24e5f512ef85a63b0151b2fd8d511017b5d13b4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 30 Nov 2023 12:32:29 -0700 Subject: [PATCH 138/173] Do not use Meier roughness by default, even with 5.1. --- bld/namelist_files/namelist_defaults_ctsm.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 0516ecbc22..1be3827d1b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -499,7 +499,6 @@ attributes from the config_cache.xml file (with keys converted to upper-case). ZengWang2007 -Meier2022 .true. .false. From da4af8ed95522d589693b2974f79e499f1ce923c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 30 Nov 2023 15:27:55 -0700 Subject: [PATCH 139/173] Update ChangeLog and ChangeSum. --- doc/ChangeLog | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 65 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 0b181a1543..aabefea0fe 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,68 @@ =============================================================== +Tag name: ctsm5.1.dev156 +Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) +Date: Thu Nov 30 15:27:18 MST 2023 +One-line Summary: Do not use Meier roughness by default + +Purpose and description of changes +---------------------------------- + +ctsm5.1.dev155 had turned on Meier2022 surface roughness calculation by default for 5.1 compsets. Several bugs have recently emerged that were not caught by pre-merge testing, so this tag reverts that change. Thus, the ZengWang2007 method is default for all compsets again. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[X] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Notes of particular relevance for users +--------------------------------------- + +Changes made to namelist defaults (e.g., changed parameter values): 5.1 compsets now use ZengWang2007 method (instead of Meier2022) for roughness calculation. + + +Testing summary: +---------------- + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- PASS + + +Answer changes +-------------- + +Changes answers relative to baseline: YES + + Summarize any changes to answers, i.e., + - what code configurations: 5.1 compsets + - what platforms/compilers: All + - nature of change (roundoff; larger than roundoff/same climate; new climate): new climate + + No climate-evaluating run performed, as this change is reverting part of a commit thats barely a week old. + + +Other details +------------- +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) +* #2273: Do not use Meier roughness by default, even with 5.1. (https://github.com/ESCOMP/CTSM/pull/2273) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev155 Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) Date: Mon Nov 27 21:16:51 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 65173a9cc1..d891a68d87 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev156 samrabin 11/30/2023 Do not use Meier roughness by default. ctsm5.1.dev155 samrabin 11/27/2023 Use baset_latvary parameters ctsm5.1.dev154 slevis 11/22/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc ctsm5.1.dev153 afoster 11/17/2023 Call new FATES-side FatesReadParameters From bdf616c3aa2904bfc66f8585471c80dafe8a988c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 30 Nov 2023 16:16:22 -0700 Subject: [PATCH 140/173] Correct ChangeLog description of aux_clm results for ctsm5.1.dev156. --- doc/ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index aabefea0fe..80ddf5be12 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -38,8 +38,8 @@ Testing summary: regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - cheyenne ---- OK - izumi ------- PASS + cheyenne ---- DIFF + izumi ------- DIFF Answer changes From f65c99ca079726e62c84ca040f0f38f25228ab2a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 00:14:00 -0700 Subject: [PATCH 141/173] Update externals for ccs_config which is needed for mpi-serial tests --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 9e7b4acc31..cf2ff73498 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.82 +tag = ccs_config_cesm0.0.84 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config From 4ce2a9d98fccbe38664659a026c5c1a64194429a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 00:15:29 -0700 Subject: [PATCH 142/173] The same intel tests are failing on Derecho as were on Cheyenne --- cime_config/testdefs/ExpectedTestFails.xml | 29 +++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index f4b4ff6f5d..55b47dc4c9 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -50,10 +50,17 @@ #1733 + + + + FAIL + #1733 + + - + FAIL FATES#701 @@ -67,6 +74,13 @@ + + + FAIL + FATES#701 + + + PEND @@ -74,6 +88,13 @@ + + + PEND + #1045 + + + PEND @@ -89,5 +110,11 @@ + + + FAIL + FATES#1089 + + From 14d666e4b8120c4491ca19ac3d8f67f8fdbc4f45 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 00:34:46 -0700 Subject: [PATCH 143/173] Fix pe-layouts for Derecho, remove new nvhpc tests on Derecho that weren't in Cheyenne, and remove the intel-oneapi tests since they fail --- cime_config/testdefs/testlist_clm.xml | 94 ++++++++++++--------------- 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 33e1f000fa..1a1bfed7e3 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -304,7 +304,7 @@ - + @@ -320,7 +320,7 @@ - + @@ -414,7 +414,7 @@ - + @@ -636,7 +636,7 @@ - + @@ -654,7 +654,7 @@ - + @@ -693,7 +693,6 @@ - @@ -702,11 +701,10 @@ - + - @@ -723,7 +721,7 @@ - + @@ -740,7 +738,7 @@ - + @@ -775,7 +773,7 @@ - + @@ -791,7 +789,7 @@ - + @@ -807,7 +805,7 @@ - + @@ -824,7 +822,7 @@ - + @@ -842,7 +840,7 @@ - + @@ -860,7 +858,7 @@ - + @@ -878,7 +876,7 @@ - + @@ -1007,7 +1005,7 @@ - + @@ -1055,7 +1053,7 @@ - + @@ -1072,7 +1070,7 @@ - + @@ -1089,7 +1087,7 @@ - + @@ -1107,7 +1105,7 @@ - + @@ -1195,7 +1193,7 @@ - + @@ -1215,7 +1213,7 @@ - + @@ -1254,7 +1252,7 @@ - + @@ -1288,7 +1286,7 @@ - + @@ -1307,7 +1305,7 @@ - + @@ -1325,7 +1323,7 @@ - + @@ -1342,7 +1340,7 @@ - + @@ -1361,7 +1359,7 @@ - + @@ -1381,7 +1379,7 @@ - + @@ -1409,7 +1407,7 @@ - + @@ -1467,7 +1465,7 @@ - + @@ -1483,8 +1481,6 @@ - - @@ -1624,7 +1620,7 @@ - + @@ -1642,7 +1638,7 @@ - + @@ -1660,7 +1656,7 @@ - + @@ -1812,7 +1808,7 @@ - + @@ -1991,8 +1987,6 @@ - - @@ -2022,7 +2016,7 @@ - + @@ -2047,8 +2041,6 @@ - - @@ -2154,7 +2146,7 @@ - + @@ -2162,7 +2154,7 @@ - + @@ -2738,7 +2730,7 @@ - + @@ -2769,7 +2761,7 @@ - + @@ -3279,7 +3271,7 @@ - + @@ -3298,7 +3290,7 @@ - + @@ -3317,7 +3309,7 @@ - + From 84bab54dcd6d7e93fc0323cd5b48c7f5b824a7bf Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 09:25:59 -0700 Subject: [PATCH 144/173] Correct expected fail --- cime_config/testdefs/ExpectedTestFails.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 55b47dc4c9..9a14b7e8da 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -88,7 +88,7 @@ - + PEND #1045 From 15c9527f5ab79acdfc41606a59c8d293b483feac Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 09:55:41 -0700 Subject: [PATCH 145/173] Add two expected fails because of a build issue presumably in ccs_config --- cime_config/testdefs/ExpectedTestFails.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 9a14b7e8da..1d83bb2e7a 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -95,6 +95,20 @@ + + + FAIL + ESMCI/ccs_config_cesm#130 + + + + + + FAIL + ESMCI/ccs_config_cesm#130 + + + PEND From 11e706cc617750609b06759f9fd19aaabf649294 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 10:04:40 -0700 Subject: [PATCH 146/173] Add more mpi-serial intel debug failed tests --- cime_config/testdefs/ExpectedTestFails.xml | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 1d83bb2e7a..bfd87c2fd9 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -57,6 +57,34 @@ #1733 + + + + FAIL + ESMCI/ccs_config_cesm#130 + + + + + + FAIL + ESMCI/ccs_config_cesm#130 + + + + + + FAIL + ESMCI/ccs_config_cesm#130 + + + + + + FAIL + ESMCI/ccs_config_cesm#130 + + From 24c088c8e9103bd639d0fece4674c4ea44b03393 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 10:38:31 -0700 Subject: [PATCH 147/173] Remove two nvhpc tests that fail, one from Cheyenne that wasn't there before and the other from Derecho --- cime_config/testdefs/testlist_clm.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 1a1bfed7e3..86dde5bebd 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -693,7 +693,6 @@ - @@ -705,7 +704,6 @@ - From 6c5c24d193f714321a02e8296b563dbef6652442 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 14:11:35 -0700 Subject: [PATCH 148/173] Add NEON tests to Derecho --- cime_config/testdefs/testlist_clm.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 86dde5bebd..8247236db2 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -2104,6 +2104,7 @@ + @@ -2115,6 +2116,7 @@ + @@ -2125,6 +2127,7 @@ + @@ -2158,6 +2161,8 @@ + + @@ -2171,6 +2176,8 @@ + + From 7ccd479e9f89b9dd755448037fa77b7c5db68c13 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 1 Dec 2023 14:12:25 -0700 Subject: [PATCH 149/173] Change some of the comments about Cheyenne to also include Derecho --- python/conda_env_ctsm_py.txt | 5 +++-- python/conda_env_ctsm_py_cgd.txt | 1 - python/ctsm/test/testinputs/README.md | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/conda_env_ctsm_py.txt b/python/conda_env_ctsm_py.txt index 0ac4656e73..0306a9ca06 100644 --- a/python/conda_env_ctsm_py.txt +++ b/python/conda_env_ctsm_py.txt @@ -1,7 +1,8 @@ # -# NOTE: Changes here should be coordinated with the cgd python environment file +# NOTE: Changes here should be coordinated with the cgd python environment file +# +# NOTE: Derecho already has conda installed for you, so you just need to do the following... # -# To install this on cheyenne with conda loaded in modules # use the top level bash script: # ../py_env_create # Do this each time you update your CTSM Version # conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script diff --git a/python/conda_env_ctsm_py_cgd.txt b/python/conda_env_ctsm_py_cgd.txt index 8312dc328a..3afcf4bba2 100644 --- a/python/conda_env_ctsm_py_cgd.txt +++ b/python/conda_env_ctsm_py_cgd.txt @@ -3,7 +3,6 @@ # # This should be coordinated with the main python environment file! # -# To install this on cheyenne with conda loaded in modules # use the top level bash script: # ../py_env_create # Do this each time you update your CTSM Version # conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script diff --git a/python/ctsm/test/testinputs/README.md b/python/ctsm/test/testinputs/README.md index 1775de48b7..ef8953d20e 100644 --- a/python/ctsm/test/testinputs/README.md +++ b/python/ctsm/test/testinputs/README.md @@ -6,6 +6,7 @@ Installing Git LFS on your machine is a two-step process; step (1) needs to be done once per machine, and step (2) needs to be done once per user: 1. Install the Git LFS tool: Follow the instructions on the [Git LFS page](https://git-lfs.github.com/) for installing Git LFS on your platform. + - On derecho the system default version of git already has Git LFS installed. - On cheyenne and casper, Git LFS is already available as long as you are using a git module rather than the default system-level git. So just make sure that you are always using git via a git module (`module load git`). From 1a2f5e8a1a20796953fa4b16d348fd4c25d28e0f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 2 Dec 2023 19:49:20 -0700 Subject: [PATCH 150/173] Move the Derecho intel/DEBUG tests to gnu keeping only one that's expected to fail --- cime_config/testdefs/testlist_clm.xml | 40 ++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 8247236db2..2e76ca5d66 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1984,11 +1984,15 @@ + + + - @@ -2104,7 +2108,10 @@ + + @@ -2116,7 +2123,10 @@ + + @@ -2127,7 +2137,10 @@ + + @@ -2161,8 +2174,12 @@ + + + @@ -2176,8 +2193,12 @@ + + + @@ -2188,7 +2209,10 @@ + + + @@ -2552,8 +2576,12 @@ + + + @@ -2571,7 +2599,10 @@ + + @@ -2896,7 +2927,10 @@ + + @@ -2922,7 +2956,10 @@ + + @@ -3201,6 +3238,7 @@ + From a6e3db81f791e99a95e6c3793c2e9fbaa9ca6f4c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 2 Dec 2023 22:46:04 -0700 Subject: [PATCH 151/173] Add FUNITCTSM to the list of expected fails --- cime_config/testdefs/ExpectedTestFails.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index bfd87c2fd9..f5053a7843 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -58,6 +58,13 @@ + + + FAIL + ESMCI/ccs_config_cesm#131 + + + FAIL From ad5c0768be91778b8298f30fa602bd7eabcb8f45 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 3 Dec 2023 17:52:34 -0700 Subject: [PATCH 152/173] Remove references to Cheyenne and make it more explicit what to do on different systems, as well as adding Derecho --- tools/modify_input_files/README.fsurdat_modifier | 9 ++++++++- tools/modify_input_files/README.mesh_mask_modifier | 9 ++++++++- tools/site_and_regional/README | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/modify_input_files/README.fsurdat_modifier b/tools/modify_input_files/README.fsurdat_modifier index 58dbadb902..885227c928 100644 --- a/tools/modify_input_files/README.fsurdat_modifier +++ b/tools/modify_input_files/README.fsurdat_modifier @@ -27,10 +27,17 @@ tools/modify_input_files/modify_fsurdat_template.cfg Instructions ------------ -To run on Cheyenne/Casper/Izumi +To run on various machines: 1) (Un)load, execute, and activate the following: +1a) First step to activate conda on your system +1a) Casper: module unload python module load conda +1a) Izumi: +module load python +1a) Derecho (nothing needs to be done for this step) +1a) Elsewhere (do what's needed to activate conda) +1b) On all systems ./py_env_create conda activate ctsm_pylib (Use "deactivate" to reverse the latter.) diff --git a/tools/modify_input_files/README.mesh_mask_modifier b/tools/modify_input_files/README.mesh_mask_modifier index 13d98b9e7f..9484058ffd 100644 --- a/tools/modify_input_files/README.mesh_mask_modifier +++ b/tools/modify_input_files/README.mesh_mask_modifier @@ -10,10 +10,17 @@ tools/modify_input_files/modify_mesh_template.cfg Instructions ------------ -To run on Cheyenne/Casper/Izumi +To run on various machines: 1) (Un)load, execute, and activate the following: +1a) First step to activate conda on your system +1a) Casper: module unload python module load conda +1a) Izumi: +module load python +1a) Derecho (nothing needs to be done +1a) Elsewhere (do what's needed to activate conda) +1b) On all systems ./py_env_create conda activate ctsm_py (Use "deactivate" to reverse the latter.) diff --git a/tools/site_and_regional/README b/tools/site_and_regional/README index 723ddb4a97..7b36b7d2b9 100644 --- a/tools/site_and_regional/README +++ b/tools/site_and_regional/README @@ -3,9 +3,9 @@ $CTSMROOT/tools/site_and_regional/README The purpose of this directory is to contain all of the scripts that involve creating CTSM input data files for single site as well as regional cases. -The python scripts require the following settings before running on cheyenne: +The python scripts require the following settings before running: -module load conda +(Do what's needed to make conda available on your system) ../../py_env_create conda activate ctsm_pylib From 9c4d186b0fd666c79112b17ba352aee6a8563235 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 4 Dec 2023 01:47:47 -0700 Subject: [PATCH 153/173] Update /glade/p references --- bld/namelist_files/createMapEntry.pl | 2 +- bld/unit_testers/build-namelist_test.pl | 2 +- bld/unit_testers/xFail/expectedClmTestFails.xml | 13 ------------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/bld/namelist_files/createMapEntry.pl b/bld/namelist_files/createMapEntry.pl index 561683bb05..f9009ba86f 100755 --- a/bld/namelist_files/createMapEntry.pl +++ b/bld/namelist_files/createMapEntry.pl @@ -23,7 +23,7 @@ my $scriptName; ($scriptName = $0) =~ s!(.*)/!!; # get name of script my $cwd = getcwd(); - my $CSMDATA = "/glade/p/cesm/cseg/inputdata"; + my $CSMDATA = "/glade/campaign/cesm/cesmdata/cseg/inputdata"; if ($#ARGV != 0 ) { usage(); diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 4c717a58bf..4d03124116 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -139,7 +139,7 @@ sub cat_and_create_namelistinfile { $inputdata_rootdir = $ENV{'CSMDATA'}; } else { # use yellowstone location as default - $inputdata_rootdir="/glade/p/cesm/cseg/inputdata"; + $inputdata_rootdir="/glade/campaign/cesm/cesmdata/cseg/inputdata"; print("WARNING: -csmdata nor CSMDATA are set, using default yellowstone location: $inputdata_rootdir\n"); } diff --git a/bld/unit_testers/xFail/expectedClmTestFails.xml b/bld/unit_testers/xFail/expectedClmTestFails.xml index 12c954d38b..c7cbfee488 100644 --- a/bld/unit_testers/xFail/expectedClmTestFails.xml +++ b/bld/unit_testers/xFail/expectedClmTestFails.xml @@ -34,19 +34,6 @@ - - - - Doesn't check for valid values - - - - - - - - From fe97097f561b2dfb8cb9c47e5b1a5142ffe6f5d7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 4 Dec 2023 01:48:21 -0700 Subject: [PATCH 154/173] Update /glade/p path references --- tools/modify_input_files/README.mesh_mask_modifier | 2 +- tools/site_and_regional/default_data.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/modify_input_files/README.mesh_mask_modifier b/tools/modify_input_files/README.mesh_mask_modifier index 9484058ffd..4e25e73826 100644 --- a/tools/modify_input_files/README.mesh_mask_modifier +++ b/tools/modify_input_files/README.mesh_mask_modifier @@ -56,7 +56,7 @@ In your copy of the CTSM (say, ~user/ctsm), go to the appropriate tool: Enter the following (or similar) selections in modify_fill_indianocean.cfg: -mesh_mask_in = /glade/p/cesmdata/cseg/inputdata/share/meshes/fv0.9x1.25_141008_polemod_ESMFmesh.nc +mesh_mask_in = /glade/campaign/cesm/cesmdata/cseg/inputdata/share/meshes/fv0.9x1.25_141008_polemod_ESMFmesh.nc mesh_mask_out = fv0.9x1.25_141008_polemod_ESMFmesh_modified.nc landmask_file = .../path_to_your_copy_of/fill_indianocean.nc diff --git a/tools/site_and_regional/default_data.cfg b/tools/site_and_regional/default_data.cfg index 7e841dca54..0425aba133 100644 --- a/tools/site_and_regional/default_data.cfg +++ b/tools/site_and_regional/default_data.cfg @@ -1,5 +1,5 @@ [main] -clmforcingindir = /glade/p/cesmdata/inputdata +clmforcingindir = /glade/campaign/cesm/cesmdata/cseg/inputdata [datm_gswp3] dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 From 12057ea300fe7590ecd96ec8669df22377356dbf Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 4 Dec 2023 02:08:16 -0700 Subject: [PATCH 155/173] Update /glade/p paths --- python/ctsm/crop_calendars/generate_gdds.py | 2 +- python/ctsm/machine_defaults.py | 2 +- python/ctsm/test/test_unit_subset_data.py | 2 +- python/ctsm/test/testinputs/default_data.cfg | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index b54e7df40f..16e3e130da 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -1,4 +1,4 @@ -paramfile_dir = "/glade/p/cesmdata/cseg/inputdata/lnd/clm2/paramdata" +paramfile_dir = "/glade/campaign/cesm/cesmdata/cseg/inputdata/lnd/clm2/paramdata" # Import other shared functions import os diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index 18f4ff4752..e30881c474 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -47,7 +47,7 @@ "cheyenne": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), - baseline_dir=os.path.join(os.path.sep, "glade", "p", "cgd", "tss", "ctsm_baselines"), + baseline_dir=os.path.join(os.path.sep, "glade", "p", "cgd", "tss", "To_Be_Safely_Deleted", "ctsm_baselines"), account_required=True, create_test_retry=0, # NOTE(wjs, 2022-02-23) By default, use the regular queue, even for diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index fd9aef631d..b8ea5b06f2 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -56,7 +56,7 @@ def test_inputdata_setup_files_basic(self): ) self.assertEqual( files["main_dir"], - "/glade/p/cesmdata/inputdata", + "/glade/campaign/cesm/cesmdata/cseg/inputdata", "main_dir directory not whats expected", ) diff --git a/python/ctsm/test/testinputs/default_data.cfg b/python/ctsm/test/testinputs/default_data.cfg index 7e841dca54..0425aba133 100644 --- a/python/ctsm/test/testinputs/default_data.cfg +++ b/python/ctsm/test/testinputs/default_data.cfg @@ -1,5 +1,5 @@ [main] -clmforcingindir = /glade/p/cesmdata/inputdata +clmforcingindir = /glade/campaign/cesm/cesmdata/cseg/inputdata [datm_gswp3] dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 From 8b249d532f7dd39c56867ea93a92fbf6dad6fac1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 4 Dec 2023 02:47:08 -0700 Subject: [PATCH 156/173] First stab at getting the test/tools framework to work on Derecho, isn't working as conda and module is not available in my default shell --- test/tools/README | 4 +- test/tools/test_driver.sh | 71 ++++++++++++++++++++++++--- test/tools/tests_pretag_derecho_nompi | 9 ++++ 3 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 test/tools/tests_pretag_derecho_nompi diff --git a/test/tools/README b/test/tools/README index a2acbcae40..c545f625b8 100644 --- a/test/tools/README +++ b/test/tools/README @@ -11,7 +11,7 @@ To use... ./test_driver.sh -i -on cheyenne +on Derecho qcmd -l walltime=08:00:00 -- ./test_driver.sh -i >& run.out & @@ -33,7 +33,7 @@ To run neon-specific tests, please use login nodes: env CLM_INPUT_TESTS=`pwd`/tests_pretag_nompi_neon ./test_driver.sh -i > & run_neon.out & -Intended for use on NCAR machines cheyenne, geyser (DAV) and hobart. +Intended for use on NCAR machines Derecho, Casper (DAV) and izumi. II. RUNNING test_driver.sh TOOLS TESTING: diff --git a/test/tools/test_driver.sh b/test/tools/test_driver.sh index d4676cca69..6b1811c8ab 100755 --- a/test/tools/test_driver.sh +++ b/test/tools/test_driver.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh # # test_driver.sh: driver script for the offline testing of CLM of tools # @@ -26,15 +26,74 @@ hostname=`hostname` echo $hostname case $hostname in - ##cheyenne - cheyenne* | r*i*n*) - submit_script="test_driver_cheyenne${cur_time}.sh" + ##Derecho + derecho* | dec*) + submit_script="test_driver_derecho${cur_time}.sh" ##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv cat > ./${submit_script} << EOF #!/bin/sh # +interactive="YES" +input_file="tests_pretag_derecho_nompi" +c_threads=128 + +export INITMODULES="/glade/u/apps/derecho/23.06/spack/opt/spack/lmod/8.7.20/gcc/7.5.0/pdxb/lmod/lmod/init/sh" +. \$INITMODULES + +module --force purge +module load ncarenv +module load craype +module load intel +module load mkl +module load ncarcompilers +module load netcdf +module load nco +module load ncl + +#omp threads +if [ -z "\$CLM_THREADS" ]; then #threads NOT set on command line + export CLM_THREADS=\$c_threads +fi + +# Stop on first failed test +if [ -z "\$CLM_SOFF" ]; then #CLM_SOFF NOT set + export CLM_SOFF=FALSE +fi + +export CESM_MACH="derecho" +export CESM_COMP="intel" + +export NETCDF_DIR=\$NETCDF +export INC_NETCDF=\$NETCDF/include +export LIB_NETCDF=\$NETCDF/lib +export MAKE_CMD="gmake -j " +export CFG_STRING="" +export TOOLS_MAKE_STRING="USER_FC=ifort USER_LINKER=ifort USER_CPPDEFS=-DLINUX" +export MACH_WORKSPACE=\$SCRATCH +export CPRNC_EXE="$CESMDATAROOT/cprnc/cprnc" +dataroot="$CESMDATAROOT/inputdata" +export TOOLSLIBS="" +export REGRID_PROC=1 +export TOOLS_CONF_STRING="--mpilib mpi-serial" + + +echo_arg="" + +EOF +#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ + ;; + + ##cheyenne + cheyenne* | r*i*n*) + submit_script="test_driver_cheyenne${cur_time}.sh" + +#vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv +at > ./${submit_script} << EOF +!/bin/sh + + interactive="YES" input_file="tests_pretag_cheyenne_nompi" c_threads=36 @@ -314,7 +373,7 @@ EOF ;; * ) - echo "Only setup to work on: cheyenne, hobart and izumi" + echo "Only setup to work on: derecho, cheyenne, hobart and izumi" exit @@ -633,7 +692,7 @@ case $arg1 in * ) echo "" echo "**********************" - echo "usage on cheyenne, hobart, and izumi: " + echo "usage on derecho, cheyenne, hobart, and izumi: " echo "./test_driver.sh -i" echo "" echo "valid arguments: " diff --git a/test/tools/tests_pretag_derecho_nompi b/test/tools/tests_pretag_derecho_nompi new file mode 100644 index 0000000000..5fdaf335ae --- /dev/null +++ b/test/tools/tests_pretag_derecho_nompi @@ -0,0 +1,9 @@ +smba1 blba1 +smbd1 blbd1 +sm0a1 bl0a1 +sm0c1 bl0c1 +smaa2 blaa2 +smba1 blba1 +smb81 blb81 +smbc1 blbc1 +smbd1 blbd1 From 8085fa344c0119e2eeb299b521082a0e74636158 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 4 Dec 2023 03:10:02 -0700 Subject: [PATCH 157/173] Update paths to use /glade/campaign/cesm/cesmdata/cseg/inputdata resolving most of #2196 --- .../testmods_dirs/clm/DA_multidrv/user_nl_clm_0001 | 1 + .../testmods_dirs/clm/DA_multidrv/user_nl_clm_0002 | 1 + .../testdefs/testmods_dirs/clm/USUMB_mct/shell_commands | 7 +++---- .../testdefs/testmods_dirs/clm/USUMB_mct/user_nl_clm | 5 ++--- .../testdefs/testmods_dirs/clm/USUMB_nuopc/shell_commands | 3 +-- .../testdefs/testmods_dirs/clm/USUMB_nuopc/user_nl_clm | 5 ++--- .../testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm | 2 +- .../clm/collapse_pfts_78_to_16_decStart_f10/README | 4 ++-- .../clm/collapse_pfts_78_to_16_decStart_f10/user_nl_clm | 4 ++-- .../testdefs/testmods_dirs/clm/cplhist/shell_commands | 2 +- .../modify_smallville_with_dynurban.ncl | 1 + .../testdefs/testmods_dirs/clm/sowingWindows/user_nl_clm | 6 +++--- 12 files changed, 20 insertions(+), 21 deletions(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0001 b/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0001 index 719785e25d..96b8d387db 100644 --- a/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0001 +++ b/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0001 @@ -10,4 +10,5 @@ hist_type1d_pertape = ' ',' ',' ' use_init_interp = .true. + ! IMPORTANT: The path for this file NEEDS to be updated for the removal of Cheyenne and /glade/p finidat = '/glade/p/cisl/dares/RDA_strawman/CESM_ensembles/CLM/CLM5BGC-Crop/ctsm_2001-01-01-00000/clm5_f09_spinup80.clm2_0001.r.2001-01-01-00000.nc' diff --git a/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0002 b/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0002 index 37d5b2b24e..6a606720d5 100644 --- a/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0002 +++ b/cime_config/testdefs/testmods_dirs/clm/DA_multidrv/user_nl_clm_0002 @@ -10,4 +10,5 @@ hist_type1d_pertape = ' ',' ',' ' use_init_interp = .true. + ! IMPORTANT: The path for this file NEEDS to be updated for the removal of Cheyenne and /glade/p finidat = '/glade/p/cisl/dares/RDA_strawman/CESM_ensembles/CLM/CLM5BGC-Crop/ctsm_2001-01-01-00000/clm5_f09_spinup80.clm2_0002.r.2001-01-01-00000.nc' diff --git a/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/shell_commands b/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/shell_commands index 446125abf9..4fb282d511 100755 --- a/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/shell_commands @@ -5,8 +5,8 @@ ./xmlchange DATM_CLMNCEP_YR_END=2006 # Comment this out if NINST_LND is greater than 1 (see: http://bugs.cgd.ucar.edu/show_bug.cgi?id=2521) ./xmlchange MPILIB=mpi-serial -./xmlchange ATM_DOMAIN_PATH=/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB -./xmlchange LND_DOMAIN_PATH=/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB +./xmlchange ATM_DOMAIN_PATH='$DIN_LOC_ROOT/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB' +./xmlchange LND_DOMAIN_PATH='$DIN_LOC_ROOT/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB' ./xmlchange ATM_DOMAIN_FILE=domain.lnd.1x1pt_US-UMB_navy.171024.nc ./xmlchange LND_DOMAIN_FILE=domain.lnd.1x1pt_US-UMB_navy.171024.nc ./xmlchange --append CLM_BLDNML_OPTS='-mask navy -no-crop' @@ -15,5 +15,4 @@ ./xmlchange ATM_NCPL=24 ./xmlchange RUN_STARTDATE=1999-01-01 ./xmlchange DATM_CLMNCEP_YR_ALIGN=1999 -./xmlchange DIN_LOC_ROOT=/glade/p/cesm/cseg/inputdata -./xmlchange DIN_LOC_ROOT_CLMFORC=/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024 +./xmlchange DIN_LOC_ROOT_CLMFORC='$DIN_LOC_ROOT/lnd/clm2/PTCLMmydatafiles.c171024' diff --git a/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/user_nl_clm index 38ce400297..8bb7848d49 100644 --- a/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/USUMB_mct/user_nl_clm @@ -1,5 +1,4 @@ -! user_nl_clm namelist options written by PTCLMmkdata: -! ./PTCLMmkdata --cesm_root ../../../.. -s US-UMB -d /glade/p/cesm/cseg/inputdata --mydatadir=/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024 - fsurdat = '/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB/surfdata_1x1pt_US-UMB_16pfts_Irrig_CMIP6_simyr2000_c171024.nc' +! user_nl_clm namelist options written by PTCLMmkdata, which is no longer available + fsurdat = '$DIN_LOC_ROOT/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB/surfdata_1x1pt_US-UMB_16pfts_Irrig_CMIP6_simyr2000_c171024.nc' hist_nhtfrq = 0 hist_mfilt = 1200 diff --git a/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/shell_commands b/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/shell_commands index 08a9014abe..ec6eb898d7 100755 --- a/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/shell_commands @@ -11,7 +11,6 @@ ./xmlchange ATM_NCPL=24 ./xmlchange RUN_STARTDATE=1999-01-01 ./xmlchange DATM_YR_ALIGN=1999 -./xmlchange DIN_LOC_ROOT=/glade/p/cesm/cseg/inputdata -./xmlchange DIN_LOC_ROOT_CLMFORC=/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024 +./xmlchange DIN_LOC_ROOT_CLMFORC='$DIN_LOC_ROOT/lnd/clm2/PTCLMmydatafiles.c171024' ./xmlchange PTS_LON=275.2862 ./xmlchange PTS_LAT=45.5598 diff --git a/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/user_nl_clm index 38ce400297..8bb7848d49 100644 --- a/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/USUMB_nuopc/user_nl_clm @@ -1,5 +1,4 @@ -! user_nl_clm namelist options written by PTCLMmkdata: -! ./PTCLMmkdata --cesm_root ../../../.. -s US-UMB -d /glade/p/cesm/cseg/inputdata --mydatadir=/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024 - fsurdat = '/glade/p/cesm/cseg/inputdata/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB/surfdata_1x1pt_US-UMB_16pfts_Irrig_CMIP6_simyr2000_c171024.nc' +! user_nl_clm namelist options written by PTCLMmkdata, which is no longer available + fsurdat = '$DIN_LOC_ROOT/lnd/clm2/PTCLMmydatafiles.c171024/1x1pt_US-UMB/surfdata_1x1pt_US-UMB_16pfts_Irrig_CMIP6_simyr2000_c171024.nc' hist_nhtfrq = 0 hist_mfilt = 1200 diff --git a/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm index 57ac8a6461..534cbb84b8 100644 --- a/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/ciso_cwd_hr/user_nl_clm @@ -1,2 +1,2 @@ -paramfile = '/glade/p/cesm/cseg/inputdata/lnd/clm2/paramdata/ctsm51_ciso_cwd_hr_params.c231117.nc' +paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/ctsm51_ciso_cwd_hr_params.c231117.nc' hist_fincl1 = 'CWDC_HR','C13_CWDC_HR','C14_CWDC_HR','CWD_HR_L2','CWD_HR_L2_vr','CWD_HR_L3','CWD_HR_L3_vr' diff --git a/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/README b/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/README index 81fb991ed0..af5d819ffc 100644 --- a/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/README +++ b/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/README @@ -9,10 +9,10 @@ According to the file the following two files used in this test are default files for the following options: -fsurdat = '/glade/p/cesmdata/cseg/inputdata/lnd/clm2/surfdata_map/surfdata_10x15_78pfts_CMIP6_simyr1850_c170824.nc' +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_10x15_78pfts_CMIP6_simyr1850_c170824.nc' hgrid="10x15" sim_year="1850" use_crop=".true." -flanduse_timeseries = '/glade/p/cesmdata/cseg/inputdata/lnd/clm2/surfdata_map/landuse.timeseries_10x15_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc' +flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_10x15_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc' hgrid="10x15" sim_year_range="1850-2000" use_crop=".true." hgrid="10x15" rcp="8.5" sim_year_range="1850-2100" use_crop=".true." hgrid="10x15" rcp="6" sim_year_range="1850-2100" use_crop=".true." diff --git a/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/user_nl_clm index ff78e0122c..8c4fed6873 100644 --- a/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/collapse_pfts_78_to_16_decStart_f10/user_nl_clm @@ -1,2 +1,2 @@ -fsurdat = '/glade/p/cesmdata/cseg/inputdata/lnd/clm2/surfdata_map/surfdata_10x15_78pfts_CMIP6_simyr1850_c170824.nc' -flanduse_timeseries = '/glade/p/cesmdata/cseg/inputdata/lnd/clm2/surfdata_map/landuse.timeseries_10x15_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc' +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_10x15_78pfts_CMIP6_simyr1850_c170824.nc' +flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_10x15_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc' diff --git a/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands b/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands index a284aefcb9..f0eb85010b 100755 --- a/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands @@ -1,5 +1,5 @@ ./xmlchange DATM_CPLHIST_CASE=cam6ctsm51_cesm23a12c_ne30pg3g17_CPLHIST_1850 -./xmlchange DATM_CPLHIST_DIR=/glade/p/cgd/tss/CTSM_datm_forcing_data/atm_forcing.cdeps_datm.CPLHIST_cam6ctsm51_cesm23a12c_ne30pg3g17_1850 +./xmlchange DATM_CPLHIST_DIR='$DIN_LOC_ROOT/atm/datm7/atm_forcing.cdeps_datm.CPLHIST_cam6ctsm51_cesm23a12c_ne30pg3g17_1850' ./xmlchange DATM_YR_START=1 ./xmlchange DATM_YR_END=1 ./xmlchange DATM_YR_ALIGN=1 diff --git a/cime_config/testdefs/testmods_dirs/clm/smallville_dynurban_monthly/modify_smallville_with_dynurban.ncl b/cime_config/testdefs/testmods_dirs/clm/smallville_dynurban_monthly/modify_smallville_with_dynurban.ncl index 15ec0469be..30e73535a4 100644 --- a/cime_config/testdefs/testmods_dirs/clm/smallville_dynurban_monthly/modify_smallville_with_dynurban.ncl +++ b/cime_config/testdefs/testmods_dirs/clm/smallville_dynurban_monthly/modify_smallville_with_dynurban.ncl @@ -18,6 +18,7 @@ begin print ("Start Time: "+systemfunc("date") ) print ("=========================================") + ; IMPORTANT NOTE: The path for these files needs to be updated with the removal of Cheyenne and /glade/p (See history note below as well) infile = "/glade/p/cgd/tss/people/oleson/modify_surfdata/landuse.timeseries_1x1_smallvilleIA_hist_78pfts_simyr1850-1855_c160127.nc" outfile = "/glade/p/cgd/tss/people/oleson/modify_surfdata/landuse.timeseries_1x1_smallvilleIA_hist_78pfts_simyr1850-1855_dynUrban_c220223.nc" diff --git a/cime_config/testdefs/testmods_dirs/clm/sowingWindows/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/sowingWindows/user_nl_clm index d3d922f721..03165bb306 100644 --- a/cime_config/testdefs/testmods_dirs/clm/sowingWindows/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/sowingWindows/user_nl_clm @@ -1,5 +1,5 @@ -stream_fldFileName_swindow_start = '/glade/p/cesmdata/cseg/inputdata/lnd/clm2/cropdata/calendars/processed/swindow_starts_ggcmi_crop_calendar_phase3_v1.01.2000-2000.20231005_145103.nc' -stream_fldFileName_swindow_end = '/glade/p/cesmdata/cseg/inputdata/lnd/clm2/cropdata/calendars/processed/swindow_ends_ggcmi_crop_calendar_phase3_v1.01.2000-2000.20231005_145103.nc' -stream_meshfile_cropcal = '/glade/p/cesmdata/cseg/inputdata/share/meshes/360x720_120830_ESMFmesh_c20210507_cdf5.nc' +stream_fldFileName_swindow_start = '$DIN_LOC_ROOT/lnd/clm2/cropdata/calendars/processed/swindow_starts_ggcmi_crop_calendar_phase3_v1.01.2000-2000.20231005_145103.nc' +stream_fldFileName_swindow_end = '$DIN_LOC_ROOT/lnd/clm2/cropdata/calendars/processed/swindow_ends_ggcmi_crop_calendar_phase3_v1.01.2000-2000.20231005_145103.nc' +stream_meshfile_cropcal = '$DIN_LOC_ROOT/share/meshes/360x720_120830_ESMFmesh_c20210507_cdf5.nc' stream_year_first_cropcal = 2000 stream_year_last_cropcal = 2000 From 662ab292788661335ce68354d1ee2ef3fb1910a8 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 4 Dec 2023 11:19:05 -0700 Subject: [PATCH 158/173] Update ccs_config to version from ekluzek:derecho1#8085fa3. --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index f5243f9166..cf2ff73498 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.82 +tag = ccs_config_cesm0.0.84 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -98,4 +98,4 @@ tag = v1.0.8 required = False [externals_description] -schema_version = 1.0.0 \ No newline at end of file +schema_version = 1.0.0 From c5b409d0f4bdd93d9315ccb3bd31725de4bb0f78 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 5 Dec 2023 09:48:43 -0700 Subject: [PATCH 159/173] Update ChangeLog and ChangeSum. --- doc/ChangeLog | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 73 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index aabefea0fe..ca67adff02 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,76 @@ =============================================================== +Tag name: ctsm5.1.dev157 +Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) +Date: Tue Dec 5 09:48:26 MST 2023 +One-line Summary: Update Externals to work on Derecho + +Purpose and description of changes +---------------------------------- + +Updates Externals.cfg to work on Derecho. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): +* Resolves #2217 ("Tags for building CTSM library on Derecho [WRF-CTSM]", https://github.com/ESCOMP/CTSM/issues/2217) +* Resolves #2090 ("Update to cesm2_3_beta16 externals.", https://github.com/ESCOMP/CTSM/issues/2090) + +Known bugs introduced in this tag (include issue #): +* #2280: Updating Externals for Derecho causes Izumi nag tests to fail (https://github.com/ESCOMP/CTSM/issues/2280) + + +Notes of particular relevance for developers: +--------------------------------------------- + +Changes to tests or testing: +* All Izumi nag tests fail early in the run phase. This should be fixed in the next tag, which will be a more comprehensive Derecho-focused update. + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- PASS (except nag) + + +Other details +------------- + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): +* cime: cime6.0.125 -> cime6.0.175 +* cmeps: cmeps0.14.21 -> cmeps0.14.43 +* cdeps: cdeps1.0.13 -> cdeps1.0.23 +* cpl7: cpl77.0.5 -> cpl77.0.7 +* parallelio: pio2_5_10 -> pio2_6_2 + +Pull Requests that document the changes (include PR ids): +* #2270: Update Externals.cfg to work on Derecho (https://github.com/ESCOMP/CTSM/pull/2270) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev156 Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) Date: Thu Nov 30 15:27:18 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index d891a68d87..4c8c356a41 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev157 samrabin 12/05/2023 Update Externals to work on Derecho ctsm5.1.dev156 samrabin 11/30/2023 Do not use Meier roughness by default. ctsm5.1.dev155 samrabin 11/27/2023 Use baset_latvary parameters ctsm5.1.dev154 slevis 11/22/2023 New params files with changes for Meier roughness, MIMICS, and SNICAR, and changes to leafcn and k*_nonmyc From 7efee20795948f3474ed3e8b813f1859bb099284 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 6 Dec 2023 00:07:41 -0700 Subject: [PATCH 160/173] Update cdeps version to fix #2280 so NAG compiler can run --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index cf2ff73498..a17f8e2ec6 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,7 +55,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.23 +tag = cdeps1.0.24 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps From 6fccf682eaf718615407d9bacdd3903b8786a03d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 6 Dec 2023 09:42:56 -0700 Subject: [PATCH 161/173] Run through black --- python/ctsm/machine_defaults.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index e30881c474..0f3900c152 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -47,7 +47,9 @@ "cheyenne": MachineDefaults( job_launcher_type=JOB_LAUNCHER_QSUB, scratch_dir=os.path.join(os.path.sep, "glade", "scratch", get_user()), - baseline_dir=os.path.join(os.path.sep, "glade", "p", "cgd", "tss", "To_Be_Safely_Deleted", "ctsm_baselines"), + baseline_dir=os.path.join( + os.path.sep, "glade", "p", "cgd", "tss", "To_Be_Safely_Deleted", "ctsm_baselines" + ), account_required=True, create_test_retry=0, # NOTE(wjs, 2022-02-23) By default, use the regular queue, even for From 90541f66ec23ec7e8de9db8e7bbd167fa22b1afb Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 6 Dec 2023 09:44:06 -0700 Subject: [PATCH 162/173] ignore black format --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index b093c298e9..f2a9d2da8d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -23,3 +23,4 @@ e4d38681df23ccca0ae29581a45f8362574e0630 0bc3f00115d86d026a977918661c93779b3b19f9 540b256d1f3382f4619d7b0877c32d54ce5c40b6 8a168bb0895f4f2421608dd2589398e13a6663e6 +6fccf682eaf718615407d9bacdd3903b8786a03d From d7cd648a620d3d874dfc757365bf6747d818a4ed Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 6 Dec 2023 09:52:02 -0700 Subject: [PATCH 163/173] Remove new ne30np4.pg3 finidat file from CTSM5.2 as incompatible with CTSM5.1 --- bld/namelist_files/namelist_defaults_ctsm.xml | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index e9b9b7c54d..e03ad755ae 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -722,11 +722,6 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. - -hgrid=ne30np4.pg3 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. - - hgrid=1.9x2.5 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. @@ -825,12 +820,6 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. - -hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. - - hgrid=ne0np4CONUS.ne30x8 maxpft=17 mask=tx0.1v2 use_cn=.false. use_crop=.false. irrigate=.true. glc_nec=10 do_transient_pfts=.false. - -.true. - -lnd/clm2/initdata_map/clmi.I1850Clm51CplhistBgcCrop-ciso.0561-01-01.ne30np4.pg3_gx1v7_simyr1850_c231109.nc - - - + From 7be29765f622ddfbe20aff6f8b5a655cd3155177 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Dec 2023 12:24:41 -0700 Subject: [PATCH 170/173] Correct the finidat file settings for clm5_1_cam6 to point to the clm5_0_cam6 f09 files --- bld/namelist_files/namelist_defaults_ctsm.xml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index e03ad755ae..b5ae57e5ae 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -722,6 +722,11 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.false. glc_nec=10 do_transient_pfts=.false. + + hgrid=1.9x2.5 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. @@ -820,6 +825,12 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + + lnd/clm2/initdata_map/clmi.I1850Clm50BgcCrop-ciso.1366-01-01.0.9x1.25_gx1v7_simyr1850_c200428.nc +>lnd/clm2/initdata_map/clmi.B1850Clm50BgcCrop.0161-01-01.0.9x1.25_gx1v7_simyr1850_c200729.nc + + + +lnd/clm2/initdata_map/clmi.B1850Clm50BgcCrop.0161-01-01.0.9x1.25_gx1v7_simyr1850_c200729.nc From aaf740cdfcba1fe884107794d8244fc74d0f9384 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Dec 2023 22:16:13 -0700 Subject: [PATCH 172/173] Update Change files --- doc/.ChangeLog_template | 10 ++-- doc/ChangeLog | 107 ++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 3 files changed, 113 insertions(+), 5 deletions(-) diff --git a/doc/.ChangeLog_template b/doc/.ChangeLog_template index 9f4f64c587..bbd65c8c83 100644 --- a/doc/.ChangeLog_template +++ b/doc/.ChangeLog_template @@ -96,11 +96,11 @@ infrastructure should be run when appropriate, as described below. build-namelist tests (if CLMBuildNamelist.pm has changed): - cheyenne - + derecho - tools-tests (test/tools) (if tools have been changed): - cheyenne - + derecho - python testing (if python code has changed; see instructions in python/README.md; document testing done): @@ -114,15 +114,15 @@ infrastructure should be run when appropriate, as described below. doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to separately run the clm_pymods test suite, and you can remove the following line.] - clm_pymods test suite on cheyenne - + clm_pymods test suite on derecho - regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - cheyenne ---- + derecho ----- izumi ------- fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) - cheyenne ---- + derecho ----- izumi ------- any other testing (give details below): diff --git a/doc/ChangeLog b/doc/ChangeLog index 168183cee7..961ccab675 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,111 @@ =============================================================== +Tag name: ctsm5.1.dev158 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Thu 07 Dec 2023 10:15:56 PM MST +One-line Summary: First tag with testing moved to Derecho and working PE-layouts for Derecho + +Purpose and description of changes +---------------------------------- + +First tag for CTSM working and tested on Derecho. Update CDEPS so that we can run with the NAG compiler. +Working PE layouts. Changes from CESM3_dev over to main-dev. Testing added for Derecho. +Do some work to get tools testing working on Derecho, not completed. +Add Derecho to the README files under tools/modify_input_files and tools/site_and_regional +Make sure there are no /glade/p references in the code (other than in the doc directory). + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): + Resolves Derecho transition: Tests and test infrastructure #1995 + Resolves Updating Externals for Derecho causes Izumi nag tests to fail #2280 + Resolves Transient simulation with ne30np4.pg3 fails due to floating point error #2268 + Resolves Need to move location of DA_multidrv finidat files from /glade/p to /glade/campaign #2282 + Works on Add support to test/tools/test_driver.sh for Derecho for NEON tools #2276 + +Notes of particular relevance for users +--------------------------------------- + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): + Added clm5_1_cam6 option to LND_TUNING_MODE + This is important in order to enable using latest clm5_1 physics with fully coupled cases + +Changes made to namelist defaults (e.g., changed parameter values): + Make sure there are finidat files for clm5_1 with CAM6 for 1850 and 2000 (from clm5_0 version at f09 + Make sure ne30np4.pg3 is setup + Some adjustments for ne30np4 and ne30np4.pg3 to make sure landuse.timeseries files are correct + +Notes of particular relevance for developers: +--------------------------------------------- + +Changes to tests or testing: + Add cesm3_dev test list from the CESM3_dev branch + Make sure ne30np4.pg3 + + Unit tests fail on Derecho because of ESMCI/ccs_config_cesm#131 + Derecho tests with DEBUG=T, intel compiler, and mpi-serial fail because of ESMCI/ccs_config_cesm#130 + + +Testing summary: regular + fates + ctsm_sci + cesm3_dev +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + derecho - PASS + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + derecho - PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + derecho ----- OK + izumi ------- OK + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + derecho ----- OK + izumi ------- OK + + + +Answer changes +-------------- + +Changes answers relative to baseline: No, bit-for-bit + +Other details +------------- +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): cdeps + cdeps -> cdeps1.0.24 (allows to run with NAG compiler) + +Pull Requests that document the changes (include PR ids): + #2269 -- First tag with testing moved to Derecho and working PE layouts +(https://github.com/ESCOMP/ctsm/pull) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev157 Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) Date: Tue Dec 5 09:48:26 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 4c8c356a41..abf4401013 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev158 erik 12/07/2023 First tag with testing moved to Derecho and working PE-layouts for Derecho ctsm5.1.dev157 samrabin 12/05/2023 Update Externals to work on Derecho ctsm5.1.dev156 samrabin 11/30/2023 Do not use Meier roughness by default. ctsm5.1.dev155 samrabin 11/27/2023 Use baset_latvary parameters From 10d05382fb8e43fc1527d692e26a596f418cbfb3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Dec 2023 22:22:58 -0700 Subject: [PATCH 173/173] Update changelog --- doc/ChangeLog | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 961ccab675..04694b5c07 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev158 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) -Date: Thu 07 Dec 2023 10:15:56 PM MST +Date: Thu 07 Dec 2023 10:22:46 PM MST One-line Summary: First tag with testing moved to Derecho and working PE-layouts for Derecho Purpose and description of changes @@ -11,7 +11,8 @@ First tag for CTSM working and tested on Derecho. Update CDEPS so that we can r Working PE layouts. Changes from CESM3_dev over to main-dev. Testing added for Derecho. Do some work to get tools testing working on Derecho, not completed. Add Derecho to the README files under tools/modify_input_files and tools/site_and_regional -Make sure there are no /glade/p references in the code (other than in the doc directory). +Remove some /glade/p references in the code. This is still an issue in the: doc, lilac, tools/mksurfdata_map, +tools/contrib, tools/mkmapdata directories, and the namelist_defaults_ctsm_tools file. Significant changes to scientifically-supported configurations