diff --git a/.gitignore b/.gitignore index de7655b8d1..5688cc2f16 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ src/physics/carma/base src/physics/clubb src/physics/cosp2/src src/physics/silhs +src/chemistry/geoschem/geoschem_src src/physics/pumas src/physics/pumas-frozen src/dynamics/fv3/atmos_cubed_sphere diff --git a/Externals.cfg b/Externals.cfg index 74d4338fe7..116cc05bbe 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -4,7 +4,7 @@ [parallelio] protocol = git -tag = pio2_5_10 +tag = pio2_6_2 repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio required = True diff --git a/Externals_CAM.cfg b/Externals_CAM.cfg index f3ff155e36..2fd38dfda2 100644 --- a/Externals_CAM.cfg +++ b/Externals_CAM.cfg @@ -7,7 +7,7 @@ required = True [carma] protocol = git -tag = carma4_00 +tag = carma4_01 repo_url = https://github.com/ESCOMP/CARMA_base.git local_path = src/physics/carma/base required = True @@ -22,7 +22,7 @@ required = True [clubb] protocol = git -tag = clubb_4ncar_20221129_59cb19f_20230330_branchtag +tag = clubb_4ncar_20231115_5406350 repo_url = https://github.com/larson-group/clubb_release local_path = src/physics/clubb sparse = ../.clubb_sparse_checkout @@ -30,7 +30,7 @@ required = True [pumas] protocol = git -tag = pumas_cam-release_v1.29 +tag = pumas_cam-release_v1.35 repo_url = https://github.com/ESCOMP/PUMAS local_path = src/physics/pumas required = True @@ -51,7 +51,7 @@ required = True [atmos_phys] protocol = git -tag = atmos_phys0_00_011 +tag = atmos_phys0_02_006 repo_url = https://github.com/NCAR/atmospheric_physics local_path = src/atmos_phys required = True @@ -72,11 +72,25 @@ required = True [hemco] protocol = git -tag = hemco-cesm1_2_0_hemco3_6_2_cesm +tag = hemco-cesm1_2_1_hemco3_6_3_cesm repo_url = https://github.com/ESCOMP/HEMCO_CESM.git local_path = src/hemco required = True externals = Externals_HCO.cfg +[rte-rrtmgp] +local_path = src/physics/rrtmgp/ext +protocol = git +repo_url = https://github.com/earth-system-radiation/rte-rrtmgp.git +tag = v1.7 +required = True + +[rrtmgp-data] +local_path = src/physics/rrtmgp/data +protocol = git +repo_url = https://github.com/earth-system-radiation/rrtmgp-data.git +tag = v1.8 +required = True + [externals_description] schema_version = 1.0.01 diff --git a/bld/config_files/definition.xml b/bld/config_files/definition.xml index e834645434..164ac87f92 100644 --- a/bld/config_files/definition.xml +++ b/bld/config_files/definition.xml @@ -66,10 +66,10 @@ Switch to turn on Harmonized Emissions Component (HEMCO) for chemistry: 0 => no, Option to turn on NorESM modifications to baseline CESM/CAM physics and dynamics - + Microphysics package: rk (Rasch and Kristjansson), mg1 (Morrison and Gettelman two moment scheme CAM5.1), mg2 (Morrison and Gettelman second -version CAM6), mg3 (MG scheme 3rd version, graupel), SPCAM_m2005, SPCAM_sam1mom. +version CAM6), mg3 (MG scheme 3rd version, graupel), PUMAS, SPCAM_m2005, SPCAM_sam1mom. Macrophysics package: RK, Park, CLUBB_SGS, SPCAM_sam1mom, SPCAM_m2005. @@ -94,9 +94,9 @@ on Mapes and Neale (2011): 0 => no, 1 => yes PBL package: uw (University of Washington), hb (Holtslag and Boville), hbr (Holtslag, Boville, and Rasch), clubb_sgs, spcam_sam1om, spcam_m2005, none. - + Radiative transfer calculation: -camrt (CAM3 and CAM4 RT package), rrtmg (RRTMG package from AER). +camrt (CAM3 and CAM4 RT package), rrtmg (RRTMG package from AER), rrtmgp (updated version). CARMA sectional microphysics: @@ -106,8 +106,8 @@ meteor_smoke (Meteor Smoke), mixed_sulfate (Meteor Smoke and Sulfate), pmc (Pola sulfate (Sulfate Aerosols), tholin (early earth haze), test_detrain (Detrainment), test_growth (Particle Growth), test_passive (Passive Dust), test_radiative (Radiatively Active Dust), test_swelling (Sea Salt), test_tracers (Asian Monsoon), test_tracers2 (Guam). - - Chemistry package: none,ghg_mam4,terminator,trop_mam3,trop_mam4,trop_mam7,trop_mam_oslo,trop_mozart,trop_strat_mam4_ts2,trop_strat_mam4_vbs,trop_strat_mam4_vbsext,trop_strat_mam5_ts2,trop_strat_mam5_vbs,trop_strat_mam5_vbsext,waccm_ma,waccm_mad,waccm_ma_sulfur,waccm_sc,waccm_sc_mam4,waccm_mad_mam4,waccm_ma_mam4,waccm_tsmlt_mam4,waccm_tsmlt_mam4_vbsext,waccm_mad_mam5,waccm_ma_mam5,waccm_tsmlt_mam5,waccm_tsmlt_mam5_vbsext + + Chemistry package: none,ghg_mam4,terminator,trop_mam3,trop_mam4,trop_mam7,trop_mam_oslo,trop_mozart,trop_strat_mam4_ts2,trop_strat_mam4_vbs,trop_strat_mam4_vbsext,trop_strat_mam5_ts2,trop_strat_mam5_vbs,trop_strat_mam5_vbsext,waccm_ma,waccm_mad,waccm_ma_sulfur,waccm_sc,waccm_sc_mam4,waccm_mad_mam4,waccm_ma_mam4,waccm_tsmlt_mam4,waccm_tsmlt_mam4_vbsext,waccm_mad_mam5,waccm_ma_mam5,waccm_tsmlt_mam5,waccm_tsmlt_mam5_vbsext,geoschem_mam4 Prognostic mozart species packages: list of any subset of the following: DST,SSLT,SO4,GHG,OC,BC,CARBON16 diff --git a/bld/configure b/bld/configure index 4cd9c05127..585d9a9c65 100755 --- a/bld/configure +++ b/bld/configure @@ -69,7 +69,7 @@ OPTIONS trop_strat_mam4_vbs | trop_strat_mam4_vbsext | trop_strat_mam5_ts2 | trop_strat_mam5_vbs | trop_strat_mam5_vbsext | waccm_ma | waccm_mad | waccm_ma_sulfur | waccm_sc | waccm_sc_mam4 | waccm_mad_mam4 | waccm_ma_mam4 | waccm_tsmlt_mam4 | waccm_tsmlt_mam4_vbsext | waccm_mad_mam5 | - waccm_ma_mam5 | waccm_tsmlt_mam5 | waccm_tsmlt_mam5_vbsext ]. + waccm_ma_mam5 | waccm_tsmlt_mam5 | waccm_tsmlt_mam5_vbsext | geoschem_mam4 ]. Default: trop_mam_oslo when -camnor is specified Default: trop_mam5 for cam7 Default: trop_mam4 for cam6 @@ -93,7 +93,7 @@ OPTIONS -macrophys Specify the macrophysics option [rk | park | clubb_sgs]. -max_n_rad_cnst Maximum number of constituents that are either radiatively active, or in any single diagnostic list for the radiation. - -microphys Specify the microphysics option [mg1 | mg2 | mg3| rk]. + -microphys Specify the microphysics option [mg1 | mg2 | mg3| rk | pumas]. -model_top Specify the model_top option [ lt | mt ]. -nadv Set total number of advected species to . -nadv_tt Set number of advected test tracers . @@ -108,7 +108,7 @@ OPTIONS -prog_species Comma-separate list of prognostic mozart species packages. Currently available: DST,SSLT,SO4,GHG,OC,BC,CARBON16 -psubcols Maximum number of sub-columns in a run - set to 1 if not using sub-columns (default) - -rad Specify the radiation package [rrtmg | camrt] + -rad Specify the radiation package [rrtmg | rrtmgp | rrtmgp_gpu | camrt] -silhs Switch on SILHS. -spcam_clubb_sgs Turn on the SPCAM version of CLUBB -spcam_nx SPCAM x-grid. - defaults to 4 (note the CRM requires spcam_nx to be greater than or equal to 4) @@ -861,6 +861,10 @@ if (defined $opts{'microphys'}) { $microphys_pkg = lc($opts{'microphys'}); } +if($microphys_pkg eq 'pumas') { + $microphys_pkg = 'mg3'; +} + $cfg_ref->set('microphys', $microphys_pkg); if ($print>=2) { print "Microphysics package: $microphys_pkg$eol"; } @@ -879,7 +883,7 @@ if ($carma_pkg =~ m/cirrus/i) { unless ($microphys_pkg =~ /^mg/) { die <<"EOF"; ** ERROR: microphysics package set to: $microphys_pkg -** The CARMA cirrus model only works with MG microphysics. +** The CARMA cirrus model only works with MG or PUMAS microphysics. EOF } } @@ -902,11 +906,11 @@ if (defined $opts{'clubb_sgs'}) { # consistency checks... -# CLUBB_SGS only works with mg microphysics +# CLUBB_SGS only works with mg or PUMAS microphysics if ($clubb_sgs and not ($microphys_pkg =~ m/^mg/ )) { die <<"EOF"; ** ERROR: microphysics package set to: $microphys_pkg -** CLUBB_SGS only works with MG microphysics. +** CLUBB_SGS only works with MG or PUMAS microphysics. EOF } @@ -1041,12 +1045,12 @@ if (defined $opts{'pbl'}) { # consistency checks... -# UW PBL only works with mg microphysics +# UW PBL only works with mg or PUMAS microphysics if ($pbl_pkg =~ m/uw/i) { unless ($microphys_pkg =~ /^mg/) { die <<"EOF"; ** ERROR: microphysics package set to: $microphys_pkg -** The UW PBL scheme only works with MG microphysics. +** The UW PBL scheme only works with MG or PUMAS microphysics. EOF } } @@ -1088,8 +1092,16 @@ elsif ($phys_pkg =~ m/^cam[56]$|^cam_dev$|^spcam_m2005$/) { $rad_pkg = 'rrtmg'; } # Allow the user to override the default via the commandline. +my $use_rrtmgp_gpu = 0; if (defined $opts{'rad'}) { $rad_pkg = lc($opts{'rad'}); + # If the radiation package is set to rrtmgp_gpu then will add the gpu code version + # (openmp and openacc) to the Filepath file, but strip off the "_gpu" when setting + # the radiation package name in the config_cache file. + if ($rad_pkg eq 'rrtmgp_gpu') { + $use_rrtmgp_gpu = 1; + $rad_pkg = 'rrtmgp'; + } } # consistency checks... @@ -1102,13 +1114,18 @@ if ($rad_pkg eq 'camrt') { " with aerosol package $chem_pkg\n"; } } -elsif ($rad_pkg eq 'rrtmg') { +elsif ($rad_pkg =~ m/rrtmg/) { # The rrtmg package doesn't work with the CAM3 prescribed aerosols if ($phys_pkg eq 'cam3') { die "configure ERROR: radiation package: $rad_pkg is not compatible\n". " with physics package $phys_pkg\n"; } + + # RRTMGP not currently working with CARMA + if ($rad_pkg eq 'rrtmgp' and $carma_pkg ne 'none') { + die "configure ERROR: The CARMA microphysics package does not currently work with RRTMGP\n"; + } } $cfg_ref->set('rad', $rad_pkg); @@ -1375,7 +1392,11 @@ my $chem_cppdefs = ''; my $chem_src_dir = ''; if (!$prog_species) { - $chem_src_dir = "$cam_dir/src/chemistry/pp_$chem_pkg"; + if ($chem_pkg =~ 'geoschem') { + $chem_src_dir = "$cam_dir/src/chemistry/geoschem"; + } else { + $chem_src_dir = "$cam_dir/src/chemistry/pp_$chem_pkg"; + } $cfg_ref->set('chem_src_dir', $chem_src_dir); } @@ -1402,10 +1423,10 @@ if ($customize) { } if ($print>=2) { print "Chem preprocessor compiler: $chemproc_fc $eol"; } ($chem_nadv) = chem_preprocess($cfg_ref,$print,$chemproc_fc); -} elsif ($chem_pkg ne 'none') { +} elsif ($chem_pkg ne 'none' and $chem_pkg !~ 'geoschem') { # copy over chem docs - copy("$chem_src_dir/chem_mech.doc",$cam_bld) or die "copy failed $! \n"; - copy("$chem_src_dir/chem_mech.in" ,$cam_bld) or die "copy failed $! \n"; + copy("$chem_src_dir/chem_mech.doc",$cam_bld) or die "copy of chem_mech.doc failed $! \n"; + copy("$chem_src_dir/chem_mech.in" ,$cam_bld) or die "copy of chem_mech.in failed $! \n"; ($chem_nadv) = chem_number_adv($chem_src_dir); } @@ -1421,6 +1442,14 @@ if ($chem_pkg =~ '_mam3') { $chem_cppdefs = ' -DOSLO_AERO'; } +# Customize GEOS-Chem advected species and chemistry CPP definitions +if ($chem_pkg =~ 'geoschem') { + $chem_cppdefs .= ' -DEXTERNAL_GRID -DEXTERNAL_FORCING '; + if ($chem_pkg =~ '_mam4') { + $chem_nadv = 267; # includes GC advected species (233), CO2 (1), and MAM aerosols (33) + } +} + # CARMA sectional microphysics # # New CARMA models need to define the number of advected constituents. @@ -1583,8 +1612,6 @@ $nadv = $cfg_ref->get('nadv'); if ($print>=2) { print "Total advected constituents: $nadv$eol"; } #----------------------------------------------------------------------------------------------- - -##----------------------------------------------------------------------------------------------- # Makefile configuration ####################################################################### #----------------------------------------------------------------------------------------------- @@ -1692,7 +1719,7 @@ elsif ($fc =~ /nvfor/) { $fc_type = 'nvhpc'; } # User override for Fortran compiler type if (defined $opts{'fc_type'}) { $fc_type = $opts{'fc_type'}; } -if ($fc_type == "oneapi") {$fc_type = 'intel'; } +if ($fc_type == "intel") {$fc_type = 'intel'; } if ($fc_type) { $cfg_ref->set('fc_type', $fc_type); if ($print>=2) { print "Fortran compiler type: $fc_type$eol"; } @@ -1961,6 +1988,12 @@ if ($unicon) { $cfg_cppdefs .= ' -DUSE_UNICON'; } # HEMCO_CESM - indicates CESM model environment. Deprecated, will be removed soon. $cfg_cppdefs .= ' -DMODEL_ -DMODEL_CESM -DHEMCO_CESM -DUSE_REAL8 '; +# Compiler CPP definitions for GEOS-Chem +if ($chem_pkg =~ 'geoschem') { + if ($fc_type eq 'intel') { $cfg_cppdefs .= ' -DLINUX_IFORT'; } + elsif ($fc_type eq 'gnu') { $cfg_cppdefs .= ' -DLINUX_GFORTRAN'; } +} + #----------------------------------------------------------------------------------------------- # CPP defines to put on Makefile @@ -2157,11 +2190,21 @@ sub write_filepath print $fh "$chem_src_dir\n"; } if ($chem !~/_oslo/) { - if ($chem =~ /_mam/) { - print $fh "$camsrcdir/src/chemistry/modal_aero\n"; - } else { - print $fh "$camsrcdir/src/chemistry/bulk_aero\n"; - } + # GEOS-Chem must be prior to Mozart + if ($chem_pkg =~ 'geoschem') { + print $fh "$chem_src_dir/geoschem_src/GeosCore\n"; + print $fh "$chem_src_dir/geoschem_src/GeosUtil\n"; + print $fh "$chem_src_dir/geoschem_src/Headers\n"; + print $fh "$chem_src_dir/geoschem_src/ISORROPIA\n"; + print $fh "$chem_src_dir/geoschem_src/KPP/fullchem\n"; + print $fh "$camsrcdir/src/chemistry/pp_none\n"; + } + + if ($chem =~ /_mam/) { + print $fh "$camsrcdir/src/chemistry/modal_aero\n"; + } else { + print $fh "$camsrcdir/src/chemistry/bulk_aero\n"; + } } print $fh "$camsrcdir/src/chemistry/aerosol\n"; @@ -2204,6 +2247,19 @@ sub write_filepath elsif ($rad eq 'camrt') { print $fh "$camsrcdir/src/physics/camrt\n"; } + elsif ($rad eq 'rrtmgp') { + print $fh "$camsrcdir/src/physics/rrtmgp\n"; + print $fh "$camsrcdir/src/physics/rrtmgp/ext/extensions\n"; + print $fh "$camsrcdir/src/physics/rrtmgp/ext/gas-optics\n"; + print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp-frontend\n"; + print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte-frontend\n"; + if ($use_rrtmgp_gpu) { + print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp-kernels/accel\n"; + print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte-kernels/accel\n"; + } + print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp-kernels\n"; + print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte-kernels\n"; + } if ($clubb_sgs) { print $fh "$camsrcdir/src/physics/clubb/src/CLUBB_core\n"; @@ -2248,6 +2304,9 @@ sub write_filepath # in the list of filepaths. print $fh "$camsrcdir/src/physics/cam\n"; + #Add the CCPP'ized subdirectories + print $fh "$camsrcdir/src/atmos_phys/zhang_mcfarlane\n"; + # Dynamics package and test utilities print $fh "$camsrcdir/src/dynamics/$dyn\n"; if($dyn eq 'se') { @@ -2271,6 +2330,7 @@ sub write_filepath print $fh "$camsrcdir/src/cpl/$cpl\n"; print $fh "$camsrcdir/src/control\n"; print $fh "$camsrcdir/src/utils\n"; + print $fh "$camsrcdir/src/utils/cam_ccpp\n"; print $fh "$camsrcdir/src/atmos_phys/utilities\n";