Updated some diff files with changes from ESCOMP

gold2718 · May 2, 2024 · 7301fd2 · 7301fd2
1 parent 0310b57
commit 7301fd2
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 33 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,7 @@ src/physics/carma/base
 src/physics/clubb
 src/physics/cosp2/src
 src/physics/silhs
+src/chemistry/geoschem/geoschem_src
 src/physics/pumas
 src/physics/pumas-frozen
 src/dynamics/fv3/atmos_cubed_sphere

diff --git a/Externals.cfg b/Externals.cfg
@@ -4,7 +4,7 @@
 
 [parallelio]
 protocol = git
-tag = pio2_5_10
+tag = pio2_6_2
 repo_url = https://github.com/NCAR/ParallelIO
 local_path = libraries/parallelio
 required = True

diff --git a/Externals_CAM.cfg b/Externals_CAM.cfg
@@ -7,7 +7,7 @@ required = True
 
 [carma]
 protocol = git
-tag = carma4_00
+tag = carma4_01
 repo_url = https://github.com/ESCOMP/CARMA_base.git
 local_path = src/physics/carma/base
 required = True
@@ -22,15 +22,15 @@ required = True
 
 [clubb]
 protocol = git
-tag = clubb_4ncar_20221129_59cb19f_20230330_branchtag
+tag = clubb_4ncar_20231115_5406350
 repo_url = https://github.com/larson-group/clubb_release
 local_path = src/physics/clubb
 sparse = ../.clubb_sparse_checkout
 required = True
 
 [pumas]
 protocol = git
-tag = pumas_cam-release_v1.29
+tag = pumas_cam-release_v1.35
 repo_url = https://github.com/ESCOMP/PUMAS
 local_path = src/physics/pumas
 required = True
@@ -51,7 +51,7 @@ required = True
 
 [atmos_phys]
 protocol = git
-tag = atmos_phys0_00_011
+tag = atmos_phys0_02_006
 repo_url = https://github.com/NCAR/atmospheric_physics
 local_path = src/atmos_phys
 required = True
@@ -72,11 +72,25 @@ required = True
 
 [hemco]
 protocol = git
-tag = hemco-cesm1_2_0_hemco3_6_2_cesm
+tag = hemco-cesm1_2_1_hemco3_6_3_cesm
 repo_url = https://github.com/ESCOMP/HEMCO_CESM.git
 local_path = src/hemco
 required = True
 externals = Externals_HCO.cfg
 
+[rte-rrtmgp]
+local_path = src/physics/rrtmgp/ext
+protocol = git
+repo_url = https://github.com/earth-system-radiation/rte-rrtmgp.git
+tag = v1.7
+required = True
+
+[rrtmgp-data]
+local_path = src/physics/rrtmgp/data
+protocol = git
+repo_url = https://github.com/earth-system-radiation/rrtmgp-data.git
+tag = v1.8
+required = True
+
 [externals_description]
 schema_version = 1.0.01
diff --git a/bld/config_files/definition.xml b/bld/config_files/definition.xml
@@ -66,10 +66,10 @@ Switch to turn on Harmonized Emissions Component (HEMCO) for chemistry: 0 => no,
 <entry id="camnor" valid_values="0,1" value="0">
 Option to turn on NorESM modifications to baseline CESM/CAM physics and dynamics
 </entry>
-<entry id="microphys" valid_values="rk,mg1,mg2,mg3,spcam_m2005,spcam_sam1mom,none" value="">
+<entry id="microphys" valid_values="rk,mg1,mg2,mg3,pumas,spcam_m2005,spcam_sam1mom,none" value="">
 Microphysics package: rk (Rasch and Kristjansson), mg1 (Morrison and
 Gettelman two moment scheme CAM5.1), mg2 (Morrison and Gettelman second
-version CAM6), mg3 (MG scheme 3rd version, graupel), SPCAM_m2005, SPCAM_sam1mom.
+version CAM6), mg3 (MG scheme 3rd version, graupel), PUMAS, SPCAM_m2005, SPCAM_sam1mom.
 </entry>
 <entry id="macrophys" valid_values="rk,park,clubb_sgs,spcam_sam1mom,spcam_m2005,none" value="">
 Macrophysics package: RK, Park, CLUBB_SGS, SPCAM_sam1mom, SPCAM_m2005.
@@ -94,9 +94,9 @@ on Mapes and Neale (2011): 0 => no, 1 => yes
 PBL package: uw (University of Washington), hb (Holtslag and Boville), hbr
  (Holtslag, Boville, and Rasch), clubb_sgs, spcam_sam1om, spcam_m2005, none.
 </entry>
-<entry id="rad" valid_values="rrtmg,camrt,none" value="">
+<entry id="rad" valid_values="rrtmgp,rrtmg,camrt,none" value="">
 Radiative transfer calculation:
-camrt (CAM3 and CAM4 RT package), rrtmg (RRTMG package from AER).
+camrt (CAM3 and CAM4 RT package), rrtmg (RRTMG package from AER), rrtmgp (updated version).
 </entry>
 <entry id="carma" valid_values="none,bc_strat,cirrus,cirrus_dust,dust,meteor_impact,meteor_smoke,mixed_sulfate,pmc,pmc_sulfate,sea_salt,sulfate,tholin,test_detrain,test_growth,test_passive,test_radiative,test_swelling,test_tracers,test_tracers2" value="none">
 CARMA sectional microphysics:
@@ -106,8 +106,8 @@ meteor_smoke (Meteor Smoke), mixed_sulfate (Meteor Smoke and Sulfate), pmc (Pola
 sulfate (Sulfate Aerosols), tholin (early earth haze), test_detrain (Detrainment), test_growth (Particle Growth), test_passive (Passive Dust),
 test_radiative (Radiatively Active Dust), test_swelling (Sea Salt), test_tracers (Asian Monsoon), test_tracers2 (Guam).
 </entry>
-<entry id="chem" valid_values="none,ghg_mam4,terminator,trop_mam3,trop_mam4,trop_mam7,trop_mam_oslo,trop_mozart,trop_strat_mam4_ts2,trop_strat_mam4_vbs,trop_strat_mam4_vbsext,trop_strat_mam5_ts2,trop_strat_mam5_vbs,trop_strat_mam5_vbsext,waccm_ma,waccm_mad,waccm_ma_sulfur,waccm_sc,waccm_sc_mam4,waccm_mad_mam4,waccm_ma_mam4,waccm_tsmlt_mam4,waccm_tsmlt_mam4_vbsext,waccm_mad_mam5,waccm_ma_mam5,waccm_tsmlt_mam5,waccm_tsmlt_mam5_vbsext" value="">
-  Chemistry package: none,ghg_mam4,terminator,trop_mam3,trop_mam4,trop_mam7,trop_mam_oslo,trop_mozart,trop_strat_mam4_ts2,trop_strat_mam4_vbs,trop_strat_mam4_vbsext,trop_strat_mam5_ts2,trop_strat_mam5_vbs,trop_strat_mam5_vbsext,waccm_ma,waccm_mad,waccm_ma_sulfur,waccm_sc,waccm_sc_mam4,waccm_mad_mam4,waccm_ma_mam4,waccm_tsmlt_mam4,waccm_tsmlt_mam4_vbsext,waccm_mad_mam5,waccm_ma_mam5,waccm_tsmlt_mam5,waccm_tsmlt_mam5_vbsext
+<entry id="chem" valid_values="none,ghg_mam4,terminator,trop_mam3,trop_mam4,trop_mam7,trop_mam_oslo,trop_mozart,trop_strat_mam4_ts2,trop_strat_mam4_vbs,trop_strat_mam4_vbsext,trop_strat_mam5_ts2,trop_strat_mam5_vbs,trop_strat_mam5_vbsext,waccm_ma,waccm_mad,waccm_ma_sulfur,waccm_sc,waccm_sc_mam4,waccm_mad_mam4,waccm_ma_mam4,waccm_tsmlt_mam4,waccm_tsmlt_mam4_vbsext,waccm_mad_mam5,waccm_ma_mam5,waccm_tsmlt_mam5,waccm_tsmlt_mam5_vbsext,geoschem_mam4" value="">
+  Chemistry package: none,ghg_mam4,terminator,trop_mam3,trop_mam4,trop_mam7,trop_mam_oslo,trop_mozart,trop_strat_mam4_ts2,trop_strat_mam4_vbs,trop_strat_mam4_vbsext,trop_strat_mam5_ts2,trop_strat_mam5_vbs,trop_strat_mam5_vbsext,waccm_ma,waccm_mad,waccm_ma_sulfur,waccm_sc,waccm_sc_mam4,waccm_mad_mam4,waccm_ma_mam4,waccm_tsmlt_mam4,waccm_tsmlt_mam4_vbsext,waccm_mad_mam5,waccm_ma_mam5,waccm_tsmlt_mam5,waccm_tsmlt_mam5_vbsext,geoschem_mam4
 </entry>
 <entry id="prog_species" valid_values="DST,SSLT,SO4,GHG,OC,BC,CARBON16" value="" list="1">
 Prognostic mozart species packages: list of any subset of the following: DST,SSLT,SO4,GHG,OC,BC,CARBON16

diff --git a/bld/configure b/bld/configure
@@ -69,7 +69,7 @@ OPTIONS
                          trop_strat_mam4_vbs | trop_strat_mam4_vbsext | trop_strat_mam5_ts2 | trop_strat_mam5_vbs |
                          trop_strat_mam5_vbsext | waccm_ma | waccm_mad | waccm_ma_sulfur | waccm_sc | waccm_sc_mam4 |
                          waccm_mad_mam4 | waccm_ma_mam4 | waccm_tsmlt_mam4 | waccm_tsmlt_mam4_vbsext | waccm_mad_mam5 |
-                         waccm_ma_mam5 | waccm_tsmlt_mam5 | waccm_tsmlt_mam5_vbsext ].
+                         waccm_ma_mam5 | waccm_tsmlt_mam5 | waccm_tsmlt_mam5_vbsext | geoschem_mam4 ].
                         Default: trop_mam_oslo when -camnor is specified
                         Default: trop_mam5 for cam7
                         Default: trop_mam4 for cam6
@@ -93,7 +93,7 @@ OPTIONS
      -macrophys <name>  Specify the macrophysics option [rk | park | clubb_sgs].
      -max_n_rad_cnst <n> Maximum number of constituents that are either radiatively
                         active, or in any single diagnostic list for the radiation.
-     -microphys <name>  Specify the microphysics option [mg1 | mg2 | mg3| rk].
+     -microphys <name>  Specify the microphysics option [mg1 | mg2 | mg3| rk | pumas].
      -model_top <name>  Specify the model_top option [ lt | mt ].
      -nadv <n>          Set total number of advected species to <n>.
      -nadv_tt <n>       Set number of advected test tracers <n>.
@@ -108,7 +108,7 @@ OPTIONS
      -prog_species <list>Comma-separate list of prognostic mozart species packages.
                         Currently available: DST,SSLT,SO4,GHG,OC,BC,CARBON16
      -psubcols <n>      Maximum number of sub-columns in a run - set to 1 if not using sub-columns (default)
-     -rad <name>        Specify the radiation package [rrtmg | camrt]
+     -rad <name>        Specify the radiation package [rrtmg | rrtmgp | rrtmgp_gpu | camrt]
      -silhs             Switch on SILHS.
      -spcam_clubb_sgs   Turn on the SPCAM version of CLUBB
      -spcam_nx <n>      SPCAM x-grid. - defaults to 4 (note the CRM requires spcam_nx to be greater than or equal to 4)
@@ -861,6 +861,10 @@ if (defined $opts{'microphys'}) {
     $microphys_pkg = lc($opts{'microphys'});
 }
 
+if($microphys_pkg eq 'pumas') {
+    $microphys_pkg = 'mg3';
+}
+
 $cfg_ref->set('microphys', $microphys_pkg);
 
 if ($print>=2) { print "Microphysics package: $microphys_pkg$eol"; }
@@ -879,7 +883,7 @@ if ($carma_pkg =~ m/cirrus/i) {
     unless ($microphys_pkg =~ /^mg/) {
         die <<"EOF";
 **    ERROR: microphysics package set to: $microphys_pkg
-**    The CARMA cirrus model only works with MG microphysics.
+**    The CARMA cirrus model only works with MG or PUMAS microphysics.
 EOF
     }
 }
@@ -902,11 +906,11 @@ if (defined $opts{'clubb_sgs'}) {
 
 # consistency checks...
 
-# CLUBB_SGS only works with mg microphysics
+# CLUBB_SGS only works with mg or PUMAS microphysics
 if ($clubb_sgs and not ($microphys_pkg =~ m/^mg/ )) {
     die <<"EOF";
 **    ERROR: microphysics package set to: $microphys_pkg
-**    CLUBB_SGS only works with MG microphysics.
+**    CLUBB_SGS only works with MG or PUMAS microphysics.
 EOF
 }
 
@@ -1041,12 +1045,12 @@ if (defined $opts{'pbl'}) {
 
 # consistency checks...
 
-# UW PBL only works with mg microphysics
+# UW PBL only works with mg or PUMAS microphysics
 if ($pbl_pkg =~ m/uw/i) {
     unless ($microphys_pkg =~ /^mg/) {
         die <<"EOF";
 **    ERROR: microphysics package set to: $microphys_pkg
-**    The UW PBL scheme only works with MG microphysics.
+**    The UW PBL scheme only works with MG or PUMAS microphysics.
 EOF
     }
 }
@@ -1088,8 +1092,16 @@ elsif ($phys_pkg =~ m/^cam[56]$|^cam_dev$|^spcam_m2005$/) {
     $rad_pkg = 'rrtmg';
 }
 # Allow the user to override the default via the commandline.
+my $use_rrtmgp_gpu = 0;
 if (defined $opts{'rad'}) {
     $rad_pkg = lc($opts{'rad'});
+    # If the radiation package is set to rrtmgp_gpu then will add the gpu code version
+    # (openmp and openacc) to the Filepath file, but strip off the "_gpu" when setting
+    # the radiation package name in the config_cache file.
+    if ($rad_pkg eq 'rrtmgp_gpu') {
+        $use_rrtmgp_gpu = 1;
+        $rad_pkg = 'rrtmgp';
+    }
 }
 
 # consistency checks...
@@ -1102,13 +1114,18 @@ if ($rad_pkg eq 'camrt') {
             "                 with aerosol package $chem_pkg\n";
     }
 }
-elsif ($rad_pkg eq 'rrtmg') {
+elsif ($rad_pkg =~ m/rrtmg/) {
 
     # The rrtmg package doesn't work with the CAM3 prescribed aerosols
     if ($phys_pkg eq 'cam3') {
         die "configure ERROR: radiation package: $rad_pkg is not compatible\n".
             "                 with physics package $phys_pkg\n";
     }
+
+    # RRTMGP not currently working with CARMA
+    if ($rad_pkg eq 'rrtmgp' and $carma_pkg ne 'none') {
+        die "configure ERROR: The CARMA microphysics package does not currently work with RRTMGP\n";
+    }
 }
 
 $cfg_ref->set('rad', $rad_pkg);
@@ -1375,7 +1392,11 @@ my $chem_cppdefs = '';
 my $chem_src_dir = '';
 
 if (!$prog_species) {
-  $chem_src_dir = "$cam_dir/src/chemistry/pp_$chem_pkg";
+  if ($chem_pkg =~ 'geoschem') {
+    $chem_src_dir = "$cam_dir/src/chemistry/geoschem";
+  } else {
+    $chem_src_dir = "$cam_dir/src/chemistry/pp_$chem_pkg";
+  }
   $cfg_ref->set('chem_src_dir', $chem_src_dir);
 }
 
@@ -1402,10 +1423,10 @@ if ($customize) {
     }
     if ($print>=2) { print "Chem preprocessor compiler: $chemproc_fc $eol"; }
     ($chem_nadv) = chem_preprocess($cfg_ref,$print,$chemproc_fc);
-} elsif ($chem_pkg ne 'none') {
+} elsif ($chem_pkg ne 'none' and $chem_pkg !~ 'geoschem') {
    # copy over chem docs
-    copy("$chem_src_dir/chem_mech.doc",$cam_bld) or die "copy failed $! \n";
-    copy("$chem_src_dir/chem_mech.in" ,$cam_bld) or die "copy failed $! \n";
+    copy("$chem_src_dir/chem_mech.doc",$cam_bld) or die "copy of chem_mech.doc failed $! \n";
+    copy("$chem_src_dir/chem_mech.in" ,$cam_bld) or die "copy of chem_mech.in failed $! \n";
     ($chem_nadv) = chem_number_adv($chem_src_dir);
 }
 
@@ -1421,6 +1442,14 @@ if ($chem_pkg =~ '_mam3') {
     $chem_cppdefs = ' -DOSLO_AERO';
 }
 
+# Customize GEOS-Chem advected species and chemistry CPP definitions
+if ($chem_pkg =~ 'geoschem') {
+    $chem_cppdefs .= ' -DEXTERNAL_GRID -DEXTERNAL_FORCING ';
+    if ($chem_pkg =~ '_mam4') {
+        $chem_nadv = 267; # includes GC advected species (233), CO2 (1), and MAM aerosols (33)
+    }
+}
+
 # CARMA sectional microphysics
 #
 # New CARMA models need to define the number of advected constituents.
@@ -1583,8 +1612,6 @@ $nadv = $cfg_ref->get('nadv');
 if ($print>=2) { print "Total advected constituents: $nadv$eol"; }
 
 #-----------------------------------------------------------------------------------------------
-
-##-----------------------------------------------------------------------------------------------
 # Makefile configuration #######################################################################
 #-----------------------------------------------------------------------------------------------
 
@@ -1692,7 +1719,7 @@ elsif ($fc =~ /nvfor/)  { $fc_type = 'nvhpc'; }
 
 # User override for Fortran compiler type
 if (defined $opts{'fc_type'}) { $fc_type = $opts{'fc_type'}; }
-if ($fc_type == "oneapi") {$fc_type = 'intel'; }
+if ($fc_type == "intel") {$fc_type = 'intel'; }
 if ($fc_type) {
     $cfg_ref->set('fc_type', $fc_type);
     if ($print>=2) { print "Fortran compiler type: $fc_type$eol"; }
@@ -1961,6 +1988,12 @@ if ($unicon) { $cfg_cppdefs .= ' -DUSE_UNICON'; }
 #   HEMCO_CESM - indicates CESM model environment. Deprecated, will be removed soon.
 $cfg_cppdefs .= ' -DMODEL_ -DMODEL_CESM -DHEMCO_CESM -DUSE_REAL8 ';
 
+# Compiler CPP definitions for GEOS-Chem
+if ($chem_pkg =~ 'geoschem') {
+    if    ($fc_type eq 'intel') { $cfg_cppdefs .= ' -DLINUX_IFORT'; }
+    elsif ($fc_type eq 'gnu')   { $cfg_cppdefs .= ' -DLINUX_GFORTRAN'; }
+}
+
 #-----------------------------------------------------------------------------------------------
 # CPP defines to put on Makefile
 
@@ -2157,11 +2190,21 @@ sub write_filepath
         print $fh "$chem_src_dir\n";
     }
     if ($chem !~/_oslo/) {
-        if ($chem =~ /_mam/) {
-            print $fh "$camsrcdir/src/chemistry/modal_aero\n";
-        } else {
-            print $fh "$camsrcdir/src/chemistry/bulk_aero\n";
-        }
+        # GEOS-Chem must be prior to Mozart
+      if ($chem_pkg =~ 'geoschem') {
+	  print $fh "$chem_src_dir/geoschem_src/GeosCore\n";
+          print $fh "$chem_src_dir/geoschem_src/GeosUtil\n";
+          print $fh "$chem_src_dir/geoschem_src/Headers\n";
+          print $fh "$chem_src_dir/geoschem_src/ISORROPIA\n";
+          print $fh "$chem_src_dir/geoschem_src/KPP/fullchem\n";
+          print $fh "$camsrcdir/src/chemistry/pp_none\n";
+      }
+
+      if ($chem =~ /_mam/) {
+          print $fh "$camsrcdir/src/chemistry/modal_aero\n";
+      } else {
+          print $fh "$camsrcdir/src/chemistry/bulk_aero\n";
+      }
     }
     print $fh "$camsrcdir/src/chemistry/aerosol\n";
 
@@ -2204,6 +2247,19 @@ sub write_filepath
     elsif ($rad eq 'camrt') {
         print $fh "$camsrcdir/src/physics/camrt\n";
     }
+    elsif ($rad eq 'rrtmgp') {
+        print $fh "$camsrcdir/src/physics/rrtmgp\n";
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/extensions\n";
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/gas-optics\n";
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp-frontend\n";
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte-frontend\n";
+        if ($use_rrtmgp_gpu) {
+            print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp-kernels/accel\n";
+            print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte-kernels/accel\n";
+        }
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/rrtmgp-kernels\n";
+        print $fh "$camsrcdir/src/physics/rrtmgp/ext/rte-kernels\n";
+    }
 
     if ($clubb_sgs) {
        print $fh "$camsrcdir/src/physics/clubb/src/CLUBB_core\n";
@@ -2248,6 +2304,9 @@ sub write_filepath
     # in the list of filepaths.
     print $fh "$camsrcdir/src/physics/cam\n";
 
+    #Add the CCPP'ized subdirectories
+    print $fh "$camsrcdir/src/atmos_phys/zhang_mcfarlane\n";
+
     # Dynamics package and test utilities
     print $fh "$camsrcdir/src/dynamics/$dyn\n";
     if($dyn eq 'se') {
@@ -2271,6 +2330,7 @@ sub write_filepath
     print $fh "$camsrcdir/src/cpl/$cpl\n";
     print $fh "$camsrcdir/src/control\n";
     print $fh "$camsrcdir/src/utils\n";
+    print $fh "$camsrcdir/src/utils/cam_ccpp\n";
     print $fh "$camsrcdir/src/atmos_phys/utilities\n";