Stochastic physics clean and updates to CA (#832)

* clean-up of the stochastic physics code and updates to the stand alone unit tests * changes to the CA code for reproducibility for different process layouts and ca_global restart capability * adds ocean stochastic physics capability to MOM6 Co-authored-by: @lisa-bengtsson
ufs-community · Sep 30, 2021 · 39dd5ba · 39dd5ba
1 parent 3e5cac8
commit 39dd5ba
Show file tree

Hide file tree

Showing 37 changed files with 4,152 additions and 3,890 deletions.
diff --git a/FV3 b/FV3
diff --git a/MOM6-interface/CMakeLists.txt b/MOM6-interface/CMakeLists.txt
@@ -31,6 +31,8 @@ target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BI
 target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
                                             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
 target_link_libraries(mom6_obj PRIVATE fms
+                                       esmf
+                                       stochastic_physics
                                        NetCDF::NetCDF_Fortran)
 # OpenMP is disabled in MOM6
 #if(OpenMP_Fortran_FOUND)
@@ -58,6 +60,7 @@ target_include_directories(mom6 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_
 target_include_directories(mom6 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
                                        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
 target_link_libraries(mom6 PUBLIC fms
+                                  stochastic_physics
                                   esmf
                                   NetCDF::NetCDF_Fortran)
 # OpenMP is disabled in MOM6
@@ -77,6 +80,8 @@ if(MOM6SOLO)
                                               $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
   target_link_libraries(mom6solo PRIVATE mom6_obj
                                          fms
+                                         stochastic_physics
+                                         esmf
                                          NetCDF::NetCDF_Fortran)
   # OpenMP is disabled in MOM6
   #if(OpenMP_Fortran_FOUND)

diff --git a/MOM6-interface/MOM6 b/MOM6-interface/MOM6
diff --git a/MOM6-interface/mom6_files.cmake b/MOM6-interface/mom6_files.cmake
@@ -254,6 +254,7 @@ list(APPEND mom6_src_files
   MOM6/config_src/external/ODA_hooks/ocean_da_core.F90
   MOM6/config_src/external/ODA_hooks/ocean_da_types.F90
   MOM6/config_src/external/ODA_hooks/write_ocean_obs.F90
+  MOM6/config_src/external/OCEAN_stochastic_phyiscs/MOM_stochastics.F90
 
   MOM6/config_src/infra/FMS1/MOM_coms_infra.F90
   MOM6/config_src/infra/FMS1/MOM_constants.F90

diff --git a/cmake/configure_apps.cmake b/cmake/configure_apps.cmake
@@ -27,6 +27,7 @@ endif()
 
 if(APP MATCHES "^(NG-GODAS|NG-GODAS-NEMSDATM)$")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(MOM6       ON  CACHE BOOL "Enable MOM6"                FORCE)
   set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)

diff --git a/stochastic_physics b/stochastic_physics
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log
diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -600,8 +600,16 @@ export CDMBWD_c192='0.23,1.5,1.0,1.0'
 export CDMBWD_c384='1.1,0.72,1.0,1.0'
 export CDMBWD_c768='4.0,0.15,1.0,1.0'
 
+# CA tunable parameter
+export RCELL_c48=0.1 
+export RCELL_c96=0.1 
+export RCELL_c192=0.2
+export RCELL_c384=0.72
+export RCELL_c768=0.72
+
 # set default
 export CDMBWD=${CDMBWD_c96}
+export RCELL=${RCELL_c96}
 
 # PBL
 export SATMEDMF=.false.
@@ -840,6 +848,11 @@ export MOM6_ALLOW_LANDMASK_CHANGES=False
 # MOM6 IAU
 export MOM_IAU=False
 export MOM_IAU_HRS=6
+# MOM6 stochastics
+export DO_OCN_SPPT=False
+export PERT_EPBL=False
+export OCN_SPPT=-999.
+export EPBL=-999.
 
 # CICE6 defaults; 1 degree
 export NPROC_ICE=12
@@ -969,6 +982,11 @@ export CHLCLIM=seawifs_1998-2006_smoothed_2X.nc
 # MOM6 IAU
 export MOM_IAU=False
 export MOM_IAU_HRS=6
+# MOM6 stochastics
+export DO_OCN_SPPT=False
+export PERT_EPBL=False
+export OCN_SPPT=-999.
+export EPBL=-999.
 # this must be set False for restart repro
 export MOM6_REPRO_LA=False
 # since coupling_interval_slow is set to DT_THERM, this should be always be false
@@ -1092,6 +1110,11 @@ export CHLCLIM=seawifs_1998-2006_smoothed_2X.nc
 # MOM6 IAU
 export MOM_IAU=False
 export MOM_IAU_HRS=6
+# MOM6 stochastics
+export DO_OCN_SPPT=False
+export PERT_EPBL=False
+export OCN_SPPT=-999.
+export EPBL=-999.
 # this must be set False for restart repro
 export MOM6_REPRO_LA=False
 # since coupling_interval_slow is set to DT_THERM, this should be always be false

diff --git a/tests/fv3_conf/control_run.IN b/tests/fv3_conf/control_run.IN
@@ -27,7 +27,7 @@ else
   if [ $MODEL_INITIALIZATION = false ]; then
     #read_inc and iau tests restart from fh=24
     CURRDIR=`pwd`
-    FILEDIR=${CURRDIR}/../control${SUFFIX}/RESTART/
+    FILEDIR=${CURRDIR}/../${DEP_RUN}${SUFFIX}/RESTART/
   else
     #for warm start from initialiation time
     FILEDIR=@[INPUTDATA_ROOT]/${inputdir}/INPUT_L127_GDAS
@@ -53,18 +53,22 @@ else
     cp ${FILEDIR}/phy_data.* INPUT/.
     cp ${FILEDIR}/sfc_data.* INPUT/.
   else
-    #restart test start from fh=12
-    rsync -arv ${FILEDIR} INPUT/
-    cp ${FILEDIR}/20210322.180000.* INPUT/.
+    #restart test start from fh FHROT
+    cp -r ../${DEP_RUN}${SUFFIX}/RESTART/${RESTART_FILE_PREFIX}.* ./INPUT
+    rm -f INPUT/fv_core.res.*
+    rm -f INPUT/fv_srf_wnd.res.*
+    rm -f INPUT/fv_tracer.res.*
+    rm -f INPUT/phy_data.*
+    rm -f INPUT/sfc_data.*
     cp ${FILEDIR}/../INPUT/grid_spec.nc INPUT/.
     cp ${FILEDIR}/../INPUT/*_grid.tile*.nc INPUT/.
     cp ${FILEDIR}/../INPUT/oro_data.tile*.nc INPUT/.
-    cd INPUT
-    for RFILE in 20210322.180000.*; do
+    for RFILE in ../${DEP_RUN}${SUFFIX}/RESTART/${RESTART_FILE_PREFIX}.*; do
       [ -e $RFILE ] || exit 1
-      mv $RFILE ${RFILE#20210322.180000.}
+      RFILE_OLD=$(basename $RFILE)
+      RFILE_NEW="${RFILE_OLD//${RESTART_FILE_PREFIX}./}"
+      cp $RFILE INPUT/${RFILE_NEW}
     done
-    cd ..
   fi
 
 fi

diff --git a/tests/fv3_conf/control_stochy_run.IN b/tests/fv3_conf/control_stochy_run.IN
diff --git a/tests/parm/MOM_input_template_025 b/tests/parm/MOM_input_template_025
@@ -858,6 +858,11 @@ SEA_ICE_RIGID_MASS = 100.0      !   [kg m-2] default = 1000.0
 LIQUID_RUNOFF_FROM_DATA = @[MOM6_RIVER_RUNOFF]  !   [Boolean] default = False
                                 ! If true, allows liquid river runoff to be specified via
                                 ! the data_table using the component name 'OCN'.
+! === module ocean_stochastics ===
+DO_SPPT   = @[DO_OCN_SPPT]      ! [Boolean] default = False
+                                ! If true perturb the diabatic tendencies in MOM_diabadic_driver 
+PERT_EPBL = @[PERT_EPBL]        ! [Boolean] default = False 
+                                ! If true perturb the KE dissipation and destruction in MOM_energetic_PBL
 ! === module MOM_restart ===
 RESTART_CHECKSUMS_REQUIRED = False
 ! === module MOM_file_parser ===
diff --git a/tests/parm/MOM_input_template_050 b/tests/parm/MOM_input_template_050
@@ -912,6 +912,11 @@ SEA_ICE_RIGID_MASS = 100.0      !   [kg m-2] default = 1000.0
 LIQUID_RUNOFF_FROM_DATA = @[MOM6_RIVER_RUNOFF]  !   [Boolean] default = False
                                 ! If true, allows liquid river runoff to be specified via
                                 ! the data_table using the component name 'OCN'.
+! === module ocean_stochastics ===
+DO_SPPT   = @[DO_OCN_SPPT]      ! [Boolean] default = False
+                                ! If true perturb the diabatic tendencies in MOM_diabadic_driver 
+PERT_EPBL = @[PERT_EPBL]        ! [Boolean] default = False 
+                                ! If true perturb the KE dissipation and destruction in MOM_energetic_PBL
 ! === module MOM_restart ===
 
 ! === module MOM_file_parser ===
diff --git a/tests/parm/MOM_input_template_100 b/tests/parm/MOM_input_template_100
@@ -851,6 +851,11 @@ GUST_CONST = 0.02               !   [Pa] default = 0.0
 FIX_USTAR_GUSTLESS_BUG = False  !   [Boolean] default = True
                                 ! If true correct a bug in the time-averaging of the gustless wind friction
                                 ! velocity
+! === module ocean_stochastics ===
+DO_SPPT   = @[DO_OCN_SPPT]      ! [Boolean] default = False
+                                ! If true perturb the diabatic tendencies in MOM_diabadic_driver 
+PERT_EPBL = @[PERT_EPBL]        ! [Boolean] default = False 
+                                ! If true perturb the KE dissipation and destruction in MOM_energetic_PBL
 
 ! === module MOM_restart ===
 

diff --git a/tests/parm/control_ca.nml.IN b/tests/parm/control_ca.nml.IN
@@ -187,7 +187,7 @@ deflate_level=1
   tlives         = 1800
   nseed          = 100
   nfracseed      = 0.5
-  rcell          = 0.72
+  rcell          = @[RCELL]
   ca_trigger     = .True.
   ca_entr        = .False.
   ca_closure     = .False.

diff --git a/tests/parm/input.mom6.nml.IN b/tests/parm/input.mom6.nml.IN
@@ -12,4 +12,17 @@
          parameter_filename = 'INPUT/MOM_input',
                               'INPUT/MOM_override' /
 
-
+
+&nam_stochy
+new_lscale=.true.,
+OCNSPPT=@[OCN_SPPT],
+OCNSPPT_LSCALE=500e3,
+OCNSPPT_TAU=21600,
+ISEED_OCNSPPT=201601010012,
+EPBL=@[EPBL],
+EPBL_LSCALE=500e3,
+EPBL_TAU=21600,
+ISEED_EPBL=201601010023,
+/
+&nam_sfcperts
+/
diff --git a/tests/parm/input.mom6_ccpp.nml.IN b/tests/parm/input.mom6_ccpp.nml.IN
@@ -194,7 +194,7 @@
        tlives         = 1800
        nseed          = 1000000
        nfracseed      = 0.5
-       rcell          = 0.72
+       rcell          = @[RCELL]
        ca_trigger     = .true.
        ca_entr	      = .false.
        ca_closure     = .false.

diff --git a/tests/rt.conf b/tests/rt.conf
@@ -50,6 +50,7 @@ RUN     | control_c384gdas
 RUN     | control_stochy                                                                                                          |                                         | fv3 |
 RUN     | control_stochy_restart                                                                                                  |                                         |     | control_stochy
 RUN     | control_ca                                                                                                              |                                         | fv3 |
+RUN     | control_ca_restart                                                                                                      |                                         |     | control_ca
 RUN     | control_lndp                                                                                                            |                                         | fv3 |
 RUN     | control_lheatstrg                                                                                                       |                                         | fv3 |
 RUN     | control_lseaspray                                                                                                       |                                         | fv3 |
@@ -180,6 +181,7 @@ COMPILE | -DAPP=NG-GODAS
 RUN     | datm_cdeps_control_cfsr                                                                                                 | - wcoss_cray                            | fv3 |
 RUN     | datm_cdeps_restart_cfsr                                                                                                 | - wcoss_cray                            |     | datm_cdeps_control_cfsr
 RUN     | datm_cdeps_control_gefs                                                                                                 | - wcoss_cray                            | fv3 |
+RUN     | datm_cdeps_stochy_gefs                                                                                                  | - wcoss_cray                            | fv3 |
 
 RUN     | datm_cdeps_bulk_cfsr                                                                                                    | - wcoss_cray                            | fv3 |
 RUN     | datm_cdeps_bulk_gefs                                                                                                    | - wcoss_cray                            | fv3 |

diff --git a/tests/rt.sh b/tests/rt.sh
@@ -444,7 +444,7 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20210928
+BL_DATE=20210929
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]] || [[ $MACHINE_ID = s4.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else