Merge branch 'develop' into feature/stable-nightly

CMakeLists.txt

@@ -96,6 +96,9 @@ if(BUILD_GDASBUNDLE)
   ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" TAG 51ec489 RECURSIVE )
   ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" TAG 7efe282 )
 
+  # Build JEDI/DA or other peripherals
+  ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" )
+
 # Build IODA converters
   option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON)
   if(BUILD_IODA_CONVERTERS)

build.sh

@@ -114,7 +114,7 @@ set -x
 if [[ $BUILD_JCSDA == 'YES' ]]; then
   make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE
 else
-  builddirs="fv3-jedi soca iodaconv land-imsproc land-jediincr"
+  builddirs="fv3-jedi soca iodaconv land-imsproc land-jediincr gdas-utils"
   for b in $builddirs; do
     cd $b
     make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE

parm/soca/variational/socaincr2mom6.yaml

@@ -0,0 +1,27 @@
+geometry:
+  mom6_input_nml: mom_input.nml
+  fields metadata: ./fields_metadata.yaml
+
+date: '{{ATM_WINDOW_BEGIN}}'
+
+layers variable: [hocn]
+
+increment variables: [tocn, socn, uocn, vocn, ssh]
+
+vertical geometry:
+  date: '{{ATM_WINDOW_BEGIN}}'
+  basename: ./INPUT/
+  ocn_filename: MOM.res.nc
+  read_from_file: 1
+
+soca increment:
+  date: '{{ATM_WINDOW_BEGIN}}'
+  basename: ./Data/
+  ocn_filename: 'ocn.3dvarfgat_pseudo.incr.{{ATM_WINDOW_BEGIN}}.nc'
+  read_from_file: 1
+
+mom6 iau increment:
+  datadir: ./
+  date: '{{ATM_WINDOW_BEGIN}}'
+  exp: mom6_iau
+  type: incr

scripts/exgdas_global_marine_analysis_chkpt.sh

@@ -55,10 +55,7 @@ EOF
                                                 --out "${mom6_iau_incr}" \
                                                 --nsst_yaml "nsst.yaml"
 else
-   ${HOMEgfs}/sorc/gdas.cd/ush/socaincr2mom6.py --incr "${soca_incr}" \
-                                                --bkg "${DATA}/INPUT/MOM.res.nc" \
-                                                --grid "${DATA}/soca_gridspec.nc" \
-                                                --out "${mom6_iau_incr}"
+   $APRUN_OCNANAL ${JEDI_BIN}/gdas_socaincr2mom6.x socaincr2mom6.yaml
 fi
 export err=$?
 if [ $err -gt 0  ]; then

scripts/exgdas_global_marine_analysis_prep.py

@@ -472,6 +472,14 @@ def find_clim_ens(input_date):
     yaml.dump(soca2cice_cfg, f, sort_keys=False, default_flow_style=False)
     ufsda.genYAML.genYAML('tmp.yaml', output=varchgyaml)
 
+# prepare yaml for soca to MOM6 IAU increment
+logging.info(f"---------------- generate soca to MOM6 IAU yaml")
+socaincr2mom6_yaml = os.path.join(anl_dir, 'socaincr2mom6.yaml')
+socaincr2mom6_yaml_template = os.path.join(gdas_home, 'parm', 'soca', 'variational', 'socaincr2mom6.yaml')
+s2mconfig = YAMLFile(path=socaincr2mom6_yaml_template)
+s2mconfig = Template.substitute_structure(s2mconfig, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get)
+s2mconfig.save(socaincr2mom6_yaml)
+
 ################################################################################
 # Copy initial condition
 ics_list = []

scripts/exgdas_global_marine_analysis_vrfy.py

@@ -19,108 +19,13 @@
 
 import os
 import numpy as np
-import matplotlib.pyplot as plt
-import xarray as xr
-import cartopy
-import cartopy.crs as ccrs
 import gen_eva_obs_yaml
 import marine_eva_post
 import diag_statistics
+from soca_vrfy import plot_config, plot_horizontal_slice, plot_zonal_slice
 import subprocess
 from datetime import datetime, timedelta
 
-projs = {'North': ccrs.NorthPolarStereo(),
-         'South': ccrs.SouthPolarStereo(),
-         'Global': ccrs.Mollweide(central_longitude=-150)}
-
-
-def plot_config(grid_file=[], data_file=[],
-                variable=[], levels=[], bounds=[], colormap=[], comout=[], lats=[]):
-    """
-    Prepares the configuration for the plotting functions below
-    """
-    config = {}
-    config['grid file'] = grid_file
-    config['fields file'] = data_file
-    config['variable'] = variable
-    config['levels'] = levels
-    config['bounds'] = bounds
-    config['colormap'] = colormap
-    config['lats'] = lats
-    config['comout'] = comout
-    config['max depth'] = 5000.0
-    config['proj'] = 'Global'
-    return config
-
-
-def plot_horizontal_slice(config):
-    """
-    pcolormesh of a horizontal slice of an ocean field
-    """
-    grid = xr.open_dataset(config['grid file'])
-    data = xr.open_dataset(config['fields file'])
-
-    dirname = os.path.join(config['comout'], config['variable'])
-    os.makedirs(dirname, exist_ok=True)
-
-    if config['variable'] in ['Temp', 'Salt', 'u', 'v']:
-        level = config['levels'][0]
-        slice_data = np.squeeze(data[config['variable']])[level, :, :]
-        label_colorbar = config['variable']+' Level '+str(level)
-        figname = os.path.join(dirname, config['variable']+'_Level_'+str(level))
-    else:
-        slice_data = np.squeeze(data[config['variable']])
-        label_colorbar = config['variable']
-        figname = os.path.join(dirname, config['variable']+'_'+config['proj'])
-
-    bounds = config['bounds']
-
-    fig, ax = plt.subplots(figsize=(8, 5), subplot_kw={'projection': projs[config['proj']]})
-    plt.pcolormesh(np.squeeze(grid.lon),
-                   np.squeeze(grid.lat),
-                   slice_data,
-                   vmin=bounds[0], vmax=bounds[1],
-                   transform=ccrs.PlateCarree(),
-                   cmap=config['colormap'])
-
-    plt.colorbar(label=label_colorbar, shrink=0.5, orientation='horizontal')
-    ax.coastlines()  # TODO: make this work on hpc
-    ax.gridlines(draw_labels=True)
-    if config['proj'] == 'South':
-        ax.set_extent([-180, 180, -90, -50], ccrs.PlateCarree())
-    if config['proj'] == 'North':
-        ax.set_extent([-180, 180, 50, 90], ccrs.PlateCarree())
-    # ax.add_feature(cartopy.feature.LAND)  # TODO: make this work on hpc
-    plt.savefig(figname, bbox_inches='tight', dpi=600)
-
-
-def plot_zonal_slice(config):
-    """
-    pcolormesh of a zonal slice of an ocean field
-    """
-    lat = float(config['lats'][0])
-    grid = xr.open_dataset(config['grid file'])
-    data = xr.open_dataset(config['fields file'])
-    lat_index = np.argmin(np.array(np.abs(np.squeeze(grid.lat)[:, 0]-lat)))
-    slice_data = np.squeeze(np.array(data[config['variable']]))[:, lat_index, :]
-    depth = np.squeeze(np.array(grid['h']))[:, lat_index, :]
-    depth[np.where(np.abs(depth) > 10000.0)] = 0.0
-    depth = np.cumsum(depth, axis=0)
-    bounds = config['bounds']
-    x = np.tile(np.squeeze(grid.lon[:, lat_index]), (np.shape(depth)[0], 1))
-    fig, ax = plt.subplots(figsize=(8, 5))
-    plt.pcolormesh(x, -depth, slice_data,
-                   vmin=bounds[0], vmax=bounds[1],
-                   cmap=config['colormap'])
-    plt.colorbar(label=config['variable']+' Lat '+str(lat), shrink=0.5, orientation='horizontal')
-    ax.set_ylim(-config['max depth'], 0)
-    dirname = os.path.join(config['comout'], config['variable'])
-    os.makedirs(dirname, exist_ok=True)
-    figname = os.path.join(dirname, config['variable'] +
-                           'zonal_lat_'+str(int(lat)) + '_' + str(int(config['max depth'])) + 'm')
-    plt.savefig(figname, bbox_inches='tight', dpi=600)
-
-
 comout = os.getenv('COM_OCEAN_ANALYSIS')
 com_ice_history = os.getenv('COM_ICE_HISTORY_PREV')
 com_ocean_history = os.getenv('COM_OCEAN_HISTORY_PREV')

test/atm/global-workflow/jjob_ens_final.sh

@@ -45,8 +45,10 @@ elif [ $machine = 'ORION' ]; then
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' -o $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
+if [ $machine = 'HERA' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
+elif [ $machine = 'ORION' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
 fi

test/atm/global-workflow/jjob_ens_init.sh

@@ -104,8 +104,10 @@ for imem in $(seq 1 $NMEM_ENS); do
 done
 
 # Execute j-job
-if [ $machine = 'HERA' -o $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
+if [ $machine = 'HERA' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
+elif [ $machine = 'ORION' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
 fi

test/atm/global-workflow/jjob_ens_run.sh

@@ -48,8 +48,10 @@ elif [ $machine = 'ORION' ]; then
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' -o $machine = 'ORION' ]; then
-    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
+if [ $machine = 'HERA' ]; then
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
+elif [ $machine = 'ORION' ]; then
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --partition=orion --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
 fi

test/atm/global-workflow/jjob_var_final.sh

@@ -45,8 +45,10 @@ elif [ $machine = 'ORION' ]; then
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' -o $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
+if [ $machine = 'HERA' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
+elif [ $machine = 'ORION' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE
 fi

test/atm/global-workflow/jjob_var_init.sh

@@ -101,8 +101,10 @@ done
 
 
 # Execute j-job
-if [ $machine = 'HERA' -o $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
+if [ $machine = 'HERA' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
+elif [ $machine = 'ORION' ]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE
 fi

test/atm/global-workflow/jjob_var_run.sh

@@ -48,8 +48,10 @@ elif [ $machine = 'ORION' ]; then
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' -o $machine = 'ORION' ]; then
-    sbatch --ntasks=6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
+if [ $machine = 'HERA' ]; then
+    sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
+elif [ $machine = 'ORION' ]; then
+    sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN
 fi

test/atm/global-workflow/run_jedi_exe_3denvar.sh

@@ -26,7 +26,7 @@ if [ "$machine" = "hera" ] ; then
     partition="hera"
     gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix"
 elif [ "$machine" = "orion" ]; then
-    partition="debug"
+    partition="orion"
     gdasfix="/work2/noaa/da/cmartin/GDASApp/fix"
 fi
 
@@ -75,7 +75,7 @@ config:
 job options:
   machine: ${machine}
   account: da-cpu
-  queue: debug
+  queue: batch
   partition: ${partition}
   walltime: '30:00'
   ntasks: 6

test/atm/global-workflow/run_jedi_exe_3dhofx.sh

@@ -26,7 +26,7 @@ if [ "$machine" = "hera" ] ; then
     partition="hera"
     gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix"
 elif [ "$machine" = "orion" ]; then
-    partition="debug"
+    partition="orion"
     gdasfix="/work2/noaa/da/cmartin/GDASApp/fix"
 fi
 
@@ -64,7 +64,7 @@ config:
 job options:
   machine: ${machine}
   account: da-cpu
-  queue: debug
+  queue: batch
   partition: ${partition}
   walltime: '30:00'
   ntasks: 36

test/atm/global-workflow/run_jedi_exe_3dvar.sh

@@ -26,7 +26,7 @@ if [ "$machine" = "hera" ] ; then
     partition="hera"
     gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix"
 elif [ "$machine" = "orion" ]; then
-    partition="debug"
+    partition="orion"
     gdasfix="/work2/noaa/da/cmartin/GDASApp/fix"
 fi
 
@@ -74,7 +74,7 @@ config:
 job options:
   machine: ${machine}
   account: da-cpu
-  queue: debug
+  queue: batch
   partition: ${partition}
   walltime: '30:00'
   ntasks: 6

test/atm/global-workflow/run_jedi_exe_letkf.sh

@@ -26,7 +26,7 @@ if [ "$machine" = "hera" ] ; then
     partition="hera"
     gdasfix="/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix"
 elif [ "$machine" = "orion" ]; then
-    partition="debug"
+    partition="orion"
     gdasfix="/work2/noaa/da/cmartin/GDASApp/fix"
 fi
 
@@ -73,7 +73,7 @@ config:
 job options:
   machine: ${machine}
   account: da-cpu
-  queue: debug
+  queue: batch
   partition: ${partition}
   walltime: '30:00'
   ntasks: 36

test/soca/gw/CMakeLists.txt

@@ -14,7 +14,7 @@ add_test(NAME test_gdasapp_soca_prep
 set(MACHINE "container")
 IF (IS_DIRECTORY /work2/noaa/da)
   set(MACHINE "orion")
-  set(PARTITION "debug")
+  set(PARTITION "orion")
 ENDIF()
 IF (IS_DIRECTORY /scratch2/NCEPDEV/)
   set(MACHINE "hera")
@@ -33,6 +33,8 @@ add_test(NAME test_gdasapp_soca_concatioda
 add_test(NAME test_gdasapp_soca_run_clean
   COMMAND  ${CMAKE_COMMAND} -E remove_directory ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/testjjobs)
 
+# Create scratch for testing apps
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch)
 
 # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_*
 set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
@@ -46,6 +48,14 @@ set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
 set(setup "")
 foreach(jjob ${jjob_list})
 
+  # Make a copy of the scratch before it is removed by post
+  IF (jjob STREQUAL "JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT")
+    add_test(NAME test_gdasapp_soca_copy_scratch
+         COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/copy_scratch.sh ${PROJECT_BINARY_DIR}
+         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw)
+  ENDIF()
+
+
   # substitute a few variables in the test yaml
   message("ctest for ${jjob}")
   set(HOMEgfs ${PROJECT_SOURCE_DIR}/../..)
@@ -66,3 +76,23 @@ foreach(jjob ${jjob_list})
   set(setup "--skip") # Only run the setup of the first test, if not, it will hang
                       # waiting for standard input from setup_expt.py
 endforeach()
+
+# Test gdas/oops applications
+set(ctest_list "socahybridweights" "socaincr2mom6")
+foreach(ctest ${ctest_list})
+  set(TEST ${ctest})
+  set(EXEC ${PROJECT_BINARY_DIR}/bin/gdas_${ctest}.x)
+  set(YAML ${PROJECT_SOURCE_DIR}/test/soca/testinput/${ctest}.yaml)
+  configure_file(${PROJECT_SOURCE_DIR}/test/soca/gw/run_gdas_apps.yaml.test
+                 ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/run_gdas_apps_${ctest}.yaml)
+  set(test_name "test_gdasapp_soca_${ctest}")
+  add_test(NAME ${test_name}
+      COMMAND ${PROJECT_SOURCE_DIR}/ush/soca/run_jjobs.py
+           -y ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/run_gdas_apps_${ctest}.yaml
+           --skip --ctest True
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch)
+
+  set_tests_properties(${test_name}
+      PROPERTIES
+      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+endforeach()

test/soca/gw/copy_scratch.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+project_binary_dir=$1
+
+cp -r ${project_binary_dir}/test/soca/gw/testrun/testjjobs/RUNDIRS/gdas_test/gdasocnanal_12/* ${project_binary_dir}/test/soca/gw/apps_scratch