NOAA-EMC · guillaumevernieres · Jun 12, 2024 · Jun 12, 2024 · Jun 12, 2024 · Jun 13, 2024
diff --git a/.gitignore b/.gitignore
@@ -35,8 +35,10 @@ fix/gsi
 fix/lut
 fix/mom6
 fix/orog
+fix/orog_nest
 fix/sfc_climo
 fix/ugwd
+fix/ugwd_nest
 fix/verif
 fix/wave
 
@@ -164,6 +166,14 @@ scripts/exemcsfc_global_sfc_prep.sh
 scripts/exgdas_global_marine_analysis_ecen.py
 scripts/exglobal_prep_ocean_obs.py
 # ush symlinks
+ush/bufr2ioda_insitu_profile_argo.py
+ush/bufr2ioda_insitu_profile_bathy.py
+ush/bufr2ioda_insitu_profile_glider.py
+ush/bufr2ioda_insitu_profile_marinemammal.py
+ush/bufr2ioda_insitu_profile_tesac.py
+ush/bufr2ioda_insitu_profile_xbtctd.py
+ush/bufr2ioda_insitu_surface_altkob.py
+ush/bufr2ioda_insitu_surface_trkob.py
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
@@ -174,6 +184,8 @@ ush/fv3gfs_make_grid.sh
 ush/fv3gfs_make_orog.sh
 ush/gen_bufr2ioda_json.py
 ush/gen_bufr2ioda_yaml.py
+ush/bufr2ioda_insitu_profile*.py
+ush/bufr2ioda_insitu_surface*.py
 ush/global_chgres.sh
 ush/global_chgres_driver.sh
 ush/global_cycle.sh

diff --git a/.gitmodules b/.gitmodules
@@ -26,11 +26,3 @@
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
   url = https://github.com/NOAA-EMC/GSI-Monitor.git
-[submodule "sorc/upp.fd"]
-  path = sorc/upp.fd
-  url = https://github.com/NOAA-EMC/UPP.git
-  ignore = dirty
-[submodule "sorc/jcb"]
-  path = sorc/jcb
-  url = https://github.com/noaa-emc/jcb
-  fetchRecurseSubmodules = false
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -76,7 +76,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
                         sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)

diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml
@@ -10,3 +10,4 @@ base:
   DO_NPOESS: "YES"
   DO_GENESIS_FSU: "NO"
   FHMAX_GFS: 384
+  FHMAX_HF_GFS: 120
diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst
@@ -4,6 +4,46 @@ HPC Settings and Help
 
 Running the GFS configurations (or almost any global workflow configuration except the coarsest) is a resource intensive exercise. This page discusses recommended HPC environmental settings and contact information in case you need assistance from a particular HPC helpdesk. While most of the documentation is based on supported NOAA platforms, the learnings here can hopefully apply to other platforms. 
 
+====================================
+Minimum system software requirements
+====================================
+
+The following system software requirements are the minimum for any new or existing system and reflect the development and testing environment on which the global workflow is maintained.  Any system that does not meet these requirements will not be supported.
+
++--------------+-------------+---------------------------------------+
+| Software     | Minimum     | Notes                                 |
+|              | supported   |                                       |
+|              | version(s)  |                                       |
++==============+=============+=======================================+
+| Bash         | 4.4.20      |                                       |
++--------------+-------------+---------------------------------------+
+| Python       | * 3.8.6     | * 3.10.x is not supported by METplus  |
+|              | * 3.10.13+  |   verification software               |
+|              | * 3.11.6+   | * 3.11.6 is packaged with spack-stack |
+|              |             | * 3.9.x is untested                   |
++--------------+-------------+---------------------------------------+
+| Spack-Stack  | 1.6.0       | * Available everywhere but WCOSS2     |
++--------------+-------------+---------------------------------------+
+| lmod         | 8.3.1       |                                       |
++--------------+-------------+---------------------------------------+
+| Slurm        | 23.02.7     | * Other schedulers may be supportable |
++--------------+-------------+---------------------------------------+
+| PBSpro       | 2022.1.1    | * Other schedulers may be supportable |
++--------------+-------------+---------------------------------------+
+| Git          | 2.29.0      | * Some components e.g. GDASApp may    |
+|              |             |   need Git-LFS for downloading test   |
+|              |             |   data                                |
++--------------+-------------+---------------------------------------+
+| Rocoto       | 1.3.5       | * 1.3.7 is required for newer         |
+|              |             |   versions of Ruby (3.2+)             |
++--------------+-------------+---------------------------------------+
+| Intel        | 2021.5.1    | * GNU compilers are not supported     |
+| Compilers    |             | * Intel LLVM compilers are not yet    |
+|              |             |   supported                           |
+|              |             | * Intel 19.x is only supported on     |
+|              |             |   WCOSS2                              |
++--------------+-------------+---------------------------------------+
+
 ================================
 Experiment troubleshooting help
 ================================
@@ -50,61 +90,6 @@ Optimizing the global workflow on S4
 
 The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui ([email protected]) if you are planning on running a cycled experiment on this system to obtain optimized configuration files.
 
-============
-Git settings
-============
-
-^^^^^^
-Merges
-^^^^^^
-
-Use the following command to have merge commits include the one-line description of all the commits being merged (up to 200). You only need to do this once on each machine; it will be saved to your git settings::
-
-   git config --global merge.log 200
-
-Use the ``--no-ff`` option to make sure there is always a merge commit when a fast-forward only is available. Exception: If the merge contains only a single commit, it can be applied as a fast-forward.
-
-For any merge with multiple commits, a short synopsis of the merge should appear between the title and the list of commit titles added by merge.log.
-
-^^^^^^^
-Version
-^^^^^^^
-
-It is advised to use Git v2+ when available. At the time of writing this documentation the default Git clients on the different machines were as noted in the table below. It is recommended that you check the default modules before loading recommended ones:
-
-+----------+----------+---------------------------------------+
-| Machine  | Default  | Recommended                           |
-+----------+----------+---------------------------------------+
-| Hera     | v2.18.0  | default                               |
-+----------+----------+---------------------------------------+
-| Hercules | v2.31.1  | default                               |
-+----------+----------+---------------------------------------+
-| Orion    | v1.8.3.1 | **module load git/2.28.0**            |
-+----------+----------+---------------------------------------+
-| Jet      | v2.18.0  | default                               |
-+----------+----------+---------------------------------------+
-| WCOSS2   | v2.35.3  | default                               |
-+----------+----------+---------------------------------------+
-| S4       | v1.8.3.1 | **module load git/2.30.0**            |
-+----------+----------+---------------------------------------+
-| AWS PW   | v1.8.3.1 | default                               |
-+----------+----------+---------------------------------------+
-
-^^^^^^^^^^^^^
-Output format
-^^^^^^^^^^^^^
-
-For proper display of Git command output (e.g. git branch and git diff) type the following once per machine:
-
-::
-
-   git config --global core.pager 'less -FRX'
-
-For the manage_externals utility functioning::
-
-   Error: fatal: ssh variant 'simple' does not support setting port
-   Fix: git config --global ssh.variant ssh
-
 ========================================
 Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4)
 ========================================

diff --git a/env/HERA.env b/env/HERA.env
@@ -102,6 +102,13 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))
@@ -316,7 +323,6 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
 
-
 elif [[ "${step}" = "fit2obs" ]]; then
 
     nth_max=$((npe_node_max / npe_node_fit2obs))

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -99,6 +99,12 @@ case ${step} in
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
  ;;
+ "prepobsaero")
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+;;
  "snowanl")
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/JET.env b/env/JET.env
@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/ORION.env b/env/ORION.env
@@ -90,6 +90,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/S4.env b/env/S4.env
@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepobsaero))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -76,6 +76,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1}
+    export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --ppn ${npe_node_prepobsaero}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE
@@ -11,8 +11,8 @@ export CDUMP=${RUN/enkf}
 
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP
 MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
-  COM_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
-  COM_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL
+  COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
+  COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL
 
 ###############################################################
 # Run archive script

diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE
@@ -9,21 +9,45 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch"
 ##############################################
 export CDUMP=${RUN/enkf}
 
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMOS_GEMPAK \
-  COM_ATMOS_GENESIS COM_ATMOS_HISTORY COM_ATMOS_INPUT COM_ATMOS_MASTER COM_ATMOS_RESTART \
-  COM_ATMOS_TRACK COM_ATMOS_WMO \
-  COM_CHEM_HISTORY COM_CHEM_ANALYSIS\
-  COM_MED_RESTART \
-  COM_SNOW_ANALYSIS \
-  COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART COM_ICE_GRIB \
-  COM_OBS COM_TOP \
-  COM_OCEAN_HISTORY COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \
-  COM_OCEAN_ANALYSIS \
-  COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION COM_WAVE_RESTART \
-  COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+	COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
+	COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \
+	COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \
+	COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+	COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \
+	COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \
+	COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \
+	COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \
+	COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \
+	COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \
+	COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+	COMIN_MED_RESTART:COM_MED_RESTART_TMPL \
+	COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
+	COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \
+	COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \
+	COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
+	COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \
+	COMIN_OBS:COM_OBS_TMPL \
+	COMIN_TOP:COM_TOP_TMPL \
+	COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \
+	COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \
+	COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \
+	COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \
+	COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+	COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \
+	COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \
+	COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \
+	COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \
+	COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \
+	COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \
+	COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \
+	COMIN_CONF:COM_CONF_TMPL \
+	COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
 
 for grid in "0p25" "0p50" "1p00"; do
-  YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
+    YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \
+       "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
 done
 
 ###############################################################

diff --git a/jobs/JGLOBAL_PREP_OBS_AERO b/jobs/JGLOBAL_PREP_OBS_AERO
@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prepobsaero" -c "base prepobsaero"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export COMIN_OBS="${DATA}"
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_OBS:COM_OBS_TMPL
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_obs_aero.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0