Merge branch 'NOAA-EMC:develop' into CADS

ci/Jenkinsfile

@@ -19,6 +19,7 @@ pipeline {
     stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
              // which is used to designate the Nodes in the Jenkins Controler by the agent label
              // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+             // no op 2
 
         stage('Get Machine') {
             agent { label 'built-in' }
@@ -111,7 +112,7 @@ pipeline {
                                                     try {
                                                         sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
                                                         gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
-                                                        sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+                                                        sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
                                                     } catch (Exception error_comment) {
                                                         echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                                     }
@@ -215,6 +216,7 @@ pipeline {
                                         STATUS = 'Failed'
                                         try {
                                             sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine}\nin\\`${HOME}/RUNTESTS/${pslot}\\`" """)
                                         } catch (Exception e) {
                                             echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
                                         }

ci/scripts/utils/rocotostat.py

@@ -95,7 +95,11 @@ def rocoto_statcount():
         error_return = rocoto_status['UNKNOWN']
         rocoto_state = 'UNKNOWN'
     elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0:
-        error_return = -3
+        #
+        #  TODO for now a STALLED state will be just a warning as it can
+        #  produce a false negative if there is a timestamp on a file dependency.
+        #
+        #   error_return = -3
         rocoto_state = 'STALLED'
     else:
         rocoto_state = 'RUNNING'

env/AWSPW.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmanlfv3inc atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"

env/CONTAINER.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmanlfv3inc atmensanlrun aeroanlrun snowanl"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"

env/HERA.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -56,13 +56,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
-elif [[ "${step}" = "atmanlrun" ]]; then
+elif [[ "${step}" = "atmanlvar" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_atmanlrun))
+    nth_max=$((npe_node_max / npe_node_atmanlvar))
 
-    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
-    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
-    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}"
+    export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}}
+    [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max}
+    export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}"
 
 elif [[ "${step}" = "atmensanlrun" ]]; then
 
@@ -82,6 +82,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "atmanlfv3inc" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmanlfv3inc))
+
+    export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}}
+    [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
+    export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/HERCULES.env

@@ -53,13 +53,21 @@ case ${step} in
     export wave_mpmd=${mpmd_opt}
 
  ;;
- "atmanlrun")
+ "atmanlvar")
 
-    nth_max=$((npe_node_max / npe_node_atmanlrun))
+    nth_max=$((npe_node_max / npe_node_atmanlvar))
 
-    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
-    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
-    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}"
+    export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}}
+    [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max}
+    export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}"
+ ;;
+ "atmanlfv3inc")
+
+    nth_max=$((npe_node_max / npe_node_atmanlfv3inc))
+
+    export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}}
+    [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
+    export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
  ;;
  "atmensanlrun")
 

env/JET.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -44,13 +44,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
-elif [[ "${step}" = "atmanlrun" ]]; then
+elif [[ "${step}" = "atmanlvar" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_atmanlrun))
+    nth_max=$((npe_node_max / npe_node_atmanlvar))
 
-    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
-    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
-    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun}"
+    export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}}
+    [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max}
+    export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}"
 
 elif [[ "${step}" = "atmensanlrun" ]]; then
 
@@ -80,6 +80,14 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "atmanlfv3inc" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmanlfv3inc))
+
+    export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}}
+    [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
+    export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}"
+
 elif [[ "${step}" = "ocnanalbmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

env/ORION.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -52,13 +52,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
-elif [[ "${step}" = "atmanlrun" ]]; then
+elif [[ "${step}" = "atmanlvar" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_atmanlrun))
+    nth_max=$((npe_node_max / npe_node_atmanlvar))
 
-    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
-    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
-    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}"
+    export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}}
+    [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max}
+    export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}"
 
 elif [[ "${step}" = "atmensanlrun" ]]; then
 
@@ -88,6 +88,14 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "atmanlfv3inc" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmanlfv3inc))
+
+    export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}}
+    [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
+    export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
+
 elif [[ "${step}" = "ocnanalbmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

env/S4.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -44,13 +44,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
-elif [[ "${step}" = "atmanlrun" ]]; then
+elif [[ "${step}" = "atmanlvar" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_atmanlrun))
+    nth_max=$((npe_node_max / npe_node_atmanlvar))
 
-    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
-    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
-    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun}"
+    export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}}
+    [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max}
+    export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}"
 
 elif [[ "${step}" = "atmensanlrun" ]]; then
 
@@ -80,6 +80,14 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "atmanlfv3inc" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmanlfv3inc))
+
+    export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}}
+    [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
+    export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}"
+
 elif [[ "${step}" = "ocnanalbmat" ]]; then
      echo "WARNING: ${step} is not enabled on S4!"
 

env/WCOSS2.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc"
     echo "anal sfcanl fcst post metp"
     echo "eobs eupd ecen esfc efcs epos"
     echo "postsnd awips gempak"
@@ -38,13 +38,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
     export wavempexec="${launcher} -np"
     export wave_mpmd=${mpmd_opt}
 
-elif [[ "${step}" = "atmanlrun" ]]; then
+elif [[ "${step}" = "atmanlvar" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_atmanlrun))
+    nth_max=$((npe_node_max / npe_node_atmanlvar))
 
-    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
-    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
-    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun}"
+    export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}}
+    [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max}
+    export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}"
 
 elif [[ "${step}" = "atmensanlrun" ]]; then
 
@@ -74,6 +74,14 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "atmanlfv3inc" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_atmanlfv3inc))
+
+    export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}}
+    [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
+    export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export OMP_PLACES=cores

jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT

@@ -0,0 +1,37 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+export WIPE_DATA="NO"
+export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}}
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlfv3inc" -c "base atmanl atmanlfv3inc"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_fv3_increment.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/JGLOBAL_ATM_ANALYSIS_RUN → jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL

@@ -3,7 +3,7 @@
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}}
-source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlrun" -c "base atmanl atmanlrun"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlvar" -c "base atmanl atmanlvar"
 
 ##############################################
 # Set variables used in the script
@@ -18,7 +18,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlrun" -c "base atmanl atmanlrun"
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_run.py}
+EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_variational.py}
 ${EXSCRIPT}
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"

jobs/rocoto/atmanlfv3inc.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="atmanlfv3inc"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT"
+status=$?
+exit "${status}"

jobs/rocoto/atmanlrun.sh → jobs/rocoto/atmanlvar.sh

@@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"
 
-export job="atmanlrun"
+export job="atmanlvar"
 export jobid="${job}.$$"
 
 ###############################################################
@@ -19,6 +19,6 @@ export PYTHONPATH
 
 ###############################################################
 # Execute the JJOB
-"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN"
+"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL"
 status=$?
 exit "${status}"