Update FV3-JEDI ctest case to match MPAS-JEDI case

build.sh

@@ -144,6 +144,7 @@ if [[ $DYCORE == 'MPAS' || $DYCORE == 'FV3andMPAS' ]]; then
   # Link in case data
   echo "Linking in test data for MPAS-JEDI case"
   $dir_root/rrfs-test/scripts/link_mpasjedi_expr.sh
+  $dir_root/rrfs-test/scripts/link_fv3jedi_expr.sh
 fi
 
 CMAKE_OPTS+=" -DMPIEXEC_MAX_NUMPROCS:STRING=120 -DBUILD_SUPER_EXE=$BUILD_SUPER_EXE"

bundle/rrfs-test-data/rrfs-data_fv3jedi_2024052700

@@ -0,0 +1 @@
+../../expr/fv3_2024052700

fix/expr_data/fv3_2024052700/Data

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/Data

fix/expr_data/fv3_2024052700/DataFix

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/DataFix

fix/expr_data/fv3_2024052700/Data_static

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/Data_static

fix/expr_data/fv3_2024052700/INPUT

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/INPUT

rrfs-test/CMakeLists.txt

@@ -8,8 +8,9 @@ include(funcs.cmake)
 
 # FV3-JEDI tests
 set(rrfs_fv3jedi_tests)
-add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_hyb_2022052619"     "fv3jedi_var.x")
-add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_letkf_2022052619"   "fv3jedi_letkf.x")
+add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_2024052700_Ens3Dvar"         "fv3jedi_var.x")
+add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_2024052700_getkf_observer"   "fv3jedi_letkf.x")
+add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_2024052700_getkf_solver"     "fv3jedi_letkf.x")
 
 # MPAS-JEDI tests 
 set(rrfs_mpasjedi_tests)
@@ -30,17 +31,18 @@ set (src_yaml "${CMAKE_SOURCE_DIR}/rrfs-test/testinput")
 if(FV3_DYCORE)
 
       message(STATUS "MACHINE_ID is " ${MACHINE_ID})
-      if("${MACHINE_ID}" STREQUAL "orion")
+      set(RESTORE_MPI_ARGS ${MPI_ARGS})
+      if("${MACHINE_ID}" STREQUAL "orion" OR "${MACHINE_ID}" STREQUAL "hercules")
          message(STATUS "Because MACHINE_ID is orion, adding exclusive MPI option" )
-         set(RESTORE_MPI_ARGS ${MPI_ARGS})
          set(MPI_ARGS "${MPI_ARGS} --exclusive")
       endif()   
+      set(MPI_ARGS "${MPI_ARGS} --time=00:30:00")
 
       get_all_keys(rrfs_fv3jedi_tests caselist)
       foreach(case ${caselist})
          get_from_dictionary(rrfs_fv3jedi_tests case exe)
          set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}")
-         set(src_casedir "${rrfs-test_data_local}/rrfs-data_fv3jedi_2022052619")
+         set(src_casedir "${rrfs-test_data_local}/rrfs-data_fv3jedi_2024052700")
          if (NOT EXISTS "${casedir}")
            file(MAKE_DIRECTORY ${casedir})
          endif()
@@ -51,25 +53,31 @@ if(FV3_DYCORE)
          file(CREATE_LINK ${CMAKE_SOURCE_DIR}/rrfs-test/testoutput ${casedir}/testoutput SYMBOLIC)
          file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir} )
          set(target_test ${case})
-         ecbuild_add_test( TARGET ${target_test}
-                        MPI      80
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${target_test}
+	 if ("${target_test}" STREQUAL "rrfs_fv3jedi_2024052700_getkf_solver")
+           ecbuild_add_test( TARGET ${target_test}
+                        MPI      160
+			WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${target_test}
                         ARGS     ${target_test}.yaml
+			TEST_DEPENDS "rrfs_fv3jedi_2024052700_getkf_observer"
                         COMMAND  ${exe} )
+	 else()
+           ecbuild_add_test( TARGET ${target_test}
+                        MPI      160
+			WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${target_test}
+                        ARGS     ${target_test}.yaml
+                        COMMAND  ${exe} )
+	 endif()
       endforeach()
-
-      if("${MACHINE_ID}" STREQUAL "orion")
-          set(MPI_ARGS ${RESTORE_MPI_ARGS})
-      endif()
+      set(MPI_ARGS ${RESTORE_MPI_ARGS})
 
 endif()
 
 if(MPAS_DYCORE)
 
       message(STATUS "MACHINE_ID is " ${MACHINE_ID})
-      if("${MACHINE_ID}" STREQUAL "orion")
+      set(RESTORE_MPI_ARGS ${MPI_ARGS})
+      if("${MACHINE_ID}" STREQUAL "orion" OR "${MACHINE_ID}" STREQUAL "hercules")
          message(STATUS "Because MACHINE_ID is orion, adding exclusive MPI option" )
-         set(RESTORE_MPI_ARGS ${MPI_ARGS})
          set(MPI_ARGS "${MPI_ARGS} --exclusive")
       endif()   
       set(MPI_ARGS "${MPI_ARGS} --ntasks-per-node=4")
@@ -111,10 +119,7 @@ if(MPAS_DYCORE)
                         COMMAND  ${exe} )
 	 endif()
       endforeach()
-
-      if("${MACHINE_ID}" STREQUAL "orion")
-          set(MPI_ARGS ${RESTORE_MPI_ARGS})
-      endif()
+      set(MPI_ARGS ${RESTORE_MPI_ARGS})
 
       ########################
       ### Bufr2ioda ctests ###

rrfs-test/IODA/offline_domain_check.py

@@ -187,6 +187,8 @@ def shrink_boundary(points, centroid, factor=0.01):
 if 'grid_lat' in grid_ds.variables and 'grid_lon' in grid_ds.variables:  # FV3 grid
     grid_lat = grid_ds.variables['grid_lat'][:, :]
     grid_lon = grid_ds.variables['grid_lon'][:, :]
+    grid_lat = grid_lat.flatten()
+    grid_lon = grid_lon.flatten()
     dycore = "FV3"
 elif 'latCell' in grid_ds.variables and 'lonCell' in grid_ds.variables:  # MPAS grid
     grid_lat = np.degrees(grid_ds.variables['latCell'][:])  # Convert radians to degrees

rrfs-test/scripts/link_fv3jedi_expr.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+RDASApp=$( git rev-parse --show-toplevel 2>/dev/null )
+if [[ -z ${RDASApp} ]]; then
+  echo "Not under a clone of RDASApp!"
+  echo "Please delete line 2-7 and set RDASApp variable mannually"
+  exit
+fi
+
+#RDASApp="/path/to/RDASApp"  # set this variable if line2-7 was removed
+BUMPLOC="conus12km-401km11levels"
+exprname="fv3_2024052700"
+expdir=${RDASApp}/expr/${exprname}  # can be set to any directory 
+mkdir -p ${expdir}
+cd ${expdir}
+echo "expdir is at: ${expdir}"
+
+${RDASApp}/ush/init.sh
+cp -r ${RDASApp}/rrfs-test/testoutput ./testoutput
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_Ens3Dvar.yaml .
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_getkf_observer.yaml . 
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_getkf_solver.yaml . 
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_bumploc.yaml ./bumploc.yaml
+sed -e "s#@RDASApp@#${RDASApp}#" ${RDASApp}/rrfs-test/scripts/templates/fv3jedi_expr/run_bump.sh > run_bump.sh
+sed -e "s#@RDASApp@#${RDASApp}#" ${RDASApp}/rrfs-test/scripts/templates/fv3jedi_expr/run_jedi.sh > run_jedi.sh
+cp ${RDASApp}/rrfs-test/ush/colormap.py . 
+cp ${RDASApp}/rrfs-test/ush/fv3jedi_increment_singleob.py .
+cp ${RDASApp}/rrfs-test/ush/fv3jedi_increment_fulldom.py . 
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/Data Data
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/DataFix DataFix
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/Data_static Data_static
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/INPUT INPUT

rrfs-test/scripts/setup_experiment.sh

@@ -154,6 +154,7 @@ if [[ $GSI_TEST_DATA == "YES" ]]; then
   sed -i "s#@YOUR_PATH_TO_GSI@#${YOUR_PATH_TO_GSI}#g" ./run_gsi.sh
   sed -i "s#@SLURM_ACCOUNT@#${SLURM_ACCOUNT}#g"       ./run_gsi.sh
   sed -i "s#@MACHINE_ID@#${MACHINE_ID}#g"             ./run_gsi.sh
+  sed -i "s#ANAL_TIME@#2022052619#g"                  ./run_gsi.sh
   cp -p $YOUR_PATH_TO_RDASAPP/rrfs-test/scripts/templates/run_gsi_ncdiag_template.sh run_gsi_ncdiag.sh
   sed -i "s#@YOUR_PATH_TO_RDASAPP@#${YOUR_PATH_TO_RDASAPP}#g" ./run_gsi_ncdiag.sh
   sed -i "s#@MACHINE_ID@#${MACHINE_ID}#g"                     ./run_gsi_ncdiag.sh

rrfs-test/scripts/setup_gsi_parallel.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# This script is used to set up a GSI experiment using the FV3-LAM test data in RDASApp
+# This is useful for running parallel GSI experiments against FV3-JEDI 
+# GSI fix files and namelists follow the configurations from RRFSv1
+# There are two options for YOUR_GSI_CASE right now:
+#	2024052700 (matches MPAS-JEDI case)
+#	2022052619
+
+###### USER INPUT #####
+SLURM_ACCOUNT="fv3-cam"
+#YOUR_EXPERIMENT_DIR="/path/to/your/desired/experiment/directory/jedi-assim_test"
+YOUR_EXPERIMENT_DIR="/scratch1/BMC/zrtrr/Samuel.Degelia/GSI_parallel"
+#YOUR_PATH_TO_GSI="/path/to/your/installation/of/GSI"
+YOUR_PATH_TO_GSI="/scratch1/BMC/zrtrr/Samuel.Degelia/GSI"
+YOUR_GSI_CASE="2024052700"
+#######################
+
+START=$(date +%s)
+RDASApp=$( git rev-parse --show-toplevel 2>/dev/null )
+source $RDASApp/ush/detect_machine.sh
+
+# At the moment these are the only test data that exists.
+if [[ $YOUR_GSI_CASE == "2022052619" ]]; then
+  TEST_DATA="rrfs-data_fv3jedi_2022052619"
+elif [[ $YOUR_GSI_CASE == "2024052700" ]]; then
+  TEST_DATA="rrfs-data_fv3jedi_2024052700"
+else
+  echo "Not a valid GSI case: ${YOUR_GSI_CASE}."
+  echo "exiting!!!"
+  exit 2
+fi
+
+# Print current settings to the screen.
+echo "Your current settings are:"
+echo -e "\tYOUR_EXPERIMENT_DIR=$YOUR_EXPERIMENT_DIR"
+echo -e "\tSLURM_ACCOUNT=$SLURM_ACCOUNT"
+echo -e "\tYOUR_PATH_TO_GSI=$YOUR_PATH_TO_GSI"
+
+# Get current machine so we can find GSI fix data 
+case ${MACHINE_ID} in
+  hera)
+    RDAS_DATA=/scratch1/NCEPDEV/fv3-cam/RDAS_DATA
+    ;;
+  jet)
+    RDAS_DATA=/lfs4/BMC/nrtrr/RDAS_DATA
+    ;;
+  orion|hercules)
+    RDAS_DATA=/work/noaa/rtrr/RDAS_DATA
+    ;;
+  *)
+    echo "platform not supported: ${MACHINE_ID}"
+    exit 3
+    ;;
+esac
+
+# Ensure experiment directory exists, then move into that space.
+mkdir -p $YOUR_EXPERIMENT_DIR
+cd $YOUR_EXPERIMENT_DIR
+
+# Copy GSI data
+rsync -a ${RDAS_DATA}/gsi_${YOUR_GSI_CASE}/* .
+
+# Copy run scripts and plotting scripts
+cp -p $RDASApp/rrfs-test/scripts/templates/run_gsi_template.sh run_gsi.sh
+sed -i "s#@YOUR_PATH_TO_GSI@#${YOUR_PATH_TO_GSI}#g" ./run_gsi.sh
+sed -i "s#@SLURM_ACCOUNT@#${SLURM_ACCOUNT}#g"       ./run_gsi.sh
+sed -i "s#@MACHINE_ID@#${MACHINE_ID}#g"             ./run_gsi.sh
+sed -i "s#@ANAL_TIME@#${YOUR_GSI_CASE}#g"           ./run_gsi.sh
+cp -p $RDASApp/rrfs-test/ush/fv3gsi_increment_fulldom.py .
+
+echo "done."
+END=$(date +%s)
+DIFF=$((END - START))
+echo "Time taken to run the code: $DIFF seconds"

rrfs-test/scripts/templates/fv3jedi_expr/run_bump.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+#SBATCH --account=rtrr
+#SBATCH --qos=batch # use the normal queue on Gaea
+###SBATCH -M c6 # for Gaea
+###SBATCH --partition=bigmem
+###SBATCH --partition=kjet
+###SBATCH --reservation=rrfsens
+#SBATCH --ntasks=160
+#SBATCH -t 00:58:00
+#SBATCH --job-name=fv3jedi_bump
+#SBATCH -o log.bump
+#SBATCH --open-mode=truncate
+#SBATCH --cpus-per-task 4 --exclusive
+
+RDASApp=@RDASApp@
+
+if [[ -s /apps/lmod/lmod/init/sh ]]; then
+  . /apps/lmod/lmod/init/sh
+fi
+
+module purge
+source ${RDASApp}/ush/detect_machine.sh 
+module use ${RDASApp}/modulefiles
+module load RDAS/${MACHINE_ID}.intel
+module list
+
+export OOPS_TRACE=1
+export OMP_NUM_THREADS=1
+
+ulimit -s unlimited
+ulimit -v unlimited
+ulimit -a
+
+srun -l -n 160 ${RDASApp}/build/bin/fv3jedi_error_covariance_toolbox.x ./bumploc.yaml

rrfs-test/scripts/templates/fv3jedi_expr/run_jedi.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+#SBATCH --account=rtrr
+#SBATCH --qos=batch # use the normal queue on Gaea
+###SBATCH -M c6 # for Gaea
+###SBATCH --partition=kjet
+###SBATCH --reservation=rrfsens
+#SBATCH --ntasks=160
+#SBATCH -t 00:58:00
+#SBATCH --job-name=fv3jedi_test
+#SBATCH -o log.jedi
+#SBATCH --open-mode=truncate
+#SBATCH --cpus-per-task 4 --exclusive
+
+RDASApp=@RDASApp@
+
+inputfile=./rrfs_fv3jedi_2024052700_Ens3Dvar.yaml # FOR EnVar
+#inputfile=./rrfs_fv3jedi_2024052700_getkf_observer.yaml # FOR GETKF
+#inputfile=./rrfs_fv3jedi_2024052700_getkf_solver.yaml # FOR GETKF
+
+if [[ -s /apps/lmod/lmod/init/sh ]]; then
+  . /apps/lmod/lmod/init/sh
+fi
+
+module purge
+source ${RDASApp}/ush/detect_machine.sh 
+module use ${RDASApp}/modulefiles
+module load RDAS/${MACHINE_ID}.intel
+module list
+
+export OOPS_TRACE=1
+export OMP_NUM_THREADS=1
+
+ulimit -s unlimited
+ulimit -v unlimited
+ulimit -a
+
+srun -l -n 160 ${RDASApp}/build/bin/fv3jedi_var.x    ./$inputfile    log.out # FOR HYB/ENVAR
+#srun -l -n 160 ${RDASApp}/build/bin/fv3jedi_letkf.x    ./$inputfile    log.out # FOR GETKF

rrfs-test/scripts/templates/run_gsi_template.sh

@@ -30,7 +30,7 @@ cp -p Data/bkg/* .
 
 srun --label @YOUR_PATH_TO_GSI@/build/src/gsi/gsi.x
 
-ANAL_TIME=2022052619
+ANAL_TIME=@ANAL_TIME@
 # Loop over first and last outer loops to generate innovation
 # diagnostic files for indicated observation types (groups)
 #