Update FV3-JEDI ctest case to match MPAS-JEDI case

build.sh

@@ -144,6 +144,7 @@ if [[ $DYCORE == 'MPAS' || $DYCORE == 'FV3andMPAS' ]]; then
   # Link in case data
   echo "Linking in test data for MPAS-JEDI case"
   $dir_root/rrfs-test/scripts/link_mpasjedi_expr.sh
+  $dir_root/rrfs-test/scripts/link_fv3jedi_expr.sh
 fi
 
 CMAKE_OPTS+=" -DMPIEXEC_MAX_NUMPROCS:STRING=120 -DBUILD_SUPER_EXE=$BUILD_SUPER_EXE"

bundle/rrfs-test-data/rrfs-data_fv3jedi_2024052700

@@ -0,0 +1 @@
+../../expr/fv3_2024052700

fix/expr_data/fv3_2024052700/Data

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/Data

fix/expr_data/fv3_2024052700/DataFix

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/DataFix

fix/expr_data/fv3_2024052700/Data_static

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/Data_static

fix/expr_data/fv3_2024052700/INPUT

@@ -0,0 +1 @@
+../../.agent/expr_data/fv3_2024052700/INPUT

rrfs-test/CMakeLists.txt

@@ -8,8 +8,8 @@ include(funcs.cmake)
 
 # FV3-JEDI tests
 set(rrfs_fv3jedi_tests)
-add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_hyb_2022052619"     "fv3jedi_var.x")
-add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_letkf_2022052619"   "fv3jedi_letkf.x")
+add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_2024052700_Ens3Dvar"         "fv3jedi_var.x")
+add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_2024052700_letkf"            "fv3jedi_letkf.x")
 
 # MPAS-JEDI tests 
 set(rrfs_mpasjedi_tests)
@@ -40,7 +40,7 @@ if(FV3_DYCORE)
       foreach(case ${caselist})
          get_from_dictionary(rrfs_fv3jedi_tests case exe)
          set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}")
-         set(src_casedir "${rrfs-test_data_local}/rrfs-data_fv3jedi_2022052619")
+         set(src_casedir "${rrfs-test_data_local}/rrfs-data_fv3jedi_2024052700")
          if (NOT EXISTS "${casedir}")
            file(MAKE_DIRECTORY ${casedir})
          endif()
@@ -52,7 +52,7 @@ if(FV3_DYCORE)
          file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir} )
          set(target_test ${case})
          ecbuild_add_test( TARGET ${target_test}
-                        MPI      80
+                        MPI      160
                         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${target_test}
                         ARGS     ${target_test}.yaml
                         COMMAND  ${exe} )

rrfs-test/scripts/link_fv3jedi_expr.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+RDASApp=$( git rev-parse --show-toplevel 2>/dev/null )
+if [[ -z ${RDASApp} ]]; then
+  echo "Not under a clone of RDASApp!"
+  echo "Please delete line 2-7 and set RDASApp variable mannually"
+  exit
+fi
+
+#RDASApp="/path/to/RDASApp"  # set this variable if line2-7 was removed
+BUMPLOC="conus12km-401km11levels"
+exprname="fv3_2024052700"
+expdir=${RDASApp}/expr/${exprname}  # can be set to any directory 
+mkdir -p ${expdir}
+cd ${expdir}
+echo "expdir is at: ${expdir}"
+
+${RDASApp}/ush/init.sh
+cp -r ${RDASApp}/rrfs-test/testoutput ./testoutput
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_hyb.yaml .
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_Ens3Dvar.yaml .
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_letkf.yaml . 
+cp ${RDASApp}/rrfs-test/testinput/rrfs_fv3jedi_2024052700_bumploc.yaml ./bumploc.yaml
+sed -e "s#@RDASApp@#${RDASApp}#" ${RDASApp}/rrfs-test/scripts/templates/fv3jedi_expr/run_bump.sh > run_bump.sh
+sed -e "s#@RDASApp@#${RDASApp}#" ${RDASApp}/rrfs-test/scripts/templates/fv3jedi_expr/run_jedi.sh > run_jedi.sh
+cp ${RDASApp}/rrfs-test/ush/colormap.py . 
+cp ${RDASApp}/rrfs-test/ush/fv3jedi_increment_singleob.py .
+cp ${RDASApp}/rrfs-test/ush/fv3jedi_increment_fulldom.py . 
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/Data Data
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/DataFix DataFix
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/Data_static Data_static
+ln -snf ${RDASApp}/fix/expr_data/${exprname}/INPUT INPUT
+

rrfs-test/scripts/setup_experiment.sh

@@ -154,6 +154,7 @@ if [[ $GSI_TEST_DATA == "YES" ]]; then
   sed -i "s#@YOUR_PATH_TO_GSI@#${YOUR_PATH_TO_GSI}#g" ./run_gsi.sh
   sed -i "s#@SLURM_ACCOUNT@#${SLURM_ACCOUNT}#g"       ./run_gsi.sh
   sed -i "s#@MACHINE_ID@#${MACHINE_ID}#g"             ./run_gsi.sh
+  sed -i "s#ANAL_TIME@#2022052619#g"                  ./run_gsi.sh
   cp -p $YOUR_PATH_TO_RDASAPP/rrfs-test/scripts/templates/run_gsi_ncdiag_template.sh run_gsi_ncdiag.sh
   sed -i "s#@YOUR_PATH_TO_RDASAPP@#${YOUR_PATH_TO_RDASAPP}#g" ./run_gsi_ncdiag.sh
   sed -i "s#@MACHINE_ID@#${MACHINE_ID}#g"                     ./run_gsi_ncdiag.sh

rrfs-test/scripts/setup_gsi_parallel.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# This script is used to set up a GSI experiment using the FV3-LAM test data in RDASApp
+# This is useful for running parallel GSI experiments against FV3-JEDI 
+# GSI fix files and namelists follow the configurations from RRFSv1
+# There are two options for YOUR_GSI_CASE right now:
+#	2024052700 (matches MPAS-JEDI case)
+#	2022052619
+
+###### USER INPUT #####
+SLURM_ACCOUNT="fv3-cam"
+#YOUR_EXPERIMENT_DIR="/path/to/your/desired/experiment/directory/jedi-assim_test"
+YOUR_EXPERIMENT_DIR="/scratch1/BMC/zrtrr/Samuel.Degelia/GSI_parallel"
+#YOUR_PATH_TO_GSI="/path/to/your/installation/of/GSI"
+YOUR_PATH_TO_GSI="/scratch1/BMC/zrtrr/Samuel.Degelia/GSI"
+YOUR_GSI_CASE="2024052700"
+#######################
+
+START=$(date +%s)
+RDASApp=$( git rev-parse --show-toplevel 2>/dev/null )
+source $RDASApp/ush/detect_machine.sh
+
+# At the moment these are the only test data that exists.
+if [[ $YOUR_GSI_CASE == "2022052619" ]]; then
+  TEST_DATA="rrfs-data_fv3jedi_2022052619"
+elif [[ $YOUR_GSI_CASE == "2024052700" ]]; then
+  TEST_DATA="rrfs-data_fv3jedi_2024052700"
+else
+  echo "Not a valid GSI case: ${YOUR_GSI_CASE}."
+  echo "exiting!!!"
+  exit 2
+fi
+
+# Print current settings to the screen.
+echo "Your current settings are:"
+echo -e "\tYOUR_EXPERIMENT_DIR=$YOUR_EXPERIMENT_DIR"
+echo -e "\tSLURM_ACCOUNT=$SLURM_ACCOUNT"
+echo -e "\tYOUR_PATH_TO_GSI=$YOUR_PATH_TO_GSI"
+
+# Get current machine so we can find GSI fix data 
+case ${MACHINE_ID} in
+  hera)
+    RDAS_DATA=/scratch1/NCEPDEV/fv3-cam/RDAS_DATA
+    ;;
+  jet)
+    RDAS_DATA=/lfs4/BMC/nrtrr/RDAS_DATA
+    ;;
+  orion|hercules)
+    RDAS_DATA=/work/noaa/rtrr/RDAS_DATA
+    ;;
+  *)
+    echo "platform not supported: ${MACHINE_ID}"
+    exit 3
+    ;;
+esac
+
+# Ensure experiment directory exists, then move into that space.
+mkdir -p $YOUR_EXPERIMENT_DIR
+cd $YOUR_EXPERIMENT_DIR
+
+# Copy GSI data
+rsync -a ${RDAS_DATA}/gsi_${YOUR_GSI_CASE}/* .
+
+# Copy run scripts and plotting scripts
+cp -p $RDASApp/rrfs-test/scripts/templates/run_gsi_template.sh run_gsi.sh
+sed -i "s#@YOUR_PATH_TO_GSI@#${YOUR_PATH_TO_GSI}#g" ./run_gsi.sh
+sed -i "s#@SLURM_ACCOUNT@#${SLURM_ACCOUNT}#g"       ./run_gsi.sh
+sed -i "s#@MACHINE_ID@#${MACHINE_ID}#g"             ./run_gsi.sh
+sed -i "s#@ANAL_TIME@#${YOUR_GSI_CASE}#g"           ./run_gsi.sh
+cp -p $RDASApp/rrfs-test/ush/fv3gsi_increment_fulldom.py .
+
+echo "done."
+END=$(date +%s)
+DIFF=$((END - START))
+echo "Time taken to run the code: $DIFF seconds"

rrfs-test/scripts/templates/fv3jedi_expr/run_bump.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+#SBATCH --account=rtrr
+#SBATCH --qos=batch # use the normal queue on Gaea
+###SBATCH -M c6 # for Gaea
+###SBATCH --partition=bigmem
+###SBATCH --partition=kjet
+###SBATCH --reservation=rrfsens
+#SBATCH --ntasks=160
+#SBATCH -t 00:58:00
+#SBATCH --job-name=fv3jedi_bump
+#SBATCH -o log.bump
+#SBATCH --open-mode=truncate
+#SBATCH --cpus-per-task 4 --exclusive
+
+RDASApp=@RDASApp@
+
+if [[ -s /apps/lmod/lmod/init/sh ]]; then
+  . /apps/lmod/lmod/init/sh
+fi
+
+module purge
+source ${RDASApp}/ush/detect_machine.sh 
+module use ${RDASApp}/modulefiles
+module load RDAS/${MACHINE_ID}.intel
+module list
+
+export OOPS_TRACE=1
+export OMP_NUM_THREADS=1
+
+ulimit -s unlimited
+ulimit -v unlimited
+ulimit -a
+
+srun -l -n 160 ${RDASApp}/build/bin/fv3jedi_error_covariance_toolbox.x ./bumploc.yaml

rrfs-test/scripts/templates/fv3jedi_expr/run_jedi.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+#SBATCH --account=rtrr
+#SBATCH --qos=batch # use the normal queue on Gaea
+###SBATCH -M c6 # for Gaea
+###SBATCH --partition=kjet
+###SBATCH --reservation=rrfsens
+#SBATCH --ntasks=160
+#SBATCH -t 00:58:00
+#SBATCH --job-name=fv3jedi_test
+#SBATCH -o log.jedi
+#SBATCH --open-mode=truncate
+#SBATCH --cpus-per-task 4 --exclusive
+
+RDASApp=@RDASApp@
+
+inputfile=./rrfs_fv3jedi_2024052700_Ens3Dvar.yaml # FOR EnVar
+#inputfile=./rrfs_fv3jedi_2024052700_hyb.yaml # FOR HYB
+#inputfile=./rrfs_fv3jedi_2024052700_letkf.yaml # FOR LETKF
+
+if [[ -s /apps/lmod/lmod/init/sh ]]; then
+  . /apps/lmod/lmod/init/sh
+fi
+
+module purge
+source ${RDASApp}/ush/detect_machine.sh 
+module use ${RDASApp}/modulefiles
+module load RDAS/${MACHINE_ID}.intel
+module list
+
+export OOPS_TRACE=1
+export OMP_NUM_THREADS=1
+
+ulimit -s unlimited
+ulimit -v unlimited
+ulimit -a
+
+srun -l -n 160 ${RDASApp}/build/bin/fv3jedi_var.x    ./$inputfile    log.out # FOR HYB/ENVAR
+#srun -l -n 160 ${RDASApp}/build/bin/fv3jedi_letkf.x    ./$inputfile    log.out # FOR LETKF

rrfs-test/scripts/templates/run_gsi_template.sh

@@ -30,7 +30,7 @@ cp -p Data/bkg/* .
 
 srun --label @YOUR_PATH_TO_GSI@/build/src/gsi/gsi.x
 
-ANAL_TIME=2022052619
+ANAL_TIME=@ANAL_TIME@
 # Loop over first and last outer loops to generate innovation
 # diagnostic files for indicated observation types (groups)
 #

rrfs-test/testinput/rrfs_fv3jedi_2024052700_Ens3Dvar.yaml

@@ -0,0 +1,142 @@
+cost function:
+  cost type: 3D-Var
+  time window:
+      begin: 2024-05-26T21:00:00Z
+      length: PT6H
+  analysis variables: &3dvars [eastward_wind,northward_wind,air_temperature,air_pressure_thickness,water_vapor_mixing_ratio_wrt_moist_air,cloud_liquid_ice,cloud_liquid_water,ozone_mass_mixing_ratio]
+
+  geometry:
+    fms initialization:
+      namelist filename: DataFix/fmsmpp.nml
+      field table filename: DataFix/field_table
+    namelist filename: DataFix/input_lam_C775_NP16X10.nml
+    field metadata override: Data/fieldmetadata/tlei-gfs-restart.yaml
+    akbk: DataFix/fix/akbk66.nc
+    # input.nml
+    npz: 65
+    layout: [16,10]
+    io_layout: [1,1]
+    ntiles: 1 
+
+
+  background:
+    datetime: 2024-05-27T00:00:00Z
+    filetype: fms restart
+    datapath: Data/bkg
+    filename_core: 20240527.000000.fv_core.res.tile1.nc
+    filename_trcr: 20240527.000000.fv_tracer.res.tile1.nc 
+    filename_sfcd: 20240527.000000.sfc_data.nc
+    filename_sfcw: 20240527.000000.fv_srf_wnd.res.nc
+    filename_cplr: 20240527.000000.coupler.res
+    state variables: [eastward_wind,northward_wind,u_component_of_native_D_grid_wind,v_component_of_native_D_grid_wind,air_temperature,air_pressure_thickness,water_vapor_mixing_ratio_wrt_moist_air,cloud_liquid_ice,cloud_liquid_water,ozone_mass_mixing_ratio]
+  background error:
+    covariance model: ensemble
+    members from template:
+      template:
+        datetime: 2024-05-27T00:00:00Z
+        filetype: fms restart
+        state variables: *3dvars
+        datapath: Data/inputs/mem%mem%/
+        filename_core: 20240527.000000.fv_core.res.tile1.nc
+        filename_trcr: 20240527.000000.fv_tracer.res.tile1.nc
+        filename_sfcd: 20240527.000000.sfc_data.nc
+        filename_sfcw: 20240527.000000.fv_srf_wnd.res.nc
+        filename_cplr: 20240527.000000.coupler.res
+      pattern: "%mem%"
+      nmembers: 20 
+      zero padding: 3
+    localization:
+      localization method: SABER
+      saber central block:
+        saber block name: BUMP_NICAS
+        active variables: *3dvars 
+        read:
+          io:
+            files prefix: DataFix/bump_new/bumploc_300km_0.3vloc
+          drivers:
+             multivariate strategy: duplicated
+             read local nicas: true
+          model:
+             level for 2d variables: last
+  observations:
+     observers:
+     - obs space:
+         name: sonde
+         obsdatain:
+           engine:
+             type: H5File
+             obsfile: Data/obs/rap.t00z.singleobs.ADPUPA.tm00.nc4
+         obsdataout:
+           engine:
+             type: H5File
+             obsfile: ./adpupa_hofx.nc4
+         simulated variables: [airTemperature]
+         observed variables: [airTemperature]
+       obs operator:
+         name: VertInterp
+       obs error:
+         covariance model: diagonal
+       obs filters:
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           action:
+             name: assign error
+             error parameter: 1.2  # K
+           defer to post: true
+
+
+variational:
+  minimizer:
+    algorithm: DRPCG
+  iterations:
+  - ninner: 5
+    gradient norm reduction: 1e-60
+    geometry:
+      fms initialization:
+        namelist filename: DataFix/fmsmpp.nml
+        field table filename: DataFix/field_table
+      namelist filename: DataFix/input_lam_C775_NP16X10.nml
+      field metadata override: Data/fieldmetadata/tlei-gfs-restart.yaml
+      akbk: DataFix/fix/akbk66.nc
+      # input.nml
+      npz: 65
+      layout: [16,10]
+      io_layout: [1,1]
+      ntiles: 1 
+      fieldsets:
+      - fieldset: DataFix/fix/dynamics_lam_cmaq.yaml
+    diagnostics:
+      departures: ombg
+  - ninner: 5
+    gradient norm reduction: 1e-60
+    geometry:
+      fms initialization:
+        namelist filename: DataFix/fmsmpp.nml
+        field table filename: DataFix/field_table
+      namelist filename: DataFix/input_lam_C775_NP16X10.nml
+      field metadata override: Data/fieldmetadata/tlei-gfs-restart.yaml
+      akbk: DataFix/fix/akbk66.nc
+      # input.nml
+      npz: 65
+      layout: [16,10]
+      io_layout: [1,1]
+      ntiles: 1 
+      fieldsets:
+      - fieldset: DataFix/fix/dynamics_lam_cmaq.yaml
+final:
+  diagnostics:
+    departures: oman
+
+output:
+  filetype: fms restart
+  datapath: ./
+  prefix: ens3dvar-fv3_lam-C775 #Ens3dvar-fv3_lam-C775
+  frequency: PT1H
+test:
+  reference filename: testoutput/rrfs-fv3jedi-ens3dvar.ref
+  test output filename: rrfs-fv3jedi-ens3dvar.out
+  float relative tolerance: 1.0e-3
+  float absolute tolerance: 1.0e-6
+
+