Merge branch 'develop' into feature/aero_3dvar_test

scripts/exgdas_global_marine_analysis_prep.py

@@ -517,22 +517,17 @@ def find_clim_ens(input_date):
 # write the two seaice analysis to model change of variable yamls
 varchgyamls = ['soca_2cice_arctic.yaml', 'soca_2cice_antarctic.yaml']
 soca2cice_cfg = {
-    "template": "",
-    "output": "",
-    "config": {
-        "OCN_ANA": "./Data/ocn.3dvarfgat_pseudo.an."+window_middle_iso+".nc",
-        "ICE_ANA": "./Data/ice.3dvarfgat_pseudo.an."+window_middle_iso+".nc",
-        "ICE_RST": ice_rst_ana,
-        "FCST_BEGIN": fcst_begin.strftime('%Y-%m-%dT%H:%M:%SZ')
-    }
+    "OCN_ANA": "./Data/ocn.3dvarfgat_pseudo.an."+window_middle_iso+".nc",
+    "ICE_ANA": "./Data/ice.3dvarfgat_pseudo.an."+window_middle_iso+".nc",
+    "ICE_RST": ice_rst_ana,
+    "FCST_BEGIN": fcst_begin.strftime('%Y-%m-%dT%H:%M:%SZ')
 }
 varchgyamls = ['soca_2cice_arctic.yaml', 'soca_2cice_antarctic.yaml']
 for varchgyaml in varchgyamls:
-    soca2cice_cfg['template'] = os.path.join(gdas_home, 'parm', 'soca', 'varchange', varchgyaml)
-    f = open('tmp.yaml', 'w')
-    # TODO: use YAMLFile instead
-    yaml.dump(soca2cice_cfg, f, sort_keys=False, default_flow_style=False)
-    ufsda.genYAML.genYAML('tmp.yaml', output=varchgyaml)
+    soca2cice_cfg_template = os.path.join(gdas_home, 'parm', 'soca', 'varchange', varchgyaml)
+    outyaml = YAMLFile(path=soca2cice_cfg_template)
+    outyaml = Template.substitute_structure(outyaml, TemplateConstants.DOLLAR_PARENTHESES, soca2cice_cfg.get)
+    outyaml.save(varchgyaml)
 
 # prepare yaml for soca to MOM6 IAU increment
 logging.info(f"---------------- generate soca to MOM6 IAU yaml")

test/CMakeLists.txt

@@ -121,46 +121,6 @@ add_test(NAME test_gdasapp_jedi_increment_to_fv3
          COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/jediinc2fv3.py ${PROJECT_BINARY_DIR}/test/testdata/atmges_compress.nc4 ${PROJECT_BINARY_DIR}/test/testdata/atminc_compress.nc4 ${PROJECT_BINARY_DIR}/test/testoutput/fv_increment.nc
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
 
-# test for YAML generation from a template
-add_test(NAME test_gdasapp_genYAML_prep
-         COMMAND ${PROJECT_SOURCE_DIR}/test/genYAML_prep.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
-add_test(NAME test_gdasapp_genYAML_run
-          COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/genYAML --config ${PROJECT_BINARY_DIR}/test/testoutput/genYAML_example.yaml
-          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-set_tests_properties(
-        test_gdasapp_genYAML_run
-PROPERTIES
-ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH};PARMgfs=${PROJECT_SOURCE_DIR}/parm")
-
-# test for YAML generation from a template for land DA
-add_test(NAME test_gdasapp_genYAML_land_prep
-         COMMAND ${PROJECT_SOURCE_DIR}/test/genYAML_prep_land.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
-add_test(NAME test_gdasapp_genYAML_land_run
-          COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/genYAML --config ${PROJECT_BINARY_DIR}/test/testoutput/genYAML_example_land.yaml
-          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-set_tests_properties(
-        test_gdasapp_genYAML_land_run
-PROPERTIES
-ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH};PARMgfs=${PROJECT_SOURCE_DIR}/parm")
-
-
-# test for YAML generation from a template for aero DA
-add_test(NAME test_gdasapp_genYAML_aero_prep
-         COMMAND ${PROJECT_SOURCE_DIR}/test/genYAML_prep_aero.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
-add_test(NAME test_gdasapp_genYAML_aero_run
-          COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/genYAML --config ${PROJECT_BINARY_DIR}/test/testoutput/genYAML_example_aero.yaml
-          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-set_tests_properties(
-        test_gdasapp_genYAML_aero_run
-PROPERTIES
-ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH};PARMgfs=${PROJECT_SOURCE_DIR}/parm")
-
 # test for converting ewok yaml to gdas
 add_test(NAME test_gdasapp_convert_ewok_yaml
          COMMAND ${PROJECT_SOURCE_DIR}/test/convert_ewok_yaml.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}

test/atm/global-workflow/CMakeLists.txt

@@ -1,30 +1,6 @@
 # test for creating an experiment directory within the global-workflow
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun)
 
-# test for ush/run_jedi_exe.py 3dhofx
-add_test(NAME test_gdasapp_run_jedi_exe_3dhofx
-  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dhofx.sh
-  	  ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
-# test for ush/run_jedi_exe.py 3dvar
-add_test(NAME test_gdasapp_run_jedi_exe_3dvar
-  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dvar.sh
-  	  ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
-# test for ush/run_jedi_exe.py 3denvar
-add_test(NAME test_gdasapp_run_jedi_exe_3denvar
-  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3denvar.sh
-  	  ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
-# test for ush/run_jedi_exe.py letkf
-add_test(NAME test_gdasapp_run_jedi_exe_letkf
-  COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_letkf.sh
-  	  ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
-  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-
 add_test(NAME test_gdasapp_setup_atm_cycled_exp
   COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh
           ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
@@ -60,13 +36,3 @@ add_test(NAME test_gdasapp_atm_jjob_ens_final
           ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
   WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun)
 
-# python fix
-  set_tests_properties(
-      test_gdasapp_run_jedi_exe_3dhofx
-      test_gdasapp_run_jedi_exe_3dvar
-      test_gdasapp_run_jedi_exe_3denvar
-      test_gdasapp_run_jedi_exe_letkf
-      PROPERTIES
-          ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}"
-  )
-

test/atm/global-workflow/jjob_var_init.sh

@@ -21,7 +21,7 @@ export COMIN_GES=${bindir}/test/atm/bkg
 export pid=${pid:-$$}
 export jobid=$pid
 export COMROOT=$DATAROOT
-export NMEM_ENS=0
+export NMEM_ENS=3
 export ACCOUNT=da-cpu
 export COM_TOP=$ROTDIR
 
@@ -105,7 +105,7 @@ dpath=enkfgdas.$gPDY/$gcyc
 for imem in $(seq 1 $NMEM_ENS); do
     memchar="mem"$(printf %03i $imem)
 
-    MEMDIR=${memchar} RUN=${RUN} YMD=${gPDY} HH=${gcyc} generate_com -x \
+    MEMDIR=${memchar} RUN=enkf${RUN} YMD=${gPDY} HH=${gcyc} generate_com -x \
 	COM_ATMOS_HISTORY_PREV_ENS:COM_ATMOS_HISTORY_TMPL \
 	COM_ATMOS_RESTART_PREV_ENS:COM_ATMOS_RESTART_TMPL
     COM_ATMOS_RESTART_PREV_DIRNAME_ENS=$(dirname $COM_ATMOS_RESTART_PREV_ENS)