Remove Noah-WRFv4, bug fix in Thompson MP inner loop, update HWRF reg…

…ression tests, set ECF_TRIES to 2, update Cheyenne job submission scripts; contains "Reduce memory required by MERRA2 option; fix diag_tables for P7 tests; adds C384 P7 tests to Cheyenne.intel" (#866), contains "Update CMEPS" (#775) (#831)

- Remove unused/broken HAFS-HWRF regression tests that used Ferrier-Aligo MP, and updates the old HAFS-HWRF regression tests to use Noah MP instead of Noah-WRFv4. 
- An update in rt.sh that uses the automatic rerun capability for ecflow
- Code updates to reduce memory required by MERRA2 option; fix diag_tables for P7 tests; adds C384 P7 tests to Cheyenne.intel
- CMEPS update
- Update Cheyenne job submission scripts due to a recent change on Cheyenne

Co-authored-by: denise.worthen <[email protected]>
Co-authored-by: Dusan Jovic <[email protected]>

modulefiles/ufs_wcoss_cray

@@ -63,7 +63,7 @@ module load gni-headers
 module load udreg
 module load ugni
 
-module load esmf/811
+module load esmf/820bs14
 module load fms/2021.03
 
 module swap pmi pmi/5.0.11

modulefiles/ufs_wcoss_cray_debug

@@ -63,7 +63,7 @@ module load gni-headers
 module load udreg
 module load ugni
 
-module load esmf/811-debug
+module load esmf/820bs14-debug
 module load fms/2021.03
 
 module swap pmi pmi/5.0.11

tests/fv3_conf/c96_HAFS_v0_hwrf_run.IN

@@ -30,11 +30,6 @@ cp    @[INPUTDATA_ROOT]/FV3_input_data/global_h2o_pltc.f77 ./global_h2oprdlos.f7
 cp    @[INPUTDATA_ROOT]/FV3_input_data/*grb .
 cp    @[INPUTDATA_ROOT]/FV3_input_data/diag_table .
 
-# Copy landuse/soil/vegetation parameter tables for HWRF Noah LSM
-cp    @[INPUTDATA_ROOT]/FV3_input_data_hafs/GENPARM.TBL .
-cp    @[INPUTDATA_ROOT]/FV3_input_data_hafs/SOILPARM.TBL .
-cp    @[INPUTDATA_ROOT]/FV3_input_data_hafs/VEGPARM.TBL .
-
 # Copy diag table, depending on microphysics choice
 if [ $IMP_PHYSICS = 8 ]; then
   cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/diag_table_gsd_noah diag_table

tests/fv3_conf/compile_qsub.IN_cheyenne

@@ -1,4 +1,4 @@
-#!/bin/sh -l
+#!/bin/bash -l
 #PBS -o out
 #PBS -e err
 #PBS -N @[JBNME]

tests/fv3_conf/fv3_qsub.IN_cheyenne

@@ -1,4 +1,4 @@
-#!/bin/sh -l
+#!/bin/bash -l
 #PBS -o out
 #PBS -e err
 #PBS -N @[JBNME]

tests/parm/c96_HAFS_v0_hwrf.nml.IN

@@ -153,9 +153,9 @@
   prslrd0        = 0.
   ivegsrc        = 1
   isot           = 1
-  lsm            = @[LSM]          !HWRF set to 4
-  sfc_z0_type    = @[SFC_Z0_TYPE]  !HWRF set to 4
-  iopt_dveg      = 2
+  lsm            = @[LSM]
+  sfc_z0_type    = @[SFC_Z0_TYPE]
+  iopt_dveg      = 1
   iopt_crs       = 1
   iopt_btr       = 1
   iopt_run       = 1

tests/parm/diag_table_template

@@ -1,9 +1,6 @@
 @[SYEAR]@[SMONTH]@[SDAY].@[SHOUR]Z.@[ATMRES].64bit.non-mono
 @[SYEAR] @[SMONTH] @[SDAY] @[SHOUR] 0 0
 
-"grid_spec",              -1,  "months",   1, "days",  "time"
-"atmos_4xdaily",           6,  "hours",    1, "days",  "time"
-"atmos_static",           -1,  "hours",    1, "hours", "time"
 "fv3_history",    0,  "hours",  1,  "hours",  "time"
 "fv3_history2d",  0,  "hours",  1,  "hours",  "time"
 ######################
@@ -65,101 +62,7 @@
  "ocean_model", "fprec",     "fprec",                    "ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2
  "ocean_model", "LwLatSens", "LwLatSens",                "ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2
  "ocean_model", "Heat_PmE",  "Heat_PmE",                 "ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2
-
-#=============================================================================================
-##################
 #
-#=======================
-# ATMOSPHERE DIAGNOSTICS
-#=======================
-###
-# grid_spec
-###
- "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false.,  "none", 2,
- "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false.,  "none", 2,
- "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false.,  "none", 2,
- "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false.,  "none", 2,
- "dynamics", "area",     "area",     "grid_spec", "all", .false.,  "none", 2,
-###
-# 4x daily output
-###
- "dynamics",  "slp",         "slp",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "vort850",     "vort850",    "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "vort200",     "vort200",    "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "us",          "us",         "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u1000",       "u1000",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u850",        "u850",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u700",        "u700",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u500",        "u500",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u200",        "u200",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u100",        "u100",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u50",         "u50",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "u10",         "u10",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "vs",          "vs",         "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v1000",       "v1000",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v850",        "v850",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v700",        "v700",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v500",        "v500",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v200",        "v200",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v100",        "v100",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v50",         "v50",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "v10",         "v10",        "atmos_4xdaily", "all", .false.,  "none", 2
-####
- "dynamics",  "tm",          "tm",         "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t1000",       "t1000",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t850",        "t850",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t700",        "t700",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t500",        "t500",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t200",        "t200",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t100",        "t100",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t50",         "t50",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "t10",         "t10",        "atmos_4xdaily", "all", .false.,  "none", 2
-####
- "dynamics",  "h1000",       "h1000",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h850",        "h850",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h700",        "h700",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h500",        "h500",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h200",        "h200",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h100",        "h100",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h50",         "h50",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "h10",         "h10",        "atmos_4xdaily", "all", .false.,  "none", 2
-####
-#"dynamics",  "w1000",       "w1000",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "w850",        "w850",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "w700",        "w700",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "w500",        "w500",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "w200",        "w200",       "atmos_4xdaily", "all", .false.,  "none", 2
-####
- "dynamics",  "q1000",       "q1000",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q850",        "q850",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q700",        "q700",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q500",        "q500",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q200",        "q200",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q100",        "q100",       "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q50",         "q50",        "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "q10",         "q10",        "atmos_4xdaily", "all", .false.,  "none", 2
-####
- "dynamics",  "rh1000",      "rh1000",     "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "rh850",       "rh850",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "rh700",       "rh700",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "rh500",       "rh500",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "rh200",       "rh200",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg1000",     "omg1000",    "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg850",      "omg850",     "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg700",      "omg700",     "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg500",      "omg500",     "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg200",      "omg200",     "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg100",      "omg100",     "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg50",       "omg50",      "atmos_4xdaily", "all", .false.,  "none", 2
- "dynamics",  "omg10",       "omg10",      "atmos_4xdaily", "all", .false.,  "none", 2
-###
-# gfs static data
-###
- "dynamics",      "pk",          "pk",           "atmos_static",      "all", .false.,  "none", 2
- "dynamics",      "bk",          "bk",           "atmos_static",      "all", .false.,  "none", 2
- "dynamics",      "hyam",        "hyam",         "atmos_static",      "all", .false.,  "none", 2
- "dynamics",      "hybm",        "hybm",         "atmos_static",      "all", .false.,  "none", 2
- "dynamics",      "zsurf",       "zsurf",        "atmos_static",      "all", .false.,  "none", 2
 ###
 # FV3 variabls needed for NGGPS evaluation
 ###
@@ -172,8 +75,14 @@
 "gfs_dyn",     "delp",        "dpres",     "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "delz",        "delz",      "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "w",           "dzdt",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "ice_wat",     "icmr",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "rainwat",     "rwmr",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "snowwat",     "snmr",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "graupel",     "grle",      "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "ps",          "pressfc",   "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "hs",          "hgtsfc",    "fv3_history",    "all",  .false.,  "none",  2
+#"gfs_dyn",     "ice_nc",      "nicp",      "fv3_history",  "all",  .false.,  "none",  2
+#"gfs_dyn",     "rain_nc",     "ntrnc",     "fv3_history",  "all",  .false.,  "none",  2
 
 "gfs_phys",  "ALBDO_ave",     "albdo_ave",   "fv3_history2d",         "all",  .false.,  "none",  2
 "gfs_phys",  "cnvprcp_ave",   "cprat_ave",   "fv3_history2d",         "all",  .false.,  "none",  2
@@ -218,6 +127,7 @@
 "gfs_phys",  "v-gwd_ave",     "v-gwd_ave",   "fv3_history2d",         "all",  .false.,  "none",  2
 "gfs_phys",  "dusfc",         "uflx_ave",    "fv3_history2d",         "all",  .false.,  "none",  2
 "gfs_phys",  "dvsfc",         "vflx_ave",    "fv3_history2d",         "all",  .false.,  "none",  2
+#"gfs_phys",  "cnvw",          "cnvcldwat",   "fv3_history2d",         "all",  .false.,  "none",  2
 
 "gfs_phys",    "psurf",       "pressfc",   "fv3_history2d",  "all",  .false.,  "none",  2
 "gfs_phys",    "u10m",        "ugrd10m",   "fv3_history2d",  "all",  .false.,  "none",  2
@@ -325,6 +235,12 @@
 "gfs_phys",    "nddsf_ave",    "nddsf_ave",     "fv3_history2d",  "all",  .false.,  "none",  2
 "gfs_phys",    "trans_ave",    "trans_ave",     "fv3_history2d",  "all",  .false.,  "none",  2
 
+"gfs_phys",    "AOD_550",      "aod550",        "fv3_history2d",  "all",  .false.,  "none",  2
+"gfs_phys",    "DU_AOD_550",   "du_aod550",     "fv3_history2d",  "all",  .false.,  "none",  2
+"gfs_phys",    "SU_AOD_550",   "su_aod550",     "fv3_history2d",  "all",  .false.,  "none",  2
+"gfs_phys",    "BC_AOD_550",   "bc_aod550",     "fv3_history2d",  "all",  .false.,  "none",  2
+"gfs_phys",    "OC_AOD_550",   "oc_aod550",     "fv3_history2d",  "all",  .false.,  "none",  2
+"gfs_phys",    "SS_AOD_550",   "ss_aod550",     "fv3_history2d",  "all",  .false.,  "none",  2
 #=============================================================================================
 #
 #====> This file can be used with diag_manager/v2.0a (or higher) <====

tests/parm/esg_HAFS_v0_hwrf.nml.IN

@@ -162,9 +162,9 @@
   prslrd0        = 0.
   ivegsrc        = 1
   isot           = 1
-  lsm            = @[LSM]          !HWRF set to 4
-  sfc_z0_type    = @[SFC_Z0_TYPE]  !HWRF set to 4
-  iopt_dveg      = 2
+  lsm            = @[LSM]
+  sfc_z0_type    = @[SFC_Z0_TYPE]
+  iopt_dveg      = 1
   iopt_crs       = 1
   iopt_btr       = 1
   iopt_run       = 1

tests/parm/nems.configure.cpld.IN

@@ -102,4 +102,7 @@ ALLCOMP_attributes::
       use_coldstart = @[use_coldstart]
       use_mommesh = @[use_mommesh]
       eps_imesh = @[eps_imesh]
+      stop_n = @[FHMAX]
+      stop_option = nhours
+      stop_ymd = -999
 ::

tests/parm/nems.configure.cpld_wave.IN

@@ -116,4 +116,7 @@ ALLCOMP_attributes::
       use_coldstart = @[use_coldstart]
       use_mommesh = @[use_mommesh]
       eps_imesh = @[eps_imesh]
+      stop_n = @[FHMAX]
+      stop_option = nhours
+      stop_ymd = -999
 ::

tests/parm/nems.configure.datm_cdeps.IN

@@ -122,4 +122,7 @@ ALLCOMP_attributes::
       orb_mode = fixed_year
       orb_mvelp = 1.e36
       orb_obliq = 1.e36
+      stop_n = @[FHMAX]
+      stop_option = nhours
+      stop_ymd = -999
 ::

tests/parm/nems.configure.hafs_atm_docn.IN

@@ -94,6 +94,9 @@ ALLCOMP_attributes::
   orb_obliq = 1.e36
   mediator_read_restart = @[USE_COLDSTART]
   mediator_present = true
+  stop_n = @[FHMAX]
+  stop_option = nhours
+  stop_ymd = -999
 ::
 
 ATM_modelio::

tests/parm/nems.configure.hafs_atm_ocn.IN

@@ -115,6 +115,9 @@ ALLCOMP_attributes::
   orb_obliq = 1.e36
   mediator_read_restart = @[USE_COLDSTART]
   mediator_present = true
+  stop_n = @[FHMAX]
+  stop_option = nhours
+  stop_ymd = -999
 ::
 
 ATM_modelio::

tests/rt.conf

@@ -14,11 +14,11 @@ RUN     | cpld_control_wave_p7
 RUN     | cpld_control_c192_p7                                                                                                    | - wcoss_cray                            | fv3 |
 RUN     | cpld_restart_c192_p7                                                                                                    | - wcoss_cray                            |     | cpld_control_c192_p7
 
-RUN     | cpld_control_c384_p7                                                                                                    | - wcoss_cray cheyenne.intel             | fv3 |
-RUN     | cpld_restart_c384_p7                                                                                                    | - wcoss_cray cheyenne.intel             |     | cpld_control_c384_p7
+RUN     | cpld_control_c384_p7                                                                                                    | - wcoss_cray                            | fv3 |
+RUN     | cpld_restart_c384_p7                                                                                                    | - wcoss_cray                            |     | cpld_control_c384_p7
 
 # Waves on, BM IC
-RUN     | cpld_bmark_p7                                                                                                           | - wcoss_cray cheyenne.intel             | fv3 |
+RUN     | cpld_bmark_p7                                                                                                           | - wcoss_cray                             | fv3 |
 
 COMPILE | -DAPP=S2S -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v16_coupled_nsstNoahmpUGWPv1                                                 | - wcoss_cray                            | fv3 |
 RUN     | cpld_debug_p7                                                                                                           | - wcoss_cray                            | fv3 |
@@ -63,7 +63,7 @@ RUN     | fv3_rap
 RUN     | fv3_hrrr                                                                                                                |                                         | fv3 |
 RUN     | fv3_rrfs_v1beta                                                                                                         |                                         | fv3 |
 
-COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_RRTMGP,FV3_GFS_v16_csawmg,FV3_GFS_v16_flake,FV3_GFS_v16_ugwpv1,FV3_GFS_v16_ras,FV3_GFS_v16_thompson,FV3_GFS_v16_noahmp,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf | | fv3 |
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_RRTMGP,FV3_GFS_v16_csawmg,FV3_GFS_v16_flake,FV3_GFS_v16_ugwpv1,FV3_GFS_v16_ras,FV3_GFS_v16_thompson,FV3_GFS_v16_noahmp,FV3_HAFS_v0_hwrf_thompson | | fv3 |
 # fv3_gfs_v15p2 and fv3_gfs_v15p2_RRTMGP fail w/ sat.vap pressure error when cdmbgwd=0.14,1.8,1.0,1.0
 RUN     | control_rrtmgp                                                                                                          |                                         | fv3 |
 #RUN     | control_rrtmgp_2threads                                                                                                 |                                         |     |
@@ -77,7 +77,6 @@ RUN     | control_thompson
 RUN     | control_thompson_no_aero                                                                                                | - wcoss_cray                            | fv3 |
 
 RUN     | fv3_HAFS_v0_hwrf_thompson                                                                                               |                                         | fv3 |
-#RUN     | fv3_HAFS_v0_hwrf                                                                                                       |                                         | fv3 |
 RUN     | fv3_esg_HAFS_v0_hwrf_thompson                                                                                           |                                         | fv3 |
 
 # Run WAM test in REPRO mode to avoid numerical instability in the deep atmosphere
@@ -120,7 +119,7 @@ RUN     | fv3_gsd_diag_debug
 RUN     | fv3_rrfs_v1beta_debug                                                                                                   |                                         | fv3 |
 RUN     | fv3_rrfs_v1alpha_debug                                                                                                  |                                         | fv3 |
 
-COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -DDEBUG=ON                                           |                                         | fv3 |
+COMPILE | -DAPP=ATM -DCCPP_SUITES=FV3_HAFS_v0_hwrf_thompson -DDEBUG=ON                                                            |                                         | fv3 |
 RUN     | fv3_HAFS_v0_hwrf_thompson_debug                                                                                         |                                         | fv3 |
 RUN     | fv3_esg_HAFS_v0_hwrf_thompson_debug                                                                                     |                                         | fv3 |
 

tests/rt.sh

@@ -460,7 +460,7 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20211027
+BL_DATE=20211028
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]] || [[ $MACHINE_ID = s4.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else
@@ -494,7 +494,7 @@ JOB_NR=0
 TEST_NR=0
 COMPILE_NR=0
 COMPILE_PREV_WW3_NR=''
-rm -f fail_test
+rm -f fail_test* fail_compile*
 
 export LOG_DIR=${PATHRT}/log_$MACHINE_ID
 rm -rf ${LOG_DIR}
@@ -590,7 +590,7 @@ suite ${ECFLOW_SUITE}
     edit ECF_HOME '${ECFLOW_RUN}'
     edit ECF_INCLUDE '${ECFLOW_RUN}'
     edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1
-    edit ECF_TRIES 1
+    edit ECF_TRIES 2
     label src_dir '${PATHTR}'
     label run_dir '${RUNDIR_ROOT}'
     limit max_builds ${MAX_BUILDS}
@@ -823,6 +823,14 @@ fi
 set +e
 cat ${LOG_DIR}/compile_*_time.log              >> ${REGRESSIONTEST_LOG}
 cat ${LOG_DIR}/rt_*.log                        >> ${REGRESSIONTEST_LOG}
+
+FILES="fail_test_* fail_compile_*"
+for f in $FILES; do
+  if [[ -f "$f" ]]; then
+    cat "$f" >> fail_test
+  fi
+done
+
 if [[ -e fail_test ]]; then
   echo "FAILED TESTS: "
   echo "FAILED TESTS: "                        >> ${REGRESSIONTEST_LOG}