Adding a mechanism to deal with the RAVE duplicate data files (#1160)

* update exaqm_fire_emission.sh script to handle the duplicate RAVE data files

scripts/exaqm_fire_emission.sh

@@ -77,49 +77,61 @@ hh_mh1=${CDATE_mh1:8:2}
 # Retrieve fire file to FIRE_EMISSION_STAGING_DIR
 #
 #-----------------------------------------------------------------------
-#
+
 aqm_fire_file_fn="${AQM_FIRE_FILE_PREFIX}_${yyyymmdd}_t${hh}z${AQM_FIRE_FILE_SUFFIX}"
 
 # Check if the fire file exists in the designated directory
 if [ -e "${DCOMINfire}/${aqm_fire_file_fn}" ]; then
   cpreq "${DCOMINfire}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
-else
-  # Copy raw data 
-  for ihr in {0..23}; do
-    download_time=`$NDATE -$ihr ${yyyymmdd_mh1}${hh_mh1}`
-    FILE_curr=Hourly_Emissions_13km_${download_time}00_${download_time}00.nc
-    FILE_13km=RAVE-HrlyEmiss-13km_v2r0_blend_s${download_time}00000_e${download_time}59590_c*.nc
-    yyyymmdd_dn=${download_time:0:8}
-    hh_dn=${download_time:8:2}
-    missing_download_time=`$NDATE -24 ${yyyymmdd_dn}${hh_dn}`
-    yyyymmdd_dn_md1=${missing_download_time:0:8}
-    FILE_13km_md1=RAVE-HrlyEmiss-13km_v2r0_blend_s${missing_download_time}00000_e${missing_download_time}59590_c*.nc
-    if ls ${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km} 1> /dev/null 2>&1; then
-       if [ $(stat -c %s ${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km}) -gt 4000000 ]; then
-           cpreq ${DCOMINfire}/${yyyymmdd_dn}/rave/${FILE_13km} ${FILE_curr}
+ else
+   # Copy raw data
+   for ihr in {0..23}; do
+     download_time=$($NDATE -$ihr ${yyyymmdd_mh1}${hh_mh1})
+     FILE_curr="Hourly_Emissions_13km_${download_time}00_${download_time}00.nc"
+     FILE_13km_pattern="RAVE-HrlyEmiss-13km_v2r0_blend_s${download_time}00000_e${download_time}59590_c*.nc"
+     yyyymmdd_dn=${download_time:0:8}
+     hh_dn=${download_time:8:2}
+     missing_download_time=$($NDATE -24 ${yyyymmdd_dn}${hh_dn})
+     yyyymmdd_dn_md1=${missing_download_time:0:8}
+     FILE_13km_md1_pattern="RAVE-HrlyEmiss-13km_v2r0_blend_s${missing_download_time}00000_e${missing_download_time}59590_c*.nc"
+
+    # Function to find the latest file based on creation time
+    find_latest_file() {
+      local dir=$1
+      local pattern=$2
+      ls ${dir}/${pattern} 2>/dev/null | \
+        awk -F'_' '{split($NF, a, "c"); print $0, a[2]}' | \
+        sort -k2,2nr | head -n 1 | awk '{print $1}'
+    }
+
+    FILE_13km=$(find_latest_file "${DCOMINfire}/${yyyymmdd_dn}/rave" "${FILE_13km_pattern}")
+    FILE_13km_md1=$(find_latest_file "${DCOMINfire}/${yyyymmdd_dn_md1}/rave" "${FILE_13km_md1_pattern}")
+
+    if ls ${FILE_13km} 1> /dev/null 2>&1; then
+       if [ $(stat -c %s ${FILE_13km}) -gt 4000000 ]; then
+           cpreq ${FILE_13km} ${FILE_curr}
        fi
-    elif ls ${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1} 1> /dev/null 2>&1; then
-       if [ $(stat -c %s ${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1}) -gt 4000000 ]; then
+    elif ls ${FILE_13km_md1} 1> /dev/null 2>&1; then
+       if [ $(stat -c %s ${FILE_13km_md1}) -gt 4000000 ]; then
            echo "WARNING: ${FILE_13km} does not exist or is broken. Replacing with the file from the previous date..."
-           cpreq ${DCOMINfire}/${yyyymmdd_dn_md1}/rave/${FILE_13km_md1} ${FILE_curr}
+           cpreq ${FILE_13km_md1} ${FILE_curr}
        fi
     else
-      message_txt="WARNING Fire Emission RAW data does not exist or broken:
+      message_txt="WARNING Fire Emission RAW data does not exist or is broken:
   FILE_13km_md1 = \"${FILE_13km_md1}\"
   DCOMINfire = \"${DCOMINfire}\""
 
-        cpreq ${FIXaqmfire}/Hourly_Emissions_13km_dummy.nc ${FILE_curr}
-        message_warning="WARNING: ${message_txt}. Replacing with the dummy file :: AQM RUN SOFT FAILED."
-        print_info_msg "${message_warning}"
-        if [ "${EMAIL_SDM^^}" = "YES" ] ; then
-	  MAILFROM=${MAILFROM:-"[email protected]"}
-          MAILTO=${MAILTO:-"${maillist}"}
-          subject="${cyc}Z ${RUN^^} Output for ${basinname:-} WILDFIRE EMIS "
-          mail.py -s "${subject}" -v "${MAILTO}" 
-	fi
-
+      cpreq "${FIXaqmfire}/Hourly_Emissions_13km_dummy.nc" "${FILE_curr}"
+      message_warning="WARNING: ${message_txt}. Replacing with the dummy file :: AQM RUN SOFT FAILED."
+      print_info_msg "${message_warning}"
+      if [ "${EMAIL_SDM^^}" = "YES" ]; then
+        MAILFROM=${MAILFROM:-"[email protected]"}
+        MAILTO=${MAILTO:-"${maillist}"}
+        subject="${cyc}Z ${RUN^^} Output for ${basinname:-} WILDFIRE EMIS "
+        mail.py -s "${subject}" -v "${MAILTO}"
+      fi
     fi
-  done  
+  done
 
   ncks -O -h --mk_rec_dmn time Hourly_Emissions_13km_${download_time}00_${download_time}00.nc temp.nc
   export err=$?
@@ -165,6 +177,14 @@ else
       err_exit "${message_txt}"
   fi
 
+  mv ${aqm_fire_file_fn} temp.nc
+  ncrename -v PM2.5,PM25 temp.nc temp1.nc
+  ncap2 -s 'where(Latitude > 30 && Latitude <=49 && land_cover == 1 ) PM25 = PM25 * 0.22222' temp1.nc temp2.nc
+  ncap2 -s 'where(Latitude <=30 && land_cover == 1 ) PM25 = PM25 * 0.4'       temp2.nc temp3.nc
+  ncap2 -s 'where(Latitude <=30 && land_cover >= 2 ) PM25 = PM25 * 0.55555'   temp3.nc temp4.nc
+  ncrename -v PM25,PM2.5 temp4.nc temp5.nc
+  mv temp5.nc ${aqm_fire_file_fn}
+
   # Copy the final fire emission file to STAGING_DIR 
   cpreq "${DATA}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
 

versions/run.ver

@@ -28,7 +28,6 @@ export envvar_ver=1.0
 
 export gfs_ver=v16.3
 export gefs_ver=v12.3
-export aqm_ver=v7.0.8
 
 export udunits_ver=2.2.28
 export gsl_ver=2.7