diff --git a/dump/mapping/bufr2ioda_adpupa_prepbufr.py b/dump/mapping/bufr2ioda_adpupa_prepbufr.py new file mode 100644 index 0000000..8dbca27 --- /dev/null +++ b/dump/mapping/bufr2ioda_adpupa_prepbufr.py @@ -0,0 +1,132 @@ +import os +import sys +sys.path.append('/work2/noaa/da/pkumar/HERCULES/EMC-bufr-query/bufr-query/build/lib') +sys.path.append('/work2/noaa/da/pkumar/HERCULES/EMC-bufr-query/bufr-query/build/lib/python3.10') +sys.path.append('/work2/noaa/da/pkumar/HERCULES/EMC-bufr-query/bufr-query/build/lib/python3.10/site-packages') + +import bufr +from bufr.encoders import netcdf +import copy +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +from datetime import datetime +import argparse + + +def Mask_typ_for_var(typ, var): + + typ_var = copy.deepcopy(typ) + for i in range(len(typ_var)): + if ma.is_masked(var[i]): + typ_var[i] = typ_var.fill_value + + return typ_var + + +def create_obs_group(input_path, output_path, mapping_path): + + container = bufr.Parser(input_path, mapping_path).parse() + + print("Get the Variables/Paths") + lat_path = container.get_paths('variables/latitude') + typ_path = container.get_paths('variables/observationType') + typ = container.get('variables/observationType') + cat = container.get('variables/prepbufrDataLevelCategory') + tpc = container.get('variables/temperatureEventProgramCode') + tob = container.get('variables/airTemperature') + pob = container.get('variables/pressure') + #tvo = container.get('variables/virtualTemperature') + uob = container.get('variables/windEastward') + vob = container.get('variables/windNorthward') + qob = container.get('variables/specificHumidity') + pqm = container.get('variables/pressureQualityMarker') + tqm = container.get('variables/airTemperatureQualityMarker') + poe = container.get('variables/pressureError') + toe = container.get('variables/airTemperatureError') + + print("ObsValue- Derived Variable Calculation") + ps = np.full(pob.shape[0], pob.fill_value) + ps = np.where(cat == 0, pob, ps) + + tsen = np.full(tob.shape[0], tob.fill_value) + tsen = np.where(((tpc >= 1) & (tpc < 8)), tob, tsen) + + tvo = np.full(tob.shape[0], tob.fill_value) + tvo = np.where((tpc == 8), tob, tvo) + + print("QualityMarker- Derived Variable Calculation") + psqm = np.full(pqm.shape[0], pqm.fill_value) + psqm = np.where(cat == 0, pqm, psqm) + + tsenqm = np.full(tqm.shape[0], tqm.fill_value) + tsenqm = np.where(((tpc >= 1) & (tpc < 8)), tqm, tsenqm) + + tvoqm = np.full(tqm.shape[0], tqm.fill_value) + tvoqm = np.where((tpc == 8), tqm, tvoqm) + + print("ObsError- Derived Variable Calculation") + psoe = ma.array(np.full(poe.shape[0], poe.fill_value)) + psoe = ma.where(cat == 0, poe, psoe) + + tsenoe = np.full(toe.shape[0], toe.fill_value) + tsenoe = np.where(((tpc >= 1) & (tpc < 8)), toe, tsenoe) + + tvooe = np.full(toe.shape[0], toe.fill_value) + tvooe = np.where((tpc == 8), toe, tvooe) + + print("Add Variables to Container") + container.add('variables/stationPressure', ps, lat_path) + container.add('variables/stationPressureQualityMarker', psqm, lat_path) + container.add('variables/stationPressureError', psoe, lat_path) + + container.add('variables/sensibleTemperature', tsen, lat_path) + container.add('variables/sensibleTemperatureQualityMarker', tsenqm, lat_path) + container.add('variables/sensibleTemperatureError', tsenoe, lat_path) + + #container.add('variables/virtualTemperature', tvo, lat_path) + container.add('variables/virtualTemperatureQualityMarker', tvoqm, lat_path) + container.add('variables/virtualTemperatureError', tvooe, lat_path) + + print("Create ObsType Variables") + typ_pob = Mask_typ_for_var(typ, pob) + typ_ps = Mask_typ_for_var(typ, ps) + typ_tob = Mask_typ_for_var(typ, tob) + typ_tvo = Mask_typ_for_var(typ, tvo) + typ_uob = Mask_typ_for_var(typ, uob) + typ_vob = Mask_typ_for_var(typ, vob) + typ_qob = Mask_typ_for_var(typ, qob) + + print("Add ObsType Variables to Container") + container.add('variables/pressureObsType', typ_pob, typ_path) + container.add('variables/stationPressureObsType', typ_ps, typ_path) + container.add('variables/airTemperatureObsType', typ_tob, typ_path) + container.add('variables/virtualTemperatureObsType', typ_tvo, typ_path) + container.add('variables/windEastwardObsType', typ_uob, typ_path) + container.add('variables/windNorthwardObsType', typ_vob, typ_path) + container.add('variables/specificHumidityObsType', typ_qob, typ_path) + + netcdf.Encoder(mapping_path).encode(container, output_path) + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', type=str, help='Input BUFR', required=True) + parser.add_argument('-o', '--output', type=str, help='Output NetCDF', required=True) + parser.add_argument('-m', '--mapping', type=str, help='Mapping YAML', required=True) + args = parser.parse_args() + + infile = args.input + outfile = args.output + mappingfile = args.mapping + + create_obs_group(infile, outfile, mappingfile) + + end_time = time.time() + running_time = end_time - start_time + print("Total running time: ", running_time) diff --git a/dump/mapping/bufr2ioda_adpupa_prepbufr_mapping.yaml b/dump/mapping/bufr2ioda_adpupa_prepbufr_mapping.yaml new file mode 100644 index 0000000..2765318 --- /dev/null +++ b/dump/mapping/bufr2ioda_adpupa_prepbufr_mapping.yaml @@ -0,0 +1,368 @@ +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC + +bufr: + group_by_variable: prepbufrDataLevelCategory + variables: + # ObsType + observationType: + query: "*/TYP" + + # MetaData + timestamp: + timeoffset: + timeOffset: "*/PRSLEVEL/DRFTINFO/HRDR" + transforms: + - scale: 3600 + referenceTime: "2020-11-01T12:00:00Z" + + prepbufrDataLevelCategory: + query: "*/PRSLEVEL/CAT" + + longitude: + query: "*/PRSLEVEL/DRFTINFO/XDR" + + latitude: + query: "*/PRSLEVEL/DRFTINFO/YDR" + + stationIdentification: + query: "*/SID" + + stationElevation: + query: "*/ELV" + + temperatureEventProgramCode: + query: "*/PRSLEVEL/T___INFO/T__EVENT{1}/TPC" + + pressure: + query: "*/PRSLEVEL/P___INFO/P__EVENT{1}/POB" + type: float + transforms: + - scale: 100 + + height: + query: "*/PRSLEVEL/Z___INFO/Z__EVENT{1}/ZOB" + type: float + + # ObsValue + airTemperature: + query: "*/PRSLEVEL/T___INFO/T__EVENT{1}/TOB" + transforms: + - offset: 273.15 + + dewPointTemperature: + query: "*/PRSLEVEL/Q___INFO/TDO" + transforms: + - offset: 273.15 + + virtualTemperature: + query: "*/PRSLEVEL/T___INFO/TVO" + transforms: + - offset: 273.15 + + windEastward: + query: "*/PRSLEVEL/W___INFO/W__EVENT{1}/UOB" + + windNorthward: + query: "*/PRSLEVEL/W___INFO/W__EVENT{1}/VOB" + + specificHumidity: + query: "*/PRSLEVEL/Q___INFO/Q__EVENT{1}/QOB" + type: float + transforms: + - scale: 0.000001 + + # QualityMark + pressureQualityMarker: + query: "*/PRSLEVEL/P___INFO/P__EVENT{1}/PQM" + + heightQualityMarker: + query: "*/PRSLEVEL/Z___INFO/Z__EVENT{1}/ZQM" + + airTemperatureQualityMarker: + query: "*/PRSLEVEL/T___INFO/T__EVENT{1}/TQM" + + windQualityMarker: + query: "*/PRSLEVEL/W___INFO/W__EVENT{1}/WQM" + + specificHumidityQualityMarker: + query: "*/PRSLEVEL/Q___INFO/Q__EVENT{1}/QQM" + + # ObsError + pressureError: + query: "*/PRSLEVEL/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + + airTemperatureError: + query: "*/PRSLEVEL/T___INFO/T__BACKG/TOE" + transforms: + - offset: 273.15 + + windError: + query: "*/PRSLEVEL/W___INFO/W__BACKG/WOE" + + specificHumidityError: + query: "*/PRSLEVEL/Q___INFO/Q__BACKG/QOE" + + +encoder: + type: netcdf + + globals: + - name: "data_format" + type: string + value: "prepbufr" + + - name: "data_type" + type: string + value: "ADPUPA" + + - name: "subsets" + type: string + value: "ADPUPA" + + - name: "data_provider" + type: string + value: "U.S. NOAA" + + - name: "data_description" + type: string + value: "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS" + + + variables: + # Observation Type: stationPressure + - name: "ObsType/stationPressure" + source: variables/stationPressureObsType + longName: "Observation Type" + + # Observation Type: airTemperature + - name: "ObsType/airTemperature" + source: variables/airTemperatureObsType + longName: "Observation Type" + + # Observation Type: virtualTemperature + - name: "ObsType/virtualTemperature" + source: variables/virtualTemperatureObsType + longName: "Observation Type" + + # Observation Type: windEastward + - name: "ObsType/windEastward" + source: variables/windEastwardObsType + longName: "Observation Type" + + # Observation Type: windNorthward + - name: "ObsType/windNorthward" + source: variables/windNorthwardObsType + longName: "Observation Type" + + # Observation Type: specificHumidity + - name: "ObsType/specificHumidity" + source: variables/specificHumidityObsType + longName: "Observation Type" + + # MetaData: dateTime + - name: "MetaData/dateTime" + coordinates: "longitude latitude" + source: variables/timestamp + longName: "Station ID" + units: "seconds since 1970-01-01T00:00:00Z" + + # MetaData: stationIdentification + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station ID" + + # MetaData: longitude + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [0, 360] + + # MetaData: latitude + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + # MetaData: stationElevation + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Height of Station" + units: "m" + + # MetaData: prepbufrDataLevelCategory + - name: "MetaData/prepbufrDataLevelCategory" + coordinates: "longitude latitude" + source: variables/prepbufrDataLevelCategory + longName: "Prepbufr Data Level Category" + + # MetaData: pressure + - name: "MetaData/pressure" + coordinates: "longitude latitude" + source: variables/pressure + longName: "Pressure" + units: "Pa" + + # MetaData: height + - name: "MetaData/height" + coordinates: "longitude latitude" + source: variables/height + longName: "Height of Observation" + units: "m" + + # QCFlags: qualityFlags + - name: "QCFlags/qualityFlags" + #- name: "MetaData/temperatureEventProgramCode" + source: variables/temperatureEventProgramCode + longName: "Temperature Event Program Code" + + # ObsValue: stationPressure + - name: "ObsValue/stationPressure" + source: variables/stationPressure + longName: "Station Pressure" + units: "Pa" + + # ObsValue: airTemperature + - name: "ObsValue/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperature + longName: "Temperature" + units: "K" + + # ObsValue: dewPointTemperature + - name: "ObsValue/dewPointTemperature" + coordinates: "longitude latitude" + source: variables/dewPointTemperature + longName: "DewPoint Temperature" + units: "K" + + # ObsValue: virtualTemperature + - name: "ObsValue/virtualTemperature" + coordinates: "longitude latitude" + source: variables/virtualTemperature + longName: "Virtual Temperature Non-Q Controlled" + units: "K" + + # ObsValue: windEastward + - name: "ObsValue/windEastward" + coordinates: "longitude latitude" + source: variables/windEastward + longName: "Eastward Wind" + units: "m s-1" + + # ObsValue: windNorthward + - name: "ObsValue/windNorthward" + coordinates: "longitude latitude" + source: variables/windNorthward + longName: "Northward Wind" + units: "m s-1" + + # ObsValue: specificHumidity + - name: "ObsValue/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidity + longName: "Specific Humidity" + units: "kg kg-1" + + # QualityMarker: pressure + - name: "QualityMarker/pressure" + coordinates: "longitude latitude" + source: variables/pressureQualityMarker + longName: "Pressure Quality Marker" + + # QualityMarker: height + - name: "QualityMarker/height" + coordinates: "longitude latitude" + source: variables/heightQualityMarker + longName: "Height Quality Marker" + + # QualityMarker: stationPressure + - name: "QualityMarker/stationPressure" + coordinates: "longitude latitude" + source: variables/stationPressureQualityMarker + longName: "Station Pressure Quality Marker" + + # QualityMarker: airTemperature + - name: "QualityMarker/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureQualityMarker + longName: "Temperature Quality Marker" + + # QualityMarker: virtualTemperature + - name: "QualityMarker/virtualTemperature" + coordinates: "longitude latitude" + source: variables/virtualTemperatureQualityMarker + longName: "Virtual Temperature Quality Marker" + + # QualityMarker: windEastward + - name: "QualityMarker/windEastward" + coordinates: "longitude latitude" + source: variables/windQualityMarker + longName: "Eastward Wind Quality Marker" + + # QualityMarker: windNorthward + - name: "QualityMarker/windNorthward" + coordinates: "longitude latitude" + source: variables/windQualityMarker + longName: "Northward Wind Quality Marker" + + # QualityMarker: specificHumidity + - name: "QualityMarker/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidityQualityMarker + longName: "Specific Humidity Quality Marker" + + # ObsError: pressure + - name: "ObsError/pressure" + coordinates: "longitude latitude" + source: variables/pressureError + longName: "Pressure Error" + units: "Pa" + + # ObsError: stationPressure + - name: "ObsError/stationPressure" + coordinates: "longitude latitude" + source: variables/stationPressureError + longName: "Station Pressure Error" + units: "Pa" + + # ObsError: airTemperature + - name: "ObsError/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureError + longName: "Temperature Error" + units: "K" + + # ObsError: virtualTemperature + - name: "ObsError/virtualTemperature" + coordinates: "longitude latitude" + source: variables/virtualTemperatureError + longName: "Virtual Temperature Error" + units: "K" + + # ObsError: windEastward + - name: "ObsError/windEastward" + coordinates: "longitude latitude" + source: variables/windError + longName: "East Wind Error" + units: "m s-1" + + # ObsError: windNorthward + - name: "ObsError/windNorthward" + coordinates: "longitude latitude" + source: variables/windError + longName: "North Wind Error" + units: "m s-1" + + # ObsError: specificHumidity + - name: "ObsError/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidityError + longName: "Specific Humidity Error" diff --git a/dump/mapping/make_files_please.txt b/dump/mapping/make_files_please.txt new file mode 100644 index 0000000..e69de29