diff --git a/dump/mapping/iodatest_prepbufr_acft_profiles_encoder.py b/dump/mapping/iodatest_prepbufr_acft_profiles_encoder.py new file mode 100755 index 0000000..c6a1af6 --- /dev/null +++ b/dump/mapping/iodatest_prepbufr_acft_profiles_encoder.py @@ -0,0 +1,126 @@ +import os +import sys +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/bufr_query/build/lib/python3.10') +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10') +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyioda') +sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyiodaconv') +import bufr +from pyioda.ioda.Engines.Bufr import Encoder +import copy +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +from datetime import datetime + + +def Compute_dateTime(cycleTimeSinceEpoch, dhr): + + int64_fill_value = np.int64(0) + + dateTime = np.zeros(dhr.shape, dtype=np.int64) + for i in range(len(dateTime)): + if ma.is_masked(dhr[i]): + continue + else: + dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch + + dateTime = ma.array(dateTime) + dateTime = ma.masked_values(dateTime, int64_fill_value) + + return dateTime + + +def Compute_typ_other(typ, var): + + typ_var = copy.deepcopy(typ) + typ_var[(typ_var > 300) & (typ_var < 400)] -= 200 + typ_var[(typ_var > 400) & (typ_var < 500)] -= 300 + typ_var[(typ_var > 500) & (typ_var < 600)] -= 400 + + for i in range(len(typ_var)): + if ma.is_masked(var[i]): + typ_var[i] = typ_var.fill_value + + return typ_var + + +def Compute_typ_uv(typ, var): + + typ_var = copy.deepcopy(typ) + typ_var[(typ_var > 300) & (typ_var < 400)] -= 100 + typ_var[(typ_var > 400) & (typ_var < 500)] -= 200 + typ_var[(typ_var > 500) & (typ_var < 600)] -= 300 + + for i in range(len(typ_var)): + if ma.is_masked(var[i]): + typ_var[i] = typ_var.fill_value + + return typ_var + + +def Compute_ialr_if_masked(typ, ialr): + + ialr_bc = copy.deepcopy(ialr) + for i in range(len(ialr_bc)): + if ma.is_masked(ialr_bc[i]) and (typ[i] >= 330) and (typ[i] < 340): + ialr_bc[i] = float(0) + + return ialr_bc + + +def create_obs_group(cycle_time,input_mapping,input_path): + CYCLE_TIME = round(cycle_time) + YAML_PATH = input_mapping #"./iodatest_prepbufr_acft_profiles_mapping.yaml" + INPUT_PATH = input_path + + container = bufr.Parser(INPUT_PATH, YAML_PATH).parse() + + print(" Do DateTime calculation") + otmct = container.get('variables/obsTimeMinusCycleTime') + otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime') + otmct2 = np.array(otmct) + cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(CYCLE_TIME), '%Y%m%d%H'))) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2) + + container.add('variables/dateTime', dateTime, otmct_paths) + + print(" Do ObsType calculation") + ot = container.get('variables/observationType') + ot_paths = container.get_paths('variables/observationType') + + airTemperature = container.get('variables/airTemperatureObsValue') + #virtualTemperature = container.get('variables/virtualTemperatureObsValue') + specificHumidity = container.get('variables/specificHumidityObsValue') + wind = container.get('variables/windNorthwardObsValue') + + ot_airTemperature = Compute_typ_other(ot, airTemperature) + #ot_virtualTemperature = Compute_typ_other(ot, virtualTemperature) + ot_specificHumidity = Compute_typ_other(ot, specificHumidity) + ot_wind = Compute_typ_uv(ot, wind) + + print(" Change IALR to 0.0 if masked for bias correction.") + ialr = container.get('variables/instantaneousAltitudeRate0') + ialr_paths = container.get_paths('variables/instantaneousAltitudeRate0') + ialr2 = ma.array(ialr) + + ialr_bc = Compute_ialr_if_masked(ot, ialr2) + + print("Make an array of 0s for MetaData/sequenceNumber") + sequenceNum = np.zeros(ot.shape, dtype=np.int32) + + print(" Add new variables to container") + container.add('variables/airTemperatureObservationType', ot_airTemperature, ot_paths) + #container.add('variables/virtualTemperatureObservationType', ot_virtualTemperature, ot_paths) + container.add('variables/specificHumidityObservationType', ot_specificHumidity, ot_paths) + container.add('variables/windObservationType', ot_wind, ot_paths) + container.add('variables/instantaneousAltitudeRate', ialr_bc, ot_paths) #ialr_paths) + container.add('variables/sequenceNumber', sequenceNum, ot_paths) + + description = bufr.encoders.Description(YAML_PATH) + + print(" Add container and descriptions to dataset.") + dataset = next(iter(Encoder(description).encode(container).values())) + return dataset + diff --git a/dump/mapping/iodatest_prepbufr_acft_profiles_mapping.yaml b/dump/mapping/iodatest_prepbufr_acft_profiles_mapping.yaml new file mode 100755 index 0000000..89d0696 --- /dev/null +++ b/dump/mapping/iodatest_prepbufr_acft_profiles_mapping.yaml @@ -0,0 +1,264 @@ +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +bufr: + group_by_variable: prepbufrDataLevelCategory + variables: + # ObsType + observationType: + query: "*/TYP" + + # MetaData + prepbufrDataLevelCategory: + query: "*/PRSLEVLA/CAT" + stationIdentification: + query: "*/SID" + aircraftFlightNumber: + query: "[*/ACID, */ACID_SEQ/ACID]" + longitude: + query: "*/PRSLEVLA/DRFTINFO/XDR" + latitude: + query: "*/PRSLEVLA/DRFTINFO/YDR" + obsTimeMinusCycleTime: + query: "*/PRSLEVLA/DRFTINFO/HRDR" + heightOfStationMetaData: #Formerly aircraftFlightLevel + query: "*/PRSLEVLA/Z___INFO/Z__EVENT{1}/ZOB" + stationElevationMetaData: + query: "*/ELV" + type: float + pressureMetaData: + query: "*/PRSLEVLA/P___INFO/P__EVENT{1}/POB" + transforms: + - scale: 100 + temperatureEventCode: + query: "*/PRSLEVLA/T___INFO/T__EVENT{1}/TPC" + instantaneousAltitudeRate0: + query: "*/PRSLEVLA/IALR" + aircraftFlightPhase: + query: "*/PRSLEVLA/ACFT_SEQ{1}/POAF" + + #ObsValue + airTemperatureObsValue: + query: "*/PRSLEVLA/T___INFO/T__EVENT{1}/TOB" + transforms: + - offset: 273.15 +# virtualTemperatureObsValue: +# query: "*/PRSLEVLA/T___INFO/TVO" +# transforms: +# - offset: 273.15 + specificHumidityObsValue: + query: "*/PRSLEVLA/Q___INFO/Q__EVENT{1}/QOB" + type: float + transforms: + - scale: 0.000001 + windEastwardObsValue: + query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/UOB" + windNorthwardObsValue: + query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/VOB" + + #QualityMarker + pressureQualityMarker: + query: "*/PRSLEVLA/P___INFO/P__EVENT{1}/PQM" + airTemperatureQualityMarker: + query: "*/PRSLEVLA/T___INFO/T__EVENT{1}/TQM" + specificHumidityQualityMarker: + query: "*/PRSLEVLA/Q___INFO/Q__EVENT{1}/QQM" + windEastwardQualityMarker: + query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/WQM" + windNorthwardQualityMarker: + query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/WQM" + + #ObsError + airTemperatureObsError: + query: "*/PRSLEVLA/T___INFO/T__BACKG/TOE" + specificHumidityObsError: + query: "*/PRSLEVLA/Q___INFO/Q__BACKG/QOE" + type: float + transforms: + - scale: 0.1 + windObsError: + query: "*/PRSLEVLA/W___INFO/W__BACKG/WOE" + + +encoder: + dimensions: + - name: PressureEvent + path: "*/PRSLEVLA/P___INFO/P__EVENT" + - name: TemperatureEvent + path: "*/PRSLEVLA/T___INFO/T__EVENT" + - name: HumidityEvent + path: "*/PRSLEVLA/Q___INFO/Q__EVENT" + - name: HeightEvent + path: "*/PRSLEVLA/Z___INFO/Z__EVENT" + - name: WindEvent + path: "*/PRSLEVLA/W___INFO/W__EVENT" + + globals: + - name: "data_format" + type: string + value: "prepbufr" + + - name: "subsets" + type: string + value: "AIRCFT AIRCAR" + + - name: "source" + type: string + value: "prepBUFR" + + - name: "data_type" + type: string + value: "AIRCFT AIRCAR" + + - name: "data_description" + type: string + value: "acft_profiles_prepbufr" + + - name: "data_provider" + type: string + value: "U.S. NOAA" + + variables: + - name: "ObsType/airTemperature" + source: variables/airTemperatureObservationType + longName: "Observation Type" + +# - name: "ObsType/virtualTemperature" +# source: variables/virtualTemperatureObservationType +# longName: "Observation Type" + + - name: "ObsType/specificHumidity" + source: variables/specificHumidityObservationType + longName: "Observation Type" + + - name: "ObsType/windNorthward" + source: variables/windObservationType + longName: "Observation Type" + + - name: "ObsType/windEastward" + source: variables/windObservationType + longName: "Observation Type" + + # MetaData + - name: "MetaData/prepbufrDataLevelCategory" + source: variables/prepbufrDataLevelCategory + longName: "Prepbufr Data Level Category" + + - name: "MetaData/stationIdentification" + source: variables/stationIdentification + longName: "Station ID" + + - name: "MetaData/aircraftFlightNumber" + source: variables/aircraftFlightNumber + longName: "Aircraft Flight Number" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [0, 360] + + - name: "MetaData/dateTime" + source: variables/dateTime + units: 'seconds since 1970-01-01T00:00:00Z' + longName: "dateTime" + + - name: "MetaData/pressure" + source: variables/pressureMetaData + longName: "Pressure" + units: "Pa" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStationMetaData + longName: "Height Of Station" + units: "m" + + - name: "MetaData/stationElevation" + source: variables/stationElevationMetaData + longName: "Station Elevation" + units: "m" + + - name: "MetaData/aircraftFlightPhase" + source: variables/aircraftFlightPhase + longName: "Aircraft Flight Phase" + + - name: "MetaData/instantaneousAltitudeRate" + source: variables/instantaneousAltitudeRate + longName: "Instantaneous Rate Altitue" + units: "m s-1" + + - name: "MetaData/sequenceNumber" + source: variables/sequenceNumber + longName: "Sequence Number" + + #ObsValue + - name: "ObsValue/airTemperature" + source: variables/airTemperatureObsValue + longName: "Temperature" + units: "K" + +# - name: "ObsValue/virtualTemperature" +# source: variables/virtualTemperatureObsValue +# longName: "Virtual Temperature" +# units: "K" + + - name: "ObsValue/specificHumidity" + source: variables/specificHumidityObsValue + longName: "Specific Humidity" + units: "kg kg-1" + + - name: "ObsValue/windEastward" + source: variables/windEastwardObsValue + longName: "U component of Wind" + units: "m s-1" + + - name: "ObsValue/windNorthward" + source: variables/windNorthwardObsValue + longName: "V component of Wind" + units: "m s-1" + + #QualityMarker + - name: "QualityMarker/airTemperature" + source: variables/airTemperatureQualityMarker + longName: "Temperature Quality Marker" + + - name: "QualityMarker/specificHumidity" + source: variables/specificHumidityQualityMarker + longName: "specific Humidity Quality Marker" + + - name: "QualityMarker/windEastward" + source: variables/windEastwardQualityMarker + longName: "U Component of Wind Quality Marker" + + - name: "QualityMarker/windNorthward" + source: variables/windNorthwardQualityMarker + longName: "V Component of Wind Quality Marker" + + #ObsError + - name: "ObsError/airTemperature" + source: variables/airTemperatureObsError + longName: "Temperature Error" + units: "K" + + - name: "ObsError/specificHumidity" + source: variables/specificHumidityObsError + longName: "Specific Humidity Error" + units: "1" + + - name: "ObsError/windEastward" + source: variables/windObsError + longName: "U component of wind error" + units: "m s-1" + + - name: "ObsError/windNorthward" + source: variables/windObsError + longName: "V component of wind error" + units: "m s-1" diff --git a/dump/mapping/make_files_please.txt b/dump/mapping/make_files_please.txt new file mode 100644 index 0000000..e69de29