Feature/nicke acft profiles prepbufr

dump/mapping/iodatest_prepbufr_acft_profiles_encoder.py

@@ -0,0 +1,126 @@
+import os
+import sys
+sys.path.append('/work2/noaa/da/nesposito/backend_20240701/bufr_query/build/lib/python3.10')
+sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10')
+sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyioda')
+sys.path.append('/work2/noaa/da/nesposito/backend_20240701/ioda-bundle/build/lib/python3.10/pyiodaconv')
+import bufr
+from pyioda.ioda.Engines.Bufr import Encoder
+import copy
+import numpy as np
+import numpy.ma as ma
+import math
+import calendar
+import time
+from datetime import datetime
+
+
+def Compute_dateTime(cycleTimeSinceEpoch, dhr):
+
+    int64_fill_value = np.int64(0)
+
+    dateTime = np.zeros(dhr.shape, dtype=np.int64)
+    for i in range(len(dateTime)):
+        if ma.is_masked(dhr[i]):
+            continue
+        else:
+            dateTime[i] = np.int64(dhr[i]*3600) + cycleTimeSinceEpoch
+
+    dateTime = ma.array(dateTime)
+    dateTime = ma.masked_values(dateTime, int64_fill_value)
+
+    return dateTime
+
+
+def Compute_typ_other(typ, var):
+
+    typ_var = copy.deepcopy(typ)
+    typ_var[(typ_var > 300) & (typ_var < 400)] -= 200
+    typ_var[(typ_var > 400) & (typ_var < 500)] -= 300
+    typ_var[(typ_var > 500) & (typ_var < 600)] -= 400
+
+    for i in range(len(typ_var)):
+        if ma.is_masked(var[i]):
+            typ_var[i] = typ_var.fill_value
+
+    return typ_var
+
+
+def Compute_typ_uv(typ, var):
+
+    typ_var = copy.deepcopy(typ)
+    typ_var[(typ_var > 300) & (typ_var < 400)] -= 100
+    typ_var[(typ_var > 400) & (typ_var < 500)] -= 200
+    typ_var[(typ_var > 500) & (typ_var < 600)] -= 300
+
+    for i in range(len(typ_var)):
+        if ma.is_masked(var[i]):
+            typ_var[i] = typ_var.fill_value
+
+    return typ_var
+
+
+def Compute_ialr_if_masked(typ, ialr):
+
+    ialr_bc = copy.deepcopy(ialr)
+    for i in range(len(ialr_bc)):
+        if ma.is_masked(ialr_bc[i]) and (typ[i] >= 330) and (typ[i] < 340):
+            ialr_bc[i] = float(0)
+
+    return ialr_bc
+
+
+def create_obs_group(cycle_time,input_mapping,input_path):
+    CYCLE_TIME = round(cycle_time)
+    YAML_PATH = input_mapping #"./iodatest_prepbufr_acft_profiles_mapping.yaml"
+    INPUT_PATH = input_path
+
+    container = bufr.Parser(INPUT_PATH, YAML_PATH).parse()
+
+    print(" Do DateTime calculation")
+    otmct = container.get('variables/obsTimeMinusCycleTime')
+    otmct_paths = container.get_paths('variables/obsTimeMinusCycleTime')
+    otmct2 = np.array(otmct)
+    cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(str(CYCLE_TIME), '%Y%m%d%H')))
+    dateTime = Compute_dateTime(cycleTimeSinceEpoch, otmct2)
+
+    container.add('variables/dateTime', dateTime, otmct_paths)
+
+    print(" Do ObsType calculation")
+    ot = container.get('variables/observationType')
+    ot_paths = container.get_paths('variables/observationType')
+
+    airTemperature = container.get('variables/airTemperatureObsValue')
+    #virtualTemperature = container.get('variables/virtualTemperatureObsValue')
+    specificHumidity = container.get('variables/specificHumidityObsValue')
+    wind = container.get('variables/windNorthwardObsValue')
+
+    ot_airTemperature = Compute_typ_other(ot, airTemperature) 
+    #ot_virtualTemperature = Compute_typ_other(ot, virtualTemperature)
+    ot_specificHumidity = Compute_typ_other(ot, specificHumidity)
+    ot_wind = Compute_typ_uv(ot, wind)
+
+    print(" Change IALR to 0.0 if masked for bias correction.")
+    ialr = container.get('variables/instantaneousAltitudeRate0')
+    ialr_paths = container.get_paths('variables/instantaneousAltitudeRate0')
+    ialr2 = ma.array(ialr)
+
+    ialr_bc = Compute_ialr_if_masked(ot, ialr2)
+
+    print("Make an array of 0s for MetaData/sequenceNumber")
+    sequenceNum = np.zeros(ot.shape, dtype=np.int32)
+
+    print(" Add new variables to container")
+    container.add('variables/airTemperatureObservationType', ot_airTemperature, ot_paths)
+    #container.add('variables/virtualTemperatureObservationType', ot_virtualTemperature, ot_paths)
+    container.add('variables/specificHumidityObservationType', ot_specificHumidity, ot_paths)
+    container.add('variables/windObservationType', ot_wind, ot_paths)
+    container.add('variables/instantaneousAltitudeRate', ialr_bc, ot_paths) #ialr_paths)
+    container.add('variables/sequenceNumber', sequenceNum, ot_paths)
+
+    description = bufr.encoders.Description(YAML_PATH)
+
+    print(" Add container and descriptions to dataset.")
+    dataset = next(iter(Encoder(description).encode(container).values()))
+    return dataset
+

dump/mapping/iodatest_prepbufr_acft_profiles_mapping.yaml

@@ -0,0 +1,264 @@
+# (C) Copyright 2023 NOAA/NWS/NCEP/EMC
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+bufr:
+  group_by_variable: prepbufrDataLevelCategory
+  variables:
+    # ObsType
+    observationType:
+      query: "*/TYP"
+
+    # MetaData
+    prepbufrDataLevelCategory:
+      query: "*/PRSLEVLA/CAT"
+    stationIdentification:
+      query: "*/SID"
+    aircraftFlightNumber:
+      query: "[*/ACID, */ACID_SEQ/ACID]"
+    longitude:
+      query: "*/PRSLEVLA/DRFTINFO/XDR"
+    latitude:
+      query: "*/PRSLEVLA/DRFTINFO/YDR"
+    obsTimeMinusCycleTime:
+      query: "*/PRSLEVLA/DRFTINFO/HRDR"
+    heightOfStationMetaData: #Formerly aircraftFlightLevel
+      query: "*/PRSLEVLA/Z___INFO/Z__EVENT{1}/ZOB"
+    stationElevationMetaData:
+      query: "*/ELV"
+      type: float 
+    pressureMetaData:
+      query: "*/PRSLEVLA/P___INFO/P__EVENT{1}/POB"
+      transforms:
+        - scale: 100
+    temperatureEventCode:
+      query: "*/PRSLEVLA/T___INFO/T__EVENT{1}/TPC"
+    instantaneousAltitudeRate0:
+      query: "*/PRSLEVLA/IALR"
+    aircraftFlightPhase:
+      query: "*/PRSLEVLA/ACFT_SEQ{1}/POAF"
+
+    #ObsValue
+    airTemperatureObsValue:
+      query: "*/PRSLEVLA/T___INFO/T__EVENT{1}/TOB"
+      transforms:
+        - offset: 273.15
+#    virtualTemperatureObsValue:
+#      query: "*/PRSLEVLA/T___INFO/TVO"
+#      transforms:
+#        - offset: 273.15
+    specificHumidityObsValue:
+      query: "*/PRSLEVLA/Q___INFO/Q__EVENT{1}/QOB"
+      type: float
+      transforms:
+        - scale: 0.000001
+    windEastwardObsValue:
+      query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/UOB"
+    windNorthwardObsValue:
+      query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/VOB"
+
+    #QualityMarker
+    pressureQualityMarker:
+      query: "*/PRSLEVLA/P___INFO/P__EVENT{1}/PQM"
+    airTemperatureQualityMarker:
+      query: "*/PRSLEVLA/T___INFO/T__EVENT{1}/TQM"
+    specificHumidityQualityMarker:
+      query: "*/PRSLEVLA/Q___INFO/Q__EVENT{1}/QQM"
+    windEastwardQualityMarker:
+      query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/WQM"
+    windNorthwardQualityMarker:
+      query: "*/PRSLEVLA/W___INFO/W__EVENT{1}/WQM"
+
+    #ObsError
+    airTemperatureObsError:
+      query: "*/PRSLEVLA/T___INFO/T__BACKG/TOE"
+    specificHumidityObsError:
+      query: "*/PRSLEVLA/Q___INFO/Q__BACKG/QOE"
+      type: float
+      transforms:
+        - scale: 0.1
+    windObsError:
+      query: "*/PRSLEVLA/W___INFO/W__BACKG/WOE"
+
+
+encoder:
+  dimensions:
+    - name: PressureEvent
+      path: "*/PRSLEVLA/P___INFO/P__EVENT"
+    - name: TemperatureEvent
+      path: "*/PRSLEVLA/T___INFO/T__EVENT"
+    - name: HumidityEvent
+      path: "*/PRSLEVLA/Q___INFO/Q__EVENT"
+    - name: HeightEvent
+      path: "*/PRSLEVLA/Z___INFO/Z__EVENT"
+    - name: WindEvent
+      path: "*/PRSLEVLA/W___INFO/W__EVENT"
+
+  globals:
+    - name: "data_format"
+      type: string
+      value: "prepbufr"
+
+    - name: "subsets"
+      type: string
+      value: "AIRCFT AIRCAR"        
+
+    - name: "source"
+      type: string
+      value: "prepBUFR"
+
+    - name: "data_type"
+      type: string
+      value: "AIRCFT AIRCAR"
+
+    - name: "data_description"
+      type: string
+      value: "acft_profiles_prepbufr"
+
+    - name: "data_provider"
+      type: string
+      value: "U.S. NOAA"
+
+  variables:
+    - name: "ObsType/airTemperature"
+      source: variables/airTemperatureObservationType
+      longName: "Observation Type"
+
+#    - name: "ObsType/virtualTemperature"
+#      source: variables/virtualTemperatureObservationType
+#      longName: "Observation Type"
+
+    - name: "ObsType/specificHumidity"
+      source: variables/specificHumidityObservationType
+      longName: "Observation Type"
+
+    - name: "ObsType/windNorthward"
+      source: variables/windObservationType
+      longName: "Observation Type"
+
+    - name: "ObsType/windEastward"
+      source: variables/windObservationType
+      longName: "Observation Type"
+
+    # MetaData
+    - name: "MetaData/prepbufrDataLevelCategory"
+      source: variables/prepbufrDataLevelCategory
+      longName: "Prepbufr Data Level Category"
+
+    - name: "MetaData/stationIdentification"
+      source: variables/stationIdentification
+      longName: "Station ID"
+
+    - name: "MetaData/aircraftFlightNumber"
+      source: variables/aircraftFlightNumber
+      longName: "Aircraft Flight Number"
+
+    - name: "MetaData/latitude"
+      source: variables/latitude
+      longName: "Latitude"
+      units: "degree_north"
+      range: [-90, 90]
+
+    - name: "MetaData/longitude"
+      source: variables/longitude
+      longName: "Longitude"
+      units: "degree_east"
+      range: [0, 360]
+
+    - name: "MetaData/dateTime"
+      source: variables/dateTime
+      units: 'seconds since 1970-01-01T00:00:00Z'
+      longName: "dateTime"
+
+    - name: "MetaData/pressure"
+      source: variables/pressureMetaData
+      longName: "Pressure"
+      units: "Pa"
+
+    - name: "MetaData/heightOfStation"
+      source: variables/heightOfStationMetaData
+      longName: "Height Of Station"
+      units: "m"
+
+    - name: "MetaData/stationElevation"
+      source: variables/stationElevationMetaData
+      longName: "Station Elevation"
+      units: "m"
+
+    - name: "MetaData/aircraftFlightPhase"
+      source: variables/aircraftFlightPhase
+      longName: "Aircraft Flight Phase"
+
+    - name: "MetaData/instantaneousAltitudeRate"
+      source: variables/instantaneousAltitudeRate
+      longName: "Instantaneous Rate Altitue"
+      units: "m s-1"
+
+    - name: "MetaData/sequenceNumber"
+      source: variables/sequenceNumber
+      longName: "Sequence Number"
+
+    #ObsValue
+    - name: "ObsValue/airTemperature"
+      source: variables/airTemperatureObsValue
+      longName: "Temperature"
+      units: "K"
+
+#    - name: "ObsValue/virtualTemperature"
+#      source: variables/virtualTemperatureObsValue
+#      longName: "Virtual Temperature"
+#      units: "K"
+
+    - name: "ObsValue/specificHumidity"
+      source: variables/specificHumidityObsValue
+      longName: "Specific Humidity"
+      units: "kg kg-1"
+
+    - name: "ObsValue/windEastward"
+      source: variables/windEastwardObsValue
+      longName: "U component of Wind"
+      units: "m s-1"
+
+    - name: "ObsValue/windNorthward"
+      source: variables/windNorthwardObsValue
+      longName: "V component of Wind"
+      units: "m s-1"
+
+    #QualityMarker
+    - name: "QualityMarker/airTemperature"
+      source: variables/airTemperatureQualityMarker
+      longName: "Temperature Quality Marker"
+
+    - name: "QualityMarker/specificHumidity"
+      source: variables/specificHumidityQualityMarker
+      longName: "specific Humidity Quality Marker"
+
+    - name: "QualityMarker/windEastward"
+      source: variables/windEastwardQualityMarker
+      longName: "U Component of Wind Quality Marker"
+
+    - name: "QualityMarker/windNorthward"
+      source: variables/windNorthwardQualityMarker
+      longName: "V Component of Wind Quality Marker"
+
+    #ObsError
+    - name: "ObsError/airTemperature"
+      source: variables/airTemperatureObsError
+      longName: "Temperature Error"
+      units: "K"
+
+    - name: "ObsError/specificHumidity"
+      source: variables/specificHumidityObsError
+      longName: "Specific Humidity Error"
+      units: "1"
+
+    - name: "ObsError/windEastward"
+      source: variables/windObsError
+      longName: "U component of wind error"
+      units: "m s-1"
+
+    - name: "ObsError/windNorthward"
+      source: variables/windObsError
+      longName: "V component of wind error"
+      units: "m s-1"