Skip to content

Commit

Permalink
Adapting to L1BQC met flags over filters
Browse files Browse the repository at this point in the history
  • Loading branch information
@oceancolorcoder
oceancolorcoder committed Jul 2, 2024
1 parent 45a0ffa commit 95b671e
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 65 deletions.
2 changes: 1 addition & 1 deletion Source/ConfigFile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ def createDefaultConfig(fileName, new=1):
ConfigFile.settings["fL1bqcSpecFilterLt"] = 3

ConfigFile.settings["bL1bqcEnableQualityFlags"] = 0
ConfigFile.settings["fL1bqcCloudFlag"] = 1.0 # 1.0 basically disregards this, though cloud cover can still be used in glint correction; 0.05 Ruddick 2006, IOCCG Protocols
ConfigFile.settings["fL1bqcCloudFlag"] = 0.05 # 0.05 Ruddick 2006, IOCCG Protocols
ConfigFile.settings["fL1bqcSignificantEsFlag"] = 2.0 # Wernand 2002
ConfigFile.settings["fL1bqcDawnDuskFlag"] = 1.0 # Wernand 2002
ConfigFile.settings["fL1bqcRainfallHumidityFlag"] = 1.095 # ?? Wang? # Wernand 2002 uses Es(940/370), with >0.25 dry, 0.2-0.25 humid, <=0.25 rain
Expand Down
2 changes: 1 addition & 1 deletion Source/ConfigWindow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ def initUI(self):
self.l1bqcSpecQualityCheckPlotBoxUpdate()

# L1BQC Meteorology Flags
l1bqcQualityFlagLabel = QtWidgets.QLabel(" Enable Meteorological Filters (Experimental)", self)
l1bqcQualityFlagLabel = QtWidgets.QLabel(" Enable Meteorological Flags (Experimental/Non-exclusive)", self)
self.l1bqcQualityFlagCheckBox = QtWidgets.QCheckBox("", self)
if int(ConfigFile.settings["bL1bqcEnableQualityFlags"]) == 1:
self.l1bqcQualityFlagCheckBox.setChecked(True)
Expand Down
3 changes: 3 additions & 0 deletions Source/HDFDataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ def columnsToDataset(self):
#dtype.append((name, h5py.special_dtype(vlen=str)))
dtype.append((name, "|S" + str(len(item))))
#dtype.append((name, np.dtype(str)))
elif isinstance(item, bool):
dtype.append((name, np.bool))
# with either bool or np.bool, the dtype in the data assignment for the np.empty below is '?'
# Note: hdf4 only supports 32 bit int, convert to float64
elif isinstance(item, int):
dtype.append((name, np.float64))
Expand Down
133 changes: 70 additions & 63 deletions Source/ProcessL1bqc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def ltQuality(sasGroup):
return badTimes

@staticmethod
def metQualityCheck(refGroup, sasGroup, py6sGroup):
def metQualityCheck(refGroup, sasGroup, py6sGroup, ancGroup):
''' Perform meteorological quality control '''

esFlag = float(ConfigFile.settings["fL1bqcSignificantEsFlag"])
Expand Down Expand Up @@ -174,45 +174,47 @@ def metQualityCheck(refGroup, sasGroup, py6sGroup):
es720 = ProcessL1bqc.interpolateColumn(esColumns, 720.0)
es750 = ProcessL1bqc.interpolateColumn(esColumns, 750.0)
badTimes = []
flags1 = ancGroup.datasets['MET_FLAGS'].columns['Flag1']
flags2 = ancGroup.datasets['MET_FLAGS'].columns['Flag2']
flags3 = ancGroup.datasets['MET_FLAGS'].columns['Flag3']
flags4 = ancGroup.datasets['MET_FLAGS'].columns['Flag4']
flags5 = ancGroup.datasets['MET_FLAGS'].columns['Flag5']
for indx, dateTime in enumerate(esTime):
# Masking spectra affected by clouds (Ruddick 2006, IOCCG Protocols).
# The alternative to masking is to process them differently (e.g. See Ruddick_Rho)
# Therefore, set this very high if you don't want it triggered (e.g. 1.0, see Readme)
# Flag spectra affected by clouds (Compare with 6S Es).
if py6sGroup is not None:
if li750[indx]/es750[indx] >= cloudFLAG:
badTimes.append(dateTime)
flags1[indx] = True

# Flag spectra affected by clouds (Ruddick 2006, IOCCG Protocols).
if li750[indx]/es750[indx] >= cloudFLAG:
# msg = f"Quality Check: Li(750)/Es(750) >= cloudFLAG:{cloudFLAG}"
# print(msg)
# Utilities.writeLogFile(msg)
badTimes.append(dateTime)
flags2[indx] = True


# Threshold for significant es
# Flag for significant es
# Wernand 2002
if es480[indx] < esFlag:
# msg = f"Quality Check: es(480) < esFlag:{esFlag}"
# print(msg)
# Utilities.writeLogFile(msg)
badTimes.append(dateTime)
flags3[indx] = True

# Masking spectra affected by dawn/dusk radiation
# Flag spectra affected by dawn/dusk radiation
# Wernand 2002
#v = esXSlice["470.0"][0] / esXSlice["610.0"][0] # Fix 610 -> 680
if es470[indx]/es680[indx] < dawnDuskFlag:
# msg = f'Quality Check: ES(470.0)/ES(680.0) < dawnDuskFlag:{dawnDuskFlag}'
# print(msg)
# Utilities.writeLogFile(msg)
badTimes.append(dateTime)
flags4[indx] = True

# Masking spectra affected by rainfall and high humidity
# Flag spectra affected by rainfall and high humidity
# Wernand 2002 (940/370), Garaba et al. 2012 also uses Es(940/370), presumably 720 was developed by Wang...???
''' Follow up on the source of this flag'''
if es720[indx]/es370[indx] < humidityFlag:
# msg = f'Quality Check: ES(720.0)/ES(370.0) < humidityFlag:{humidityFlag}'
# print(msg)
# Utilities.writeLogFile(msg)
badTimes.append(dateTime)
flags5[indx] = True

badTimes = np.unique(badTimes)
badTimes = np.rot90(np.matlib.repmat(badTimes,2,1), 3) # Duplicates each element to a list of two elements in a list
msg = f'{len(np.unique(badTimes))/len(esTime)*100:.1f}% of spectra flagged'
msg = f'{len(np.unique(badTimes))/len(esTime)*100:.1f}% of spectra flagged (not filtered)'
print(msg)
Utilities.writeLogFile(msg)

Expand Down Expand Up @@ -383,6 +385,19 @@ def QC(node):
if gp.id == pyrGroup.id:
node.removeGroup(gp)

enableMetQualityCheck = ConfigFile.settings["bL1bqcEnableQualityFlags"]
if enableMetQualityCheck:
ancGroup.addDataset('MET_FLAGS')
ancGroup.datasets['MET_FLAGS'].columns['Datetag'] = ancGroup.datasets['LATITUDE'].columns['Datetag']
ancGroup.datasets['MET_FLAGS'].columns['Timetag2'] = ancGroup.datasets['LATITUDE'].columns['Timetag2']
lenAnc = len(ancGroup.datasets['MET_FLAGS'].columns['Timetag2'])
ancGroup.datasets['MET_FLAGS'].columns['Flag1'] = [False for i in range(lenAnc)]
ancGroup.datasets['MET_FLAGS'].columns['Flag2'] = [False for i in range(lenAnc)]
ancGroup.datasets['MET_FLAGS'].columns['Flag3'] = [False for i in range(lenAnc)]
ancGroup.datasets['MET_FLAGS'].columns['Flag4'] = [False for i in range(lenAnc)]
ancGroup.datasets['MET_FLAGS'].columns['Flag5'] = [False for i in range(lenAnc)]


# At this stage, all datasets in all groups of node have Timetag2
# and Datetag incorporated into data arrays. Calculate and add
# Datetime to each data array.
Expand Down Expand Up @@ -430,14 +445,6 @@ def QC(node):
Utilities.filterData(esGroup,badTimes,'L1AQC')
Utilities.filterData(liGroup,badTimes,'L1AQC')
Utilities.filterData(ltGroup,badTimes,'L1AQC')

# print('badTimes', len(badTimes))
# for dname in esGroup.datasets:
# ds = esGroup.getDataset(dname).data
# print(dname, np.shape(ds))

# # esGroup.datasets.remove('BACK_ES')
# # del esGroup.datasets['BACK_ES']

if py6sGroup is not None:
Utilities.filterData(py6sGroup,badTimes)
Expand Down Expand Up @@ -649,44 +656,44 @@ def QC(node):
if py6sGroup is not None:
Utilities.filterData(py6sGroup,badTimes)

# Next apply the Meteorological Filter prior to slicing
esData = referenceGroup.getDataset("ES")
enableMetQualityCheck = int(ConfigFile.settings["bL1bqcEnableQualityFlags"])
# Next apply the Meteorological FLAGGING prior to slicing
esData = referenceGroup.getDataset("ES")
if enableMetQualityCheck:
msg = "Applying meteorological filtering to eliminate spectra."
# msg = "Applying meteorological filtering to eliminate spectra."
msg = "Applying meteorological flags. Met flags are NOT used to eliminate spectra."
print(msg)
Utilities.writeLogFile(msg)
badTimes = ProcessL1bqc.metQualityCheck(referenceGroup, sasGroup, py6sGroup)

if badTimes is not None:
if len(badTimes) == esData.data.size:
msg = "All data flagged for deletion. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
print('Removing records...')
check = Utilities.filterData(referenceGroup, badTimes)
if check > 0.99:
msg = "Too few spectra remaining. Abort."
print(msg)
Utilities.writeLogFile(msg)
return False
Utilities.filterData(sasGroup, badTimes)
Utilities.filterData(ancGroup, badTimes)
# Filter L1AQC data for L1BQC criteria
if ConfigFile.settings['SensorType'].lower() == 'seabird':
Utilities.filterData(esDarkGroup,badTimes,'L1AQC')
Utilities.filterData(esLightGroup,badTimes,'L1AQC')
Utilities.filterData(liDarkGroup,badTimes,'L1AQC')
Utilities.filterData(liLightGroup,badTimes,'L1AQC')
Utilities.filterData(ltDarkGroup,badTimes,'L1AQC')
Utilities.filterData(ltLightGroup,badTimes,'L1AQC')
elif ConfigFile.settings['SensorType'].lower() == 'trios':
Utilities.filterData(esGroup,badTimes,'L1AQC')
Utilities.filterData(liGroup,badTimes,'L1AQC')
Utilities.filterData(ltGroup,badTimes,'L1AQC')
if py6sGroup is not None:
Utilities.filterData(py6sGroup,badTimes)
badTimes = ProcessL1bqc.metQualityCheck(referenceGroup, sasGroup, py6sGroup, ancGroup)

# if badTimes is not None:
# if len(badTimes) == esData.data.size:
# msg = "All data flagged for deletion. Abort."
# print(msg)
# Utilities.writeLogFile(msg)
# return False
# print('Removing records...')
# check = Utilities.filterData(referenceGroup, badTimes)
# if check > 0.99:
# msg = "Too few spectra remaining. Abort."
# print(msg)
# Utilities.writeLogFile(msg)
# return False
# Utilities.filterData(sasGroup, badTimes)
# Utilities.filterData(ancGroup, badTimes)
# # Filter L1AQC data for L1BQC criteria
# if ConfigFile.settings['SensorType'].lower() == 'seabird':
# Utilities.filterData(esDarkGroup,badTimes,'L1AQC')
# Utilities.filterData(esLightGroup,badTimes,'L1AQC')
# Utilities.filterData(liDarkGroup,badTimes,'L1AQC')
# Utilities.filterData(liLightGroup,badTimes,'L1AQC')
# Utilities.filterData(ltDarkGroup,badTimes,'L1AQC')
# Utilities.filterData(ltLightGroup,badTimes,'L1AQC')
# elif ConfigFile.settings['SensorType'].lower() == 'trios':
# Utilities.filterData(esGroup,badTimes,'L1AQC')
# Utilities.filterData(liGroup,badTimes,'L1AQC')
# Utilities.filterData(ltGroup,badTimes,'L1AQC')
# if py6sGroup is not None:
# Utilities.filterData(py6sGroup,badTimes)

return True

Expand Down

0 comments on commit 95b671e

Please sign in to comment.