diff --git a/Source/ProcessInstrumentUncertainties.py b/Source/ProcessInstrumentUncertainties.py index be79016..556b983 100644 --- a/Source/ProcessInstrumentUncertainties.py +++ b/Source/ProcessInstrumentUncertainties.py @@ -115,9 +115,12 @@ def Factory(self, node: HDFRoot, uncGrp: HDFGroup, stats: dict) -> dict[np.array radcal = uncGrp.getDataset(f"{sensor}_RADCAL_UNC") # SIRREX data radcal.datasetToColumns() + # added attribute in node for Factory branch which accounts for missing calibration data. + cal_start = int(node.attributes['CAL_START']) + cal_stop = int(node.attributes['CAL_STOP']) straylight = uncGrp.getDataset(f"{sensor}_STRAYDATA_CAL") straylight.datasetToColumns() - cStray[sensor] = np.asarray(list(straylight.columns['1'])) + cStray[sensor] = np.asarray(list(straylight.columns['1']))[cal_start:cal_stop] linear = uncGrp.getDataset(sensor + "_NLDATA_CAL") linear.datasetToColumns() @@ -998,9 +1001,11 @@ def rrsHyperUNCFACTORY(self, node, uncGrp, rhoScalar, rhoVec, rhoDelta, waveSubs radcal = uncGrp.getDataset(f"{sensor}_RADCAL_UNC") radcal.datasetToColumns() + cal_start = int(node.attributes['CAL_START']) + cal_stop = int(node.attributes['CAL_STOP']) straylight = uncGrp.getDataset(f"{sensor}_STRAYDATA_CAL") straylight.datasetToColumns() - cStray[sensor] = np.asarray(list(straylight.columns['1'])) + cStray[sensor] = np.asarray(list(straylight.columns['1']))[cal_start:cal_stop] linear = uncGrp.getDataset(sensor + "_NLDATA_CAL") linear.datasetToColumns() diff --git a/Source/ProcessL1b_FactoryCal.py b/Source/ProcessL1b_FactoryCal.py index b9f9056..a6b5785 100644 --- a/Source/ProcessL1b_FactoryCal.py +++ b/Source/ProcessL1b_FactoryCal.py @@ -171,6 +171,29 @@ def processGroup(gp, cf): # group, calibration file ds = gp.getDataset(cd.type) ProcessL1b_FactoryCal.processDataset(ds, cd, inttime) + @staticmethod + def get_cal_file_lines(calibrationMap): + """ + function to recover effective calibration start and stop pixel from cal files. + """ + coefs = {} + indx = {} + for k, var in calibrationMap.items(): + coefs[k] = [] + for d in var.data: + if d.type == 'ES' or d.type == 'LI' or d.type == 'LT': + coefs[k].append(d.coefficients) + + indx[k] = [] + for i, c in enumerate(coefs[k]): + if len(c) > 0: + indx[k].append(i) + + # todo: assess if this is stricly necessary, all indexes the same in examples used for testing + start = max([ind[0] for ind in indx.values()]) - 1 # -1 to cover the first pixel which has no coef but is valid + end = min([ind[-1] for ind in indx.values()]) + return start, end + @staticmethod def processL1b_SeaBird(node, calibrationMap): ''' @@ -183,6 +206,11 @@ def processL1b_SeaBird(node, calibrationMap): pyrUnits = None now = dt.datetime.now() + # get effective calibration and save to node attributes + start, end = ProcessL1b_FactoryCal.get_cal_file_lines(calibrationMap) + node.attributes['CAL_START'] = str(start) + node.attributes['CAL_STOP'] = str(end) + # node.attributes['CAL_LINES'] = lines timestr = now.strftime("%d-%b-%Y %H:%M:%S") node.attributes["FILE_CREATION_TIME"] = timestr msg = f"ProcessL1b_FactoryCal.processL1b: {timestr}" diff --git a/Source/ProcessL2.py b/Source/ProcessL2.py index bc7cf08..9db5c90 100644 --- a/Source/ProcessL2.py +++ b/Source/ProcessL2.py @@ -404,7 +404,7 @@ def spectralReflectance(node, sensor, timeObj, xSlice, F0, F0_unc, rhoScalar, rh rrsUNC[k] = 0 deleteKey = [] - for wvl in waveSubset: # loop through wavebands + for i, wvl in enumerate(waveSubset): # loop through wavebands k = str(wvl) if (any([wvl == float(x) for x in esXSlice]) and any([wvl == float(x) for x in liXSlice]) and @@ -524,7 +524,12 @@ def spectralReflectance(node, sensor, timeObj, xSlice, F0, F0_unc, rhoScalar, rh else: newRhoHyper.columns[k].append(rhoScalar) if xUNC is not None: # perhaps there is a better check for TriOS Factory branch? - newRhoUNCHyper.columns[k].append(xUNC[f'rhoUNC_{sensor}'][k]) + try: + # todo: explore why rho UNC is 1 index smaller than everything else + # last wvl is missing + newRhoUNCHyper.columns[k].append(xUNC[f'rhoUNC_{sensor}'][k]) + except KeyError: + newRhoUNCHyper.columns[k].append(0) else: newRhoUNCHyper.columns[k].append(np.nan) else: