Merge pull request #171 from ARamsay17/master

Factory branch processing for calibration less than 256 pixels

Source/ProcessInstrumentUncertainties.py

@@ -115,9 +115,12 @@ def Factory(self, node: HDFRoot, uncGrp: HDFGroup, stats: dict) -> dict[np.array
             radcal = uncGrp.getDataset(f"{sensor}_RADCAL_UNC") # SIRREX data
             radcal.datasetToColumns()
 
+            # added attribute in node for Factory branch which accounts for missing calibration data.
+            cal_start = int(node.attributes['CAL_START'])
+            cal_stop = int(node.attributes['CAL_STOP'])
             straylight = uncGrp.getDataset(f"{sensor}_STRAYDATA_CAL")
             straylight.datasetToColumns()
-            cStray[sensor] = np.asarray(list(straylight.columns['1']))
+            cStray[sensor] = np.asarray(list(straylight.columns['1']))[cal_start:cal_stop]
 
             linear = uncGrp.getDataset(sensor + "_NLDATA_CAL")
             linear.datasetToColumns()
@@ -998,9 +1001,11 @@ def rrsHyperUNCFACTORY(self, node, uncGrp, rhoScalar, rhoVec, rhoDelta, waveSubs
             radcal = uncGrp.getDataset(f"{sensor}_RADCAL_UNC")
             radcal.datasetToColumns()
 
+            cal_start = int(node.attributes['CAL_START'])
+            cal_stop = int(node.attributes['CAL_STOP'])
             straylight = uncGrp.getDataset(f"{sensor}_STRAYDATA_CAL")
             straylight.datasetToColumns()
-            cStray[sensor] = np.asarray(list(straylight.columns['1']))
+            cStray[sensor] = np.asarray(list(straylight.columns['1']))[cal_start:cal_stop]
 
             linear = uncGrp.getDataset(sensor + "_NLDATA_CAL")
             linear.datasetToColumns()

Source/ProcessL1b_FactoryCal.py

@@ -171,6 +171,29 @@ def processGroup(gp, cf): # group, calibration file
                 ds = gp.getDataset(cd.type)
                 ProcessL1b_FactoryCal.processDataset(ds, cd, inttime)
 
+    @staticmethod
+    def get_cal_file_lines(calibrationMap):
+        """
+        function to recover effective calibration start and stop pixel from cal files.
+        """
+        coefs = {}
+        indx = {}
+        for k, var in calibrationMap.items():
+            coefs[k] = []
+            for d in var.data:
+                if d.type == 'ES' or d.type == 'LI' or d.type == 'LT':
+                    coefs[k].append(d.coefficients)
+
+            indx[k] = []
+            for i, c in enumerate(coefs[k]):
+                if len(c) > 0:
+                    indx[k].append(i)
+
+        # todo: assess if this is stricly necessary, all indexes the same in examples used for testing
+        start = max([ind[0] for ind in indx.values()]) - 1  # -1 to cover the first pixel which has no coef but is valid
+        end = min([ind[-1] for ind in indx.values()])
+        return start, end
+
     @staticmethod
     def processL1b_SeaBird(node, calibrationMap):
         '''
@@ -183,6 +206,11 @@ def processL1b_SeaBird(node, calibrationMap):
         pyrUnits = None
 
         now = dt.datetime.now()
+        # get effective calibration and save to node attributes
+        start, end = ProcessL1b_FactoryCal.get_cal_file_lines(calibrationMap)
+        node.attributes['CAL_START'] = str(start)
+        node.attributes['CAL_STOP'] = str(end)
+        # node.attributes['CAL_LINES'] = lines
         timestr = now.strftime("%d-%b-%Y %H:%M:%S")
         node.attributes["FILE_CREATION_TIME"] = timestr
         msg = f"ProcessL1b_FactoryCal.processL1b: {timestr}"

Source/ProcessL2.py

@@ -404,7 +404,7 @@ def spectralReflectance(node, sensor, timeObj, xSlice, F0, F0_unc, rhoScalar, rh
                     rrsUNC[k] = 0
 
         deleteKey = []
-        for wvl in waveSubset:  # loop through wavebands
+        for i, wvl in enumerate(waveSubset):  # loop through wavebands
             k = str(wvl)
             if (any([wvl == float(x) for x in esXSlice]) and
                     any([wvl == float(x) for x in liXSlice]) and
@@ -524,7 +524,12 @@ def spectralReflectance(node, sensor, timeObj, xSlice, F0, F0_unc, rhoScalar, rh
                         else:
                             newRhoHyper.columns[k].append(rhoScalar)
                             if xUNC is not None:  # perhaps there is a better check for TriOS Factory branch?
-                                newRhoUNCHyper.columns[k].append(xUNC[f'rhoUNC_{sensor}'][k])
+                                try:
+                                    # todo: explore why rho UNC is 1 index smaller than everything else
+                                    # last wvl is missing
+                                    newRhoUNCHyper.columns[k].append(xUNC[f'rhoUNC_{sensor}'][k])
+                                except KeyError:
+                                    newRhoUNCHyper.columns[k].append(0)
                             else:
                                 newRhoUNCHyper.columns[k].append(np.nan)
                 else: