DQM output format and parsing script (#147)

* Add script file to parse a DQM FITS file and fill the ZODB with it.

* Automatically fetch DQM output archive from DIRAC in script

* Extract tar gz archive automatically

* Changing the format of the DQM output FITS files

* Lint

* Do not display PixelTimeline objects

* Improve script with argument parser, to provide a run number to process.

* Add exceptions, remove DQM results at end of script

* Streamline the WriteAllResults method

* Add exception handling

* Found a way to include TRIGGER-STATISTICS in output FITS file

* Change Bokeh app to reflect change in output format for DQM FITS files

* Streamline list of processors in DQM

* Don't test for pattern not to display in parent key, but in child key of DQM dict

* Add START-TIMES in rejected list of displays

* Adapt Bokeh app to first find the run id with the most displays before starting

* Print caught errors on update

* Reset displays on update

* Changed caught error message when filling HDU for output DQM FITS file

* Handle further type of exceptions for pedestal values

* No camera display for integrated pedestal values

---------

Co-authored-by: Jean-Philippe Lenain <[email protected]>
Co-authored-by: Halim Ashkar <[email protected]>

src/nectarchain/dqm/bokeh_app/app_hooks.py

@@ -1,4 +1,5 @@
 import collections
+import re
 
 import numpy as np
 from ctapipe.coordinates import EngineeringCameraFrame
@@ -9,11 +10,13 @@
 from ctapipe_io_nectarcam import constants
 
 NOTINDISPLAY = [
-    "Results_TriggerStatistics",
-    "Results_MeanWaveForms_HighGain",
-    "Results_MeanWaveForms_LowGain",
-    "Results_CameraMonitoring",
+    "TRIGGER-.*",
+    "PED-INTEGRATION-.*",
+    "START-TIMES",
+    "WF-.*",
+    ".*PixTimeline-.*",
 ]
+TEST_PATTERN = "(?:% s)" % "|".join(NOTINDISPLAY)
 
 geom = CameraGeometry.from_name("NectarCam-003")
 geom = geom.transform_to(EngineeringCameraFrame())
@@ -27,8 +30,8 @@ def get_rundata(src, runid):
 def make_camera_displays(db, source, runid):
     displays = collections.defaultdict(dict)
     for parentkey in db[runid].keys():
-        if parentkey not in NOTINDISPLAY:
-            for childkey in db[runid][parentkey].keys():
+        for childkey in db[runid][parentkey].keys():
+            if not re.match(TEST_PATTERN, childkey):
                 print(f"Run id {runid} Preparing plot for {parentkey}, {childkey}")
                 displays[parentkey][childkey] = make_camera_display(
                     source, parent_key=parentkey, child_key=childkey

src/nectarchain/dqm/bokeh_app/main.py

@@ -1,4 +1,7 @@
+import re
+
 import numpy as np
+from app_hooks import TEST_PATTERN, get_rundata, make_camera_displays
 
 # bokeh imports
 from bokeh.layouts import layout, row
@@ -12,15 +15,6 @@
 
 from nectarchain.dqm.db_utils import DQMDB
 
-from .app_hooks import get_rundata, make_camera_displays
-
-NOTINDISPLAY = [
-    "Results_TriggerStatistics",
-    "Results_MeanWaveForms_HighGain",
-    "Results_MeanWaveForms_LowGain",
-    "Results_CameraMonitoring",
-]
-
 geom = CameraGeometry.from_name("NectarCam-003")
 geom = geom.transform_to(EngineeringCameraFrame())
 
@@ -29,19 +23,32 @@ def update_camera_displays(attr, old, new):
     runid = run_select.value
     new_rundata = get_rundata(db, runid)
 
+    # Reset each display
+    for k in displays.keys():
+        for kk in displays[k].keys():
+            displays[k][kk].image = np.zeros(shape=constants.N_PIXELS)
+
     for parentkey in db[runid].keys():
-        if parentkey not in NOTINDISPLAY:
-            for childkey in db[runid][parentkey].keys():
+        for childkey in db[runid][parentkey].keys():
+            if not re.match(TEST_PATTERN, childkey):
                 print(f"Run id {runid} Updating plot for {parentkey}, {childkey}")
-                # try:
+
                 image = new_rundata[parentkey][childkey]
                 image = np.nan_to_num(image, nan=0.0)
                 try:
                     displays[parentkey][childkey].image = image
-                except ValueError:
+                except ValueError as e:
+                    print(
+                        f"Caught {type(e).__name__} for {childkey}, filling display"
+                        f"with zeros. Details: {e}"
+                    )
                     image = np.zeros(shape=displays[parentkey][childkey].image.shape)
                     displays[parentkey][childkey].image = image
-                except KeyError:
+                except KeyError as e:
+                    print(
+                        f"Caught {type(e).__name__} for {childkey}, filling display"
+                        f"with zeros. Details: {e}"
+                    )
                     image = np.zeros(shape=constants.N_PIXELS)
                     displays[parentkey][childkey].image = image
                 # TODO: TRY TO USE `stream` INSTEAD, ON UPDATES:
@@ -51,7 +58,20 @@ def update_camera_displays(attr, old, new):
 
 db = DQMDB(read_only=True).root
 runids = sorted(list(db.keys()))
-runid = runids[-1]
+
+# First, get the run id with the most populated result dictionary
+runid_max = runids[-1]
+largest = 0
+for runid in runids:
+    larger = 0
+    for k in db[runid].keys():
+        length = len(db[runid][k])
+        if length > larger:
+            larger = length
+    if larger > largest:
+        largest = larger
+        runid_max = runid
+runid = runid_max
 
 # runid_input = NumericInput(value=db.root.keys()[-1], title="NectarCAM run number")
 run_select = Select(value=runid, title="NectarCAM run number", options=runids)

src/nectarchain/dqm/dqm_summary_processor.py

@@ -1,3 +1,4 @@
+import numpy as np
 from astropy.io import fits
 from astropy.table import Table
 
@@ -32,58 +33,35 @@ def PlotResults(
     ):
         print("Processor 5")
 
-    def WriteAllResults(self, path, DICT):
-        data2 = Table()
-        data1 = Table()
-        data0 = Table()
+    @staticmethod
+    def _create_hdu(name, content):
         data = Table()
-        hdu, hdu0, hdu1, hdu2 = None, None, None, None
+        try:
+            data[name] = content
+        except TypeError:
+            try:
+                data = Table(content)
+            except ValueError:
+                # We may have caught just a single float value, try to pack it into
+                # the FITS output
+                content = np.array([content])
+                data = Table(content)
+        hdu = fits.BinTableHDU(data)
+        hdu.name = name
+        return hdu
+
+    def WriteAllResults(self, path, DICT):
         hdulist = fits.HDUList()
         for i, j in DICT.items():
-            if (i == "Results_TriggerStatistics"):
-                for n2, m2 in j.items():
-                    data2[n2] = m2
-                hdu2 = fits.BinTableHDU(data2)
-                hdu2.name = "Trigger"
-
-            elif (i == "Results_MeanWaveForms_HighGain") or (
-                i == "Results_MeanWaveForms_LowGain"
-            ):
-                for n1, m1 in j.items():
-                    data1[n1] = m1
-                hdu1 = fits.BinTableHDU(data1)
-                hdu1.name = "MWF"    
-
-            elif (i == "Results_PixelTimeline_HighGain") or (i == "Results_PixelTimeline_LowGain"):
-                for n0, m0 in j.items():
-                    data0[n0] = m0 
-                hdu0 = fits.BinTableHDU(data0)
-                hdu0.name = "BPX"
-
-            else:
-                for n, m in j.items():
-                    data[n] = m
-                hdu = fits.BinTableHDU(data)
-                hdu.name = "Camera"
-        if hdu2:
-            hdulist.append(hdu2)
-        else:
-            print("No trigger statistics requests")
-        if hdu1:
-            hdulist.append(hdu1)
-        else:
-            print("No MWF studies requests")
-        if hdu0:
-            hdulist.append(hdu0)
-        else:
-            print("No Pixel Timeline studies requests")
-        if hdu:
-            hdulist.append(hdu)
-        else:
-            print("No Camera studies requests")
+            for name, content in j.items():
+                try:
+                    hdu = self._create_hdu(name, content)
+                    hdulist.append(hdu)
+                except TypeError as e:
+                    print(f"Caught {type(e).__name__}, skipping {name}. Details: {e}")
+                    pass
 
-
-        FileName = path + '_Results.fits'
+        FileName = path + "_Results.fits"
         print(FileName)
         hdulist.writeto(FileName, overwrite=True)
-        return None
+        hdulist.info()

src/nectarchain/dqm/start_dqm.py

@@ -70,8 +70,6 @@ def main():
     output_path = args.output_paths
     print("Output path:", output_path)
 
-    # Defining and printing the paths of the input files.
-
     if args.runnb is not None:
         # Grab runs automatically from DIRAC is the -r option is provided
         from nectarchain.data.management import DataManagement
@@ -152,34 +150,22 @@ def CreateFigFolder(name, type):
 
     # LIST OF PROCESSES TO RUN
     ####################################################################################
-    a = TriggerStatistics(HIGH_GAIN)
-    b = MeanWaveFormsHighLowGain(HIGH_GAIN)
-    c = MeanWaveFormsHighLowGain(LOW_GAIN)
-    d = MeanCameraDisplayHighLowGain(HIGH_GAIN)
-    e = MeanCameraDisplayHighLowGain(LOW_GAIN)
-    f = ChargeIntegrationHighLowGain(HIGH_GAIN)
-    g = ChargeIntegrationHighLowGain(LOW_GAIN)
-    h = CameraMonitoring(HIGH_GAIN)
-    i = PixelParticipationHighLowGain(HIGH_GAIN)
-    j = PixelParticipationHighLowGain(LOW_GAIN)
-    k = PixelTimelineHighLowGain(HIGH_GAIN)
-    ll = PixelTimelineHighLowGain(LOW_GAIN)
-
-    processors = list()
-
-    processors.append(a)
-    processors.append(b)
-    processors.append(c)
-    processors.append(d)
-    processors.append(e)
-    processors.append(f)
-    processors.append(g)
-    processors.append(h)
-    processors.append(i)
-    processors.append(j)
-    processors.append(k)
-    processors.append(ll)
+    processors = [
+        TriggerStatistics(HIGH_GAIN),
+        MeanWaveFormsHighLowGain(HIGH_GAIN),
+        MeanWaveFormsHighLowGain(LOW_GAIN),
+        MeanCameraDisplayHighLowGain(HIGH_GAIN),
+        MeanCameraDisplayHighLowGain(LOW_GAIN),
+        ChargeIntegrationHighLowGain(HIGH_GAIN),
+        ChargeIntegrationHighLowGain(LOW_GAIN),
+        CameraMonitoring(HIGH_GAIN),
+        PixelParticipationHighLowGain(HIGH_GAIN),
+        PixelParticipationHighLowGain(LOW_GAIN),
+        PixelTimelineHighLowGain(HIGH_GAIN),
+        PixelTimelineHighLowGain(LOW_GAIN),
+    ]
 
+    # LIST OF DICT RESULTS
     NESTED_DICT = {}  # The final results dictionary
 
     NESTED_DICT_KEYS = [
@@ -197,8 +183,6 @@ def CreateFigFolder(name, type):
         "Results_PixelTimeline_LowGain",
     ]
 
-    # NESTED_DICT_KEYS = ["Results_PixelParticipation_HighGain"]
-
     # START
     for p in processors:
         Pix, Samp = p.DefineForRun(reader1)
@@ -257,7 +241,7 @@ def CreateFigFolder(name, type):
                 plt.close()
 
     end = time.time()
-    print("Processing time:", end - start)
+    print(f"Processing time: {end-start:.2f} s.")
 
     # TODO
     # Reduce code by using loops: for figs and results

src/nectarchain/dqm/trigger_statistics.py

@@ -90,22 +90,18 @@ def FinishRun(self):
 
     def GetResults(self):
         self.TriggerStat_Results_Dict["TRIGGER-TYPES"] = self.triggers
-        self.TriggerStat_Results_Dict[
-            "TRIGGER-STATISTICS"
-        ] = "All: %s, Physical: %s, Pedestals: %s, Others: %s, Wrong times: %s" % (
-            len(self.event_times),
-            len(self.event_phy_times),
-            len(self.event_ped_times),
-            len(self.event_other_times),
-            len(self.event_wrong_times),
-        )
-        self.TriggerStat_Results_Dict[
-            "START-TIMES"
-        ] = "Run start time: %s, First event: %s, Last event: %s" % (
-            self.run_start1,
-            self.run_start,
-            self.run_end,
-        )
+        self.TriggerStat_Results_Dict["TRIGGER-STATISTICS"] = {
+            "All": [len(self.event_times)],
+            "Physical": [len(self.event_phy_times)],
+            "Pedestals": [len(self.event_ped_times)],
+            "Others": [len(self.event_other_times)],
+            "Wrong times": [len(self.event_wrong_times)],
+        }
+        self.TriggerStat_Results_Dict["START-TIMES"] = {
+            "Run start time": self.run_start1,
+            "First event": self.run_start,
+            "Last event": self.run_end,
+        }
         return self.TriggerStat_Results_Dict
 
     def PlotResults(self, name, FigPath):