Merge pull request #70 from cta-observatory/night_summary_parsing

Make sure dtype of night summary table is int for counters

ctapipe_io_lst/event_time.py

@@ -1,13 +1,15 @@
-from ctapipe.core import TelescopeComponent
-from ctapipe.core.traits import TelescopeParameter
-from traitlets import Enum, Int as _Int, Bool
-from astropy.time import Time
-from astropy.table import Table
-import numpy as np
 from collections import deque, defaultdict
+import numpy as np
+
 import astropy.version
+from astropy.io.ascii import convert_numpy
+from astropy.table import Table
+from astropy.time import Time, TimeUnixTai, TimeFromEpoch
 
-from astropy.time import TimeUnixTai, TimeFromEpoch
+from ctapipe.core import TelescopeComponent
+from ctapipe.core.traits import TelescopeParameter
+
+from traitlets import Enum, Int as _Int, Bool
 
 
 if astropy.version.major == 4 and astropy.version.minor <= 2 and astropy.version.bugfix <= 0:
@@ -55,6 +57,39 @@ def datetime_cols_to_time(date, time):
 
 
 def read_night_summary(path):
+    '''
+    Read a night summary file into an astropy table
+
+    Parameters
+    ----------
+    path: str or Path
+        Path to the night summary file
+
+    Returns
+    -------
+    table: Table
+        astropy table of the night summary file.
+        The columns will have the correct dtype (int64) for the counters
+        and missing values (nan in the file) are masked.
+    '''
+
+    # convertes for each column to make sure we use the correct
+    # dtypes. The counter values in ns are so large that they cannot
+    # be exactly represented by float64 values, we need int64
+    converters = {
+        'run': [convert_numpy(np.int32)],
+        'n_subruns': [convert_numpy(np.int32)],
+        'run_type': [convert_numpy(str)],
+        'date': [convert_numpy(str)],
+        'time': [convert_numpy(str)],
+        'first_valid_event_dragon': [convert_numpy(np.int64)],
+        'ucts_t0_dragon': [convert_numpy(np.int64)],
+        'dragon_counter0': [convert_numpy(np.int64)],
+        'first_valid_event_tib': [convert_numpy(np.int64)],
+        'ucts_t0_tib': [convert_numpy(np.int64)],
+        'tib_counter0': [convert_numpy(np.int64)],
+    }
+
     summary = Table.read(
         str(path),
         format='ascii.basic',
@@ -66,8 +101,12 @@ def read_night_summary(path):
             'first_valid_event_dragon', 'ucts_t0_dragon', 'dragon_counter0',
             'first_valid_event_tib', 'ucts_t0_tib', 'tib_counter0',
         ],
+        converters=converters,
+        fill_values=("nan", -1),
         guess=False,
+        fast_reader=False,
     )
+
     summary.add_index(['run'])
     summary['timestamp'] = datetime_cols_to_time(summary['date'], summary['time'])
     return summary

ctapipe_io_lst/tests/test_event_time.py

@@ -9,6 +9,17 @@
 
 test_data = Path(os.getenv('LSTCHAIN_TEST_DATA', 'test_data'))
 test_night_summary = test_data / 'real/monitoring/NightSummary/NightSummary_20200218.txt'
+test_night_summary_with_nans = test_data / 'real/monitoring/NightSummary/NightSummary_20201215.txt'
+
+
+int_cols = [
+    'first_valid_event_dragon',
+    'ucts_t0_dragon',
+    'dragon_counter0',
+    'first_valid_event_tib',
+    'ucts_t0_tib',
+    'tib_counter0',
+]
 
 
 def test_time_unix_tai():
@@ -20,7 +31,22 @@ def test_read_night_summary():
     from ctapipe_io_lst.event_time import read_night_summary
 
     summary = read_night_summary(test_night_summary)
-    assert summary['ucts_t0_dragon'].dtype == int
+
+    for col in int_cols:
+        assert summary[col].dtype == np.int64
+
+
+def test_read_night_summary_missing():
+    from ctapipe_io_lst.event_time import read_night_summary
+
+    summary = read_night_summary(test_night_summary_with_nans)
+    for col in int_cols:
+        assert summary[col].dtype == np.int64
+
+    assert len(summary) == 26
+    assert isinstance(summary['timestamp'], Time)
+    assert np.ma.is_masked(summary['tib_counter0'])
+    assert np.count_nonzero(summary['tib_counter0'].mask) == 12
 
 
 def test_ucts_jumps():