From 9f3bbe99b46da30c6773de45bb6ce421bfca9950 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 26 Apr 2022 10:16:29 +0200 Subject: [PATCH 1/7] Add test for load_pointings option in TableLoader --- ctapipe/io/tests/test_table_loader.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index 74ea39d9bd8..a1e79f3995f 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -113,7 +113,6 @@ def test_true_parameters(test_file): def test_read_subarray_events(test_file_dl2): """Test reading subarray events""" - from ctapipe.io.tableloader import TableLoader _, dl2_file = test_file_dl2 @@ -127,9 +126,19 @@ def test_read_subarray_events(test_file_dl2): assert "time" in table.colnames +def test_pointings(test_file): + """Test joining true_parameters poitning information onto subarray events""" + from ctapipe.io.tableloader import TableLoader + + _, dl1_file = test_file + + with TableLoader(dl1_file, load_pointings=True) as table_loader: + table = table_loader.read_subarray_events() + assert "array_altitude" in table.colnames + + def test_read_telescope_events_type(test_file_dl2): """Test reading telescope events for a given telescope type""" - from ctapipe.io.tableloader import TableLoader _, dl2_file = test_file_dl2 @@ -156,7 +165,6 @@ def test_read_telescope_events_type(test_file_dl2): def test_read_telescope_events_by_type(test_file_dl2): """Test reading telescope events for by types""" - from ctapipe.io.tableloader import TableLoader _, dl2_file = test_file_dl2 From 027b0be4b4d38e4e64f9f66a5113960d3c9c4b59 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 26 Apr 2022 12:52:29 +0200 Subject: [PATCH 2/7] Add load_pointings option - create method to join interpolated pointings to table --- ctapipe/io/tableloader.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index 101e87a13fe..5de7025f73d 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -21,6 +21,7 @@ IMAGES_GROUP = "/dl1/event/telescope/images" GEOMETRY_GROUP = "/dl2/event/subarray/geometry" TRIGGER_TABLE = "/dl1/event/subarray/trigger" +POINTING_TABLE = "/dl1/monitoring/subarray/pointing" SHOWER_TABLE = "/simulation/event/subarray/shower" TRUE_IMAGES_GROUP = "/simulation/event/telescope/images" TRUE_PARAMETERS_GROUP = "/simulation/event/telescope/parameters" @@ -110,6 +111,9 @@ class TableLoader(Component): load_instrument = traits.Bool( False, help="join subarray instrument information to each event" ).tag(config=True) + load_pointings = traits.Bool( + False, help="join subarray pointings information" + ).tag(config=True) def __init__(self, input_url=None, **kwargs): # enable using input_url as posarg @@ -172,6 +176,18 @@ def _read_telescope_table(self, group, tel_id): return table + def _join_interp_pointings(self, table): + pointings = read_table(self.h5file, POINTING_TABLE) + for col in set(pointings.colnames) - set(["time"]): + interp = np.interp(table['time'].value, + pointings['time'].value, pointings[col].value) + table[col] = interp + + if not self.load_trigger: + table.remove_columns(["tels_with_trigger", "event_type"]) + + return table + def read_subarray_events(self): """Read subarray-based event information. @@ -186,6 +202,12 @@ def read_subarray_events(self): trigger = read_table(self.h5file, TRIGGER_TABLE) table = join_allow_empty(table, trigger, SUBARRAY_EVENT_KEYS, "outer") + if self.load_pointings: + if not self.load_trigger: + trigger = read_table(self.h5file, TRIGGER_TABLE) + table = join_allow_empty(table, trigger, SUBARRAY_EVENT_KEYS, "outer") + table = self._join_interp_pointings(table) + if self.load_simulated and SHOWER_TABLE in self.h5file: showers = read_table(self.h5file, SHOWER_TABLE) table = join_allow_empty(table, showers, SUBARRAY_EVENT_KEYS, "outer") @@ -302,8 +324,11 @@ def read_telescope_events(self, telescopes=None): tel_ids = self.subarray.get_tel_ids(telescopes) table = self._read_telescope_events_for_ids(tel_ids) - - if any([self.load_trigger, self.load_simulated, self.load_dl2_geometry]): + + # TODO: load telescope trigger and pointing tables instead of appending the + # subarray one + if any([self.load_trigger, self.load_simulated, + self.load_dl2_geometry, self.load_pointings]): table = self._join_subarray_info(table) return table @@ -336,7 +361,8 @@ def read_telescope_events_by_type(self, telescopes=None) -> Dict[str, Table]: by_type = {k: vstack(ts) for k, ts in by_type.items()} - if any([self.load_trigger, self.load_simulated, self.load_dl2_geometry]): + if any([self.load_trigger, self.load_simulated, + self.load_dl2_geometry, self.load_pointings]): for key, table in by_type.items(): by_type[key] = self._join_subarray_info(table) From 3b9350515d53b07dc32c0cbabae37465e99e2cf7 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 26 Apr 2022 17:10:08 +0200 Subject: [PATCH 3/7] Save one if clause --- ctapipe/io/tableloader.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index 5de7025f73d..befff731c18 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -176,16 +176,12 @@ def _read_telescope_table(self, group, tel_id): return table - def _join_interp_pointings(self, table): - pointings = read_table(self.h5file, POINTING_TABLE) + def _join_interp_pointings(self, table, pointings): for col in set(pointings.colnames) - set(["time"]): interp = np.interp(table['time'].value, pointings['time'].value, pointings[col].value) table[col] = interp - if not self.load_trigger: - table.remove_columns(["tels_with_trigger", "event_type"]) - return table def read_subarray_events(self): @@ -206,7 +202,9 @@ def read_subarray_events(self): if not self.load_trigger: trigger = read_table(self.h5file, TRIGGER_TABLE) table = join_allow_empty(table, trigger, SUBARRAY_EVENT_KEYS, "outer") - table = self._join_interp_pointings(table) + table.remove_columns(["tels_with_trigger", "event_type"]) + pointings = read_table(self.h5file, POINTING_TABLE) + table = self._join_interp_pointings(table, pointings) if self.load_simulated and SHOWER_TABLE in self.h5file: showers = read_table(self.h5file, SHOWER_TABLE) From 3cdde6e2158069d13122cc14969df1a045556e2c Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 26 Apr 2022 19:50:15 +0200 Subject: [PATCH 4/7] Adapt unit and dtype of pointings --- ctapipe/io/tableloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index befff731c18..b6d0b8c9aff 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -178,9 +178,9 @@ def _read_telescope_table(self, group, tel_id): def _join_interp_pointings(self, table, pointings): for col in set(pointings.colnames) - set(["time"]): - interp = np.interp(table['time'].value, - pointings['time'].value, pointings[col].value) - table[col] = interp + table[col] = np.interp(table['time'].mjd, pointings['time'].mjd, + pointings[col]).astype(pointings[col].dtype) + table[col].unit = pointings[col].unit return table From 0d86e1e7bd17ca502caf55b9cf3f92d4884d2f21 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 27 Apr 2022 11:38:43 +0200 Subject: [PATCH 5/7] Reformat with black --- ctapipe/io/tableloader.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index b6d0b8c9aff..4430f087a0c 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -111,9 +111,9 @@ class TableLoader(Component): load_instrument = traits.Bool( False, help="join subarray instrument information to each event" ).tag(config=True) - load_pointings = traits.Bool( - False, help="join subarray pointings information" - ).tag(config=True) + load_pointings = traits.Bool(False, help="join subarray pointings information").tag( + config=True + ) def __init__(self, input_url=None, **kwargs): # enable using input_url as posarg @@ -178,9 +178,10 @@ def _read_telescope_table(self, group, tel_id): def _join_interp_pointings(self, table, pointings): for col in set(pointings.colnames) - set(["time"]): - table[col] = np.interp(table['time'].mjd, pointings['time'].mjd, - pointings[col]).astype(pointings[col].dtype) - table[col].unit = pointings[col].unit + table[col] = np.interp( + table["time"].mjd, pointings["time"].mjd, pointings[col] + ).astype(pointings[col].dtype) + table[col].unit = pointings[col].unit return table @@ -322,11 +323,17 @@ def read_telescope_events(self, telescopes=None): tel_ids = self.subarray.get_tel_ids(telescopes) table = self._read_telescope_events_for_ids(tel_ids) - + # TODO: load telescope trigger and pointing tables instead of appending the # subarray one - if any([self.load_trigger, self.load_simulated, - self.load_dl2_geometry, self.load_pointings]): + if any( + [ + self.load_trigger, + self.load_simulated, + self.load_dl2_geometry, + self.load_pointings, + ] + ): table = self._join_subarray_info(table) return table @@ -359,8 +366,14 @@ def read_telescope_events_by_type(self, telescopes=None) -> Dict[str, Table]: by_type = {k: vstack(ts) for k, ts in by_type.items()} - if any([self.load_trigger, self.load_simulated, - self.load_dl2_geometry, self.load_pointings]): + if any( + [ + self.load_trigger, + self.load_simulated, + self.load_dl2_geometry, + self.load_pointings, + ] + ): for key, table in by_type.items(): by_type[key] = self._join_subarray_info(table) From 94aba5767b752bb0edf8a101590e653d214c9ab7 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 27 Apr 2022 12:29:52 +0200 Subject: [PATCH 6/7] typo --- ctapipe/io/tests/test_table_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index a1e79f3995f..4b7147c0e39 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -127,7 +127,7 @@ def test_read_subarray_events(test_file_dl2): def test_pointings(test_file): - """Test joining true_parameters poitning information onto subarray events""" + """Test joining true_parameters pointing information onto subarray events""" from ctapipe.io.tableloader import TableLoader _, dl1_file = test_file From 2505e888cf1fcaace1a2cecc6a4e3d9ff62259b2 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 27 Apr 2022 12:42:13 +0200 Subject: [PATCH 7/7] Make codecov happy --- ctapipe/io/tests/test_table_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index 4b7147c0e39..d87b032d732 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -132,7 +132,7 @@ def test_pointings(test_file): _, dl1_file = test_file - with TableLoader(dl1_file, load_pointings=True) as table_loader: + with TableLoader(dl1_file, load_pointings=True, load_trigger=False) as table_loader: table = table_loader.read_subarray_events() assert "array_altitude" in table.colnames