From 60e743093912650e6684fdf3f543c08ef05f5da8 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Fri, 16 Feb 2024 13:03:43 +0100 Subject: [PATCH 01/10] Adapting ImagerCleaner API --- src/ctapipe/conftest.py | 6 +-- src/ctapipe/image/cleaning.py | 38 ++++++----------- src/ctapipe/image/image_processor.py | 7 +--- src/ctapipe/image/reducer.py | 41 +++++++++++++++---- .../tests/test_image_cleaner_component.py | 7 +++- src/ctapipe/image/tests/test_reducer.py | 10 ++--- 6 files changed, 59 insertions(+), 50 deletions(-) diff --git a/src/ctapipe/conftest.py b/src/ctapipe/conftest.py index b521d82c7c5..584385ec469 100644 --- a/src/ctapipe/conftest.py +++ b/src/ctapipe/conftest.py @@ -50,7 +50,7 @@ def camera_geometry(request): def _global_example_event(): """ helper to get a single event from a MC file. Don't use this fixture - directly, rather use `test_event` + directly, rather use `example_event` """ filename = get_dataset_path("gamma_test_large.simtel.gz") @@ -93,8 +93,8 @@ def example_event(_global_example_event): example: .. code-block:: - def test_my_thing(test_event): - assert len(test_event.r0.tel) > 0 + def test_my_thing(example_event): + assert len(example_event.r0.tel) > 0 """ return deepcopy(_global_example_event) diff --git a/src/ctapipe/image/cleaning.py b/src/ctapipe/image/cleaning.py index 5041055d8f2..3ee576d7553 100644 --- a/src/ctapipe/image/cleaning.py +++ b/src/ctapipe/image/cleaning.py @@ -28,6 +28,7 @@ import numpy as np +from ..containers import ArrayEventContainer from ..core import TelescopeComponent from ..core.traits import ( BoolTelescopeParameter, @@ -465,9 +466,7 @@ class ImageCleaner(TelescopeComponent): """ @abstractmethod - def __call__( - self, tel_id: int, image: np.ndarray, arrival_times: np.ndarray = None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Identify pixels with signal, and reject those with pure noise. @@ -476,10 +475,7 @@ def __call__( tel_id: int which telescope id in the subarray is being used (determines which cut is used) - image : np.ndarray - image pixel data corresponding to the camera geometry - arrival_times: np.ndarray - image of arrival time (not used in this method) + event: `ctapipe.containers.ArrayEventContainer` Returns ------- @@ -513,16 +509,14 @@ class TailcutsImageCleaner(ImageCleaner): "removed.", ).tag(config=True) - def __call__( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Apply standard picture-boundary cleaning. See `ImageCleaner.__call__()` """ return tailcuts_clean( self.subarray.tel[tel_id].camera.geometry, - image, + event.dl1.tel[tel_id].image, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], @@ -535,16 +529,14 @@ class MARSImageCleaner(TailcutsImageCleaner): 1st-pass MARS-like Image cleaner (See `ctapipe.image.mars_cleaning_1st_pass`) """ - def __call__( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Apply MARS-style image cleaning. See `ImageCleaner.__call__()` """ return mars_cleaning_1st_pass( self.subarray.tel[tel_id].camera.geometry, - image, + event.dl1.tel[tel_id].image, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], @@ -562,14 +554,12 @@ class FACTImageCleaner(TailcutsImageCleaner): default_value=5.0, help="arrival time limit for neighboring " "pixels, in ns" ).tag(config=True) - def __call__( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" return fact_image_cleaning( geom=self.subarray.tel[tel_id].camera.geometry, - image=image, - arrival_times=arrival_times, + image=event.dl1.tel[tel_id].image, + arrival_times=event.dl1.tel[tel_id].peak_time, picture_threshold=self.picture_threshold_pe.tel[tel_id], boundary_threshold=self.boundary_threshold_pe.tel[tel_id], min_number_neighbors=self.min_picture_neighbors.tel[tel_id], @@ -591,17 +581,15 @@ class TimeConstrainedImageCleaner(TailcutsImageCleaner): help="arrival time limit for neighboring " "boundary pixels, in ns", ).tag(config=True) - def __call__( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Apply MAGIC-like image cleaning with timing information. See `ImageCleaner.__call__()` """ return time_constrained_clean( self.subarray.tel[tel_id].camera.geometry, - image, - arrival_times=arrival_times, + event.dl1.tel[tel_id].image, + arrival_times=event.dl1.tel[tel_id].peak_time, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], diff --git a/src/ctapipe/image/image_processor.py b/src/ctapipe/image/image_processor.py index 179f43d6b66..fca88aa22dd 100644 --- a/src/ctapipe/image/image_processor.py +++ b/src/ctapipe/image/image_processor.py @@ -1,6 +1,7 @@ """ High level image processing (ImageProcessor Component) """ + from copy import deepcopy import numpy as np @@ -217,11 +218,7 @@ def _process_telescope_event(self, event): if self.apply_image_modifier.tel[tel_id]: dl1_camera.image = self.modify(tel_id=tel_id, image=dl1_camera.image) - dl1_camera.image_mask = self.clean( - tel_id=tel_id, - image=dl1_camera.image, - arrival_times=dl1_camera.peak_time, - ) + dl1_camera.image_mask = self.clean(tel_id=tel_id, event=event) dl1_camera.parameters = self._parameterize_image( tel_id=tel_id, diff --git a/src/ctapipe/image/reducer.py b/src/ctapipe/image/reducer.py index 3cbeb6bd509..e37ea5c7822 100644 --- a/src/ctapipe/image/reducer.py +++ b/src/ctapipe/image/reducer.py @@ -1,6 +1,7 @@ """ Algorithms for the data volume reduction. """ + from abc import abstractmethod import numpy as np @@ -10,11 +11,11 @@ from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, + FloatTelescopeParameter, IntTelescopeParameter, TelescopeParameter, ) -from ctapipe.image import TailcutsImageCleaner -from ctapipe.image.cleaning import dilate +from ctapipe.image.cleaning import dilate, tailcuts_clean from ctapipe.image.extractor import ImageExtractor __all__ = ["DataVolumeReducer", "NullDataVolumeReducer", "TailCutsDataVolumeReducer"] @@ -129,6 +130,27 @@ class TailCutsDataVolumeReducer(DataVolumeReducer): help="Name of the ImageExtractor subclass to be used.", ).tag(config=True) + picture_threshold_pe = FloatTelescopeParameter( + default_value=10.0, + help="top-level threshold in photoelectrons for ``tailcuts_clean``", + ).tag(config=True) + + boundary_threshold_pe = FloatTelescopeParameter( + default_value=5.0, + help="second-level threshold in photoelectrons for ``tailcuts_clean``", + ).tag(config=True) + + min_picture_neighbors = IntTelescopeParameter( + default_value=2, + help="Minimum number of neighbors above threshold to consider for ``tailcuts_clean``", + ).tag(config=True) + + keep_isolated_pixels = BoolTelescopeParameter( + default_value=False, + help="If False, pixels with less neighbors than ``min_picture_neighbors`` are" + "removed for ``tailcuts_clean``.", + ).tag(config=True) + n_end_dilates = IntTelescopeParameter( default_value=1, help="Number of how many times to dilate at the end." ).tag(config=True) @@ -144,7 +166,6 @@ def __init__( subarray, config=None, parent=None, - cleaner=None, image_extractor=None, **kwargs, ): @@ -161,11 +182,6 @@ def __init__( """ super().__init__(config=config, parent=parent, subarray=subarray, **kwargs) - if cleaner is None: - self.cleaner = TailcutsImageCleaner(parent=self, subarray=self.subarray) - else: - self.cleaner = cleaner - self.image_extractors = {} if image_extractor is None: for _, _, name in self.image_extractor_type: @@ -191,7 +207,14 @@ def select_pixels(self, waveforms, tel_id=None, selected_gain_channel=None): ) # 1) Step: TailcutCleaning at first - mask = self.cleaner(tel_id, dl1.image) + mask = tailcuts_clean( + camera_geom, + dl1.tel[tel_id].image, + picture_thresh=self.picture_threshold_pe.tel[tel_id], + boundary_thresh=self.boundary_threshold_pe.tel[tel_id], + keep_isolated_pixels=self.keep_isolated_pixels.tel[tel_id], + min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], + ) pixels_above_boundary_thresh = ( dl1.image >= self.cleaner.boundary_threshold_pe.tel[tel_id] ) diff --git a/src/ctapipe/image/tests/test_image_cleaner_component.py b/src/ctapipe/image/tests/test_image_cleaner_component.py index 1b8ef34274b..ff837e49160 100644 --- a/src/ctapipe/image/tests/test_image_cleaner_component.py +++ b/src/ctapipe/image/tests/test_image_cleaner_component.py @@ -7,7 +7,7 @@ @pytest.mark.parametrize("method", ImageCleaner.non_abstract_subclasses().keys()) -def test_image_cleaner(method, prod5_mst_nectarcam, reference_location): +def test_image_cleaner(method, prod5_mst_nectarcam, example_event, reference_location): """Test that we can construct and use a component-based ImageCleaner""" config = Config( @@ -45,7 +45,10 @@ def test_image_cleaner(method, prod5_mst_nectarcam, reference_location): image[31:40] = 8.0 times = np.linspace(-5, 10, image.shape[0]) - mask = clean(tel_id=1, image=image, arrival_times=times) + example_event.dl1.tel[1].image = image + example_event.dl1.tel[1].peak_time = times + + mask = clean(tel_id=1, event=example_event) # we're not testing the algorithm here, just that it does something (for the # algorithm tests, see test_cleaning.py diff --git a/src/ctapipe/image/tests/test_reducer.py b/src/ctapipe/image/tests/test_reducer.py index ca6e8fcaba6..e993bc8bcfe 100644 --- a/src/ctapipe/image/tests/test_reducer.py +++ b/src/ctapipe/image/tests/test_reducer.py @@ -64,17 +64,15 @@ def test_tailcuts_data_volume_reducer(subarray_lst): reduction_param = Config( { "TailCutsDataVolumeReducer": { - "TailcutsImageCleaner": { - "picture_threshold_pe": 700.0, - "boundary_threshold_pe": 350.0, - "min_picture_neighbors": 0, - "keep_isolated_pixels": True, - }, "image_extractor_type": "NeighborPeakWindowSum", "NeighborPeakWindowSum": { "apply_integration_correction": False, "window_shift": 0, }, + "picture_threshold_pe": 700.0, + "boundary_threshold_pe": 350.0, + "min_picture_neighbors": 0, + "keep_isolated_pixels": True, "n_end_dilates": 1, "do_boundary_dilation": True, } From afbc01fe3efad162ff4ef5afbc4b2f45c7b5b025 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Fri, 16 Feb 2024 13:20:01 +0100 Subject: [PATCH 02/10] Fix tests --- src/ctapipe/calib/camera/tests/test_calibrator.py | 7 +++---- src/ctapipe/image/reducer.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_calibrator.py b/src/ctapipe/calib/camera/tests/test_calibrator.py index b16b9dcc68e..537fd99d376 100644 --- a/src/ctapipe/calib/camera/tests/test_calibrator.py +++ b/src/ctapipe/calib/camera/tests/test_calibrator.py @@ -1,6 +1,7 @@ """ Tests for CameraCalibrator and related functions """ + from copy import deepcopy import astropy.units as u @@ -63,9 +64,7 @@ def test_config(example_subarray): "window_width": [("type", "*", 10), ("id", 2, 8)] }, "data_volume_reducer_type": "TailCutsDataVolumeReducer", - "TailCutsDataVolumeReducer": { - "TailcutsImageCleaner": {"picture_threshold_pe": 20.0} - }, + "TailCutsDataVolumeReducer": {"picture_threshold_pe": 20.0}, } } ) @@ -93,7 +92,7 @@ def test_config(example_subarray): assert extractor_3.window_width.tel[3] == 10 assert isinstance(calibrator.data_volume_reducer, TailCutsDataVolumeReducer) - assert calibrator.data_volume_reducer.cleaner.picture_threshold_pe.tel[None] == 20 + assert calibrator.data_volume_reducer.picture_threshold_pe.tel[None] == 20 def test_check_r1_empty(example_event, example_subarray): diff --git a/src/ctapipe/image/reducer.py b/src/ctapipe/image/reducer.py index e37ea5c7822..5b1f4d05d65 100644 --- a/src/ctapipe/image/reducer.py +++ b/src/ctapipe/image/reducer.py @@ -209,14 +209,14 @@ def select_pixels(self, waveforms, tel_id=None, selected_gain_channel=None): # 1) Step: TailcutCleaning at first mask = tailcuts_clean( camera_geom, - dl1.tel[tel_id].image, + dl1.image, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], keep_isolated_pixels=self.keep_isolated_pixels.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], ) pixels_above_boundary_thresh = ( - dl1.image >= self.cleaner.boundary_threshold_pe.tel[tel_id] + dl1.image >= self.boundary_threshold_pe.tel[tel_id] ) mask_in_loop = np.array([]) # 2) Step: Add iteratively all pixels with Signal From d79025b7d7ff6c672389607bd27f5f8fe9a633a1 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Fri, 16 Feb 2024 13:43:53 +0100 Subject: [PATCH 03/10] Adding changelog --- docs/changes/2511.api.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 docs/changes/2511.api.rst diff --git a/docs/changes/2511.api.rst b/docs/changes/2511.api.rst new file mode 100644 index 00000000000..2ce5e758175 --- /dev/null +++ b/docs/changes/2511.api.rst @@ -0,0 +1,4 @@ +Change the ``ImageCleaner`` API from __call__(self, tel_id, image, peak_time) +to __call__(self, tel_id: int, event: ArrayEventContainer) so cleaning +algorithms can now access relevant information for methods +that e.g. require monitoring information. From 755ffa72aa9c431068ffd7c3646fee963639ab26 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Fri, 16 Feb 2024 13:45:53 +0100 Subject: [PATCH 04/10] Shorten line --- src/ctapipe/image/reducer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/image/reducer.py b/src/ctapipe/image/reducer.py index 5b1f4d05d65..2897dba3994 100644 --- a/src/ctapipe/image/reducer.py +++ b/src/ctapipe/image/reducer.py @@ -142,7 +142,8 @@ class TailCutsDataVolumeReducer(DataVolumeReducer): min_picture_neighbors = IntTelescopeParameter( default_value=2, - help="Minimum number of neighbors above threshold to consider for ``tailcuts_clean``", + help="Minimum number of neighbors above threshold to consider" + "for ``tailcuts_clean``", ).tag(config=True) keep_isolated_pixels = BoolTelescopeParameter( From 121df23df4446b8d2f5ff23a0c0aaebdf31f51e3 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Mon, 19 Feb 2024 10:54:27 +0100 Subject: [PATCH 05/10] Adding low-level cleaning method in ImageCleaner --- .../calib/camera/tests/test_calibrator.py | 6 +- src/ctapipe/image/cleaning.py | 63 +++++++++++++++---- src/ctapipe/image/reducer.py | 44 +++---------- src/ctapipe/image/tests/test_reducer.py | 10 +-- 4 files changed, 70 insertions(+), 53 deletions(-) diff --git a/src/ctapipe/calib/camera/tests/test_calibrator.py b/src/ctapipe/calib/camera/tests/test_calibrator.py index 537fd99d376..1e22d4c28c9 100644 --- a/src/ctapipe/calib/camera/tests/test_calibrator.py +++ b/src/ctapipe/calib/camera/tests/test_calibrator.py @@ -64,7 +64,9 @@ def test_config(example_subarray): "window_width": [("type", "*", 10), ("id", 2, 8)] }, "data_volume_reducer_type": "TailCutsDataVolumeReducer", - "TailCutsDataVolumeReducer": {"picture_threshold_pe": 20.0}, + "TailCutsDataVolumeReducer": { + "TailcutsImageCleaner": {"picture_threshold_pe": 20.0} + }, } } ) @@ -92,7 +94,7 @@ def test_config(example_subarray): assert extractor_3.window_width.tel[3] == 10 assert isinstance(calibrator.data_volume_reducer, TailCutsDataVolumeReducer) - assert calibrator.data_volume_reducer.picture_threshold_pe.tel[None] == 20 + assert calibrator.data_volume_reducer.cleaner.picture_threshold_pe.tel[None] == 20 def test_check_r1_empty(example_event, example_subarray): diff --git a/src/ctapipe/image/cleaning.py b/src/ctapipe/image/cleaning.py index 3ee576d7553..5d3fdb7a9ea 100644 --- a/src/ctapipe/image/cleaning.py +++ b/src/ctapipe/image/cleaning.py @@ -465,10 +465,10 @@ class ImageCleaner(TelescopeComponent): ``ImageCleaner.from_name()`` to construct an instance of a particular algorithm """ - @abstractmethod def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ - Identify pixels with signal, and reject those with pure noise. + Call the relevant functions to identify pixels with signal + and reject those with pure noise. Parameters ---------- @@ -482,7 +482,36 @@ def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: np.ndarray boolean mask of pixels passing cleaning """ - pass + mask = self.clean_image( + tel_id=tel_id, + image=event.dl1.tel[tel_id].image, + arrival_times=event.dl1.tel[tel_id].peak_time, + ) + return mask + + @abstractmethod + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times: np.ndarray = None + ) -> np.ndarray: + """ + Abstract method to be defined by a ImageCleaner subclass. + Call the relevant function for the required image cleaning. + + Parameters + ---------- + tel_id: int + which telescope id in the subarray is being used (determines + which cut is used) + image : np.ndarray + image pixel data corresponding to the camera geometry + arrival_times: np.ndarray + image of arrival time (not used in this method) + + Returns + ------- + np.ndarray + boolean mask of pixels passing cleaning + """ class TailcutsImageCleaner(ImageCleaner): @@ -509,14 +538,16 @@ class TailcutsImageCleaner(ImageCleaner): "removed.", ).tag(config=True) - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: """ Apply standard picture-boundary cleaning. See `ImageCleaner.__call__()` """ return tailcuts_clean( self.subarray.tel[tel_id].camera.geometry, - event.dl1.tel[tel_id].image, + image, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], @@ -529,14 +560,16 @@ class MARSImageCleaner(TailcutsImageCleaner): 1st-pass MARS-like Image cleaner (See `ctapipe.image.mars_cleaning_1st_pass`) """ - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: """ Apply MARS-style image cleaning. See `ImageCleaner.__call__()` """ return mars_cleaning_1st_pass( self.subarray.tel[tel_id].camera.geometry, - event.dl1.tel[tel_id].image, + image, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], @@ -554,12 +587,14 @@ class FACTImageCleaner(TailcutsImageCleaner): default_value=5.0, help="arrival time limit for neighboring " "pixels, in ns" ).tag(config=True) - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" return fact_image_cleaning( geom=self.subarray.tel[tel_id].camera.geometry, - image=event.dl1.tel[tel_id].image, - arrival_times=event.dl1.tel[tel_id].peak_time, + image=image, + arrival_times=arrival_times, picture_threshold=self.picture_threshold_pe.tel[tel_id], boundary_threshold=self.boundary_threshold_pe.tel[tel_id], min_number_neighbors=self.min_picture_neighbors.tel[tel_id], @@ -581,15 +616,17 @@ class TimeConstrainedImageCleaner(TailcutsImageCleaner): help="arrival time limit for neighboring " "boundary pixels, in ns", ).tag(config=True) - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: """ Apply MAGIC-like image cleaning with timing information. See `ImageCleaner.__call__()` """ return time_constrained_clean( self.subarray.tel[tel_id].camera.geometry, - event.dl1.tel[tel_id].image, - arrival_times=event.dl1.tel[tel_id].peak_time, + image, + arrival_times=arrival_times, picture_thresh=self.picture_threshold_pe.tel[tel_id], boundary_thresh=self.boundary_threshold_pe.tel[tel_id], min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], diff --git a/src/ctapipe/image/reducer.py b/src/ctapipe/image/reducer.py index 2897dba3994..63a19568f3e 100644 --- a/src/ctapipe/image/reducer.py +++ b/src/ctapipe/image/reducer.py @@ -1,7 +1,6 @@ """ Algorithms for the data volume reduction. """ - from abc import abstractmethod import numpy as np @@ -11,11 +10,11 @@ from ctapipe.core.traits import ( BoolTelescopeParameter, ComponentName, - FloatTelescopeParameter, IntTelescopeParameter, TelescopeParameter, ) -from ctapipe.image.cleaning import dilate, tailcuts_clean +from ctapipe.image import TailcutsImageCleaner +from ctapipe.image.cleaning import dilate from ctapipe.image.extractor import ImageExtractor __all__ = ["DataVolumeReducer", "NullDataVolumeReducer", "TailCutsDataVolumeReducer"] @@ -130,28 +129,6 @@ class TailCutsDataVolumeReducer(DataVolumeReducer): help="Name of the ImageExtractor subclass to be used.", ).tag(config=True) - picture_threshold_pe = FloatTelescopeParameter( - default_value=10.0, - help="top-level threshold in photoelectrons for ``tailcuts_clean``", - ).tag(config=True) - - boundary_threshold_pe = FloatTelescopeParameter( - default_value=5.0, - help="second-level threshold in photoelectrons for ``tailcuts_clean``", - ).tag(config=True) - - min_picture_neighbors = IntTelescopeParameter( - default_value=2, - help="Minimum number of neighbors above threshold to consider" - "for ``tailcuts_clean``", - ).tag(config=True) - - keep_isolated_pixels = BoolTelescopeParameter( - default_value=False, - help="If False, pixels with less neighbors than ``min_picture_neighbors`` are" - "removed for ``tailcuts_clean``.", - ).tag(config=True) - n_end_dilates = IntTelescopeParameter( default_value=1, help="Number of how many times to dilate at the end." ).tag(config=True) @@ -167,6 +144,7 @@ def __init__( subarray, config=None, parent=None, + cleaner=None, image_extractor=None, **kwargs, ): @@ -183,6 +161,11 @@ def __init__( """ super().__init__(config=config, parent=parent, subarray=subarray, **kwargs) + if cleaner is None: + self.cleaner = TailcutsImageCleaner(parent=self, subarray=self.subarray) + else: + self.cleaner = cleaner + self.image_extractors = {} if image_extractor is None: for _, _, name in self.image_extractor_type: @@ -208,16 +191,9 @@ def select_pixels(self, waveforms, tel_id=None, selected_gain_channel=None): ) # 1) Step: TailcutCleaning at first - mask = tailcuts_clean( - camera_geom, - dl1.image, - picture_thresh=self.picture_threshold_pe.tel[tel_id], - boundary_thresh=self.boundary_threshold_pe.tel[tel_id], - keep_isolated_pixels=self.keep_isolated_pixels.tel[tel_id], - min_number_picture_neighbors=self.min_picture_neighbors.tel[tel_id], - ) + mask = self.cleaner.clean_image(tel_id, dl1.image) pixels_above_boundary_thresh = ( - dl1.image >= self.boundary_threshold_pe.tel[tel_id] + dl1.image >= self.cleaner.boundary_threshold_pe.tel[tel_id] ) mask_in_loop = np.array([]) # 2) Step: Add iteratively all pixels with Signal diff --git a/src/ctapipe/image/tests/test_reducer.py b/src/ctapipe/image/tests/test_reducer.py index e993bc8bcfe..ca6e8fcaba6 100644 --- a/src/ctapipe/image/tests/test_reducer.py +++ b/src/ctapipe/image/tests/test_reducer.py @@ -64,15 +64,17 @@ def test_tailcuts_data_volume_reducer(subarray_lst): reduction_param = Config( { "TailCutsDataVolumeReducer": { + "TailcutsImageCleaner": { + "picture_threshold_pe": 700.0, + "boundary_threshold_pe": 350.0, + "min_picture_neighbors": 0, + "keep_isolated_pixels": True, + }, "image_extractor_type": "NeighborPeakWindowSum", "NeighborPeakWindowSum": { "apply_integration_correction": False, "window_shift": 0, }, - "picture_threshold_pe": 700.0, - "boundary_threshold_pe": 350.0, - "min_picture_neighbors": 0, - "keep_isolated_pixels": True, "n_end_dilates": 1, "do_boundary_dilation": True, } From 6d3ca07ef3d8d662824cc58130fdfbe68f4ae6c9 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Mon, 19 Feb 2024 11:23:25 +0100 Subject: [PATCH 06/10] Adapt docstring --- src/ctapipe/image/cleaning.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ctapipe/image/cleaning.py b/src/ctapipe/image/cleaning.py index 5d3fdb7a9ea..c129539add0 100644 --- a/src/ctapipe/image/cleaning.py +++ b/src/ctapipe/image/cleaning.py @@ -467,7 +467,7 @@ class ImageCleaner(TelescopeComponent): def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ - Call the relevant functions to identify pixels with signal + Calls the relevant functions to identify pixels with signal and reject those with pure noise. Parameters @@ -494,8 +494,7 @@ def clean_image( self, tel_id: int, image: np.ndarray, arrival_times: np.ndarray = None ) -> np.ndarray: """ - Abstract method to be defined by a ImageCleaner subclass. - Call the relevant function for the required image cleaning. + Abstract cleaning method to be defined by an ImageCleaner subclass. Parameters ---------- From 0913db5c86772c2939a4abedb455ae144595ac60 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Mon, 19 Feb 2024 12:17:26 +0100 Subject: [PATCH 07/10] Update changelog --- docs/changes/2511.api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changes/2511.api.rst b/docs/changes/2511.api.rst index 2ce5e758175..7e986cf6e66 100644 --- a/docs/changes/2511.api.rst +++ b/docs/changes/2511.api.rst @@ -2,3 +2,7 @@ Change the ``ImageCleaner`` API from __call__(self, tel_id, image, peak_time) to __call__(self, tel_id: int, event: ArrayEventContainer) so cleaning algorithms can now access relevant information for methods that e.g. require monitoring information. + +The __call__ function now internally uses an abstract +clean_image(self, tel_id, image, peak_time) method +defined by each ImageCleaner subclass. From d6a364e70a4b38cc6e5bb7850a00bd147eb10806 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Mon, 19 Feb 2024 17:44:28 +0100 Subject: [PATCH 08/10] Make __call__ abstract --- src/ctapipe/image/cleaning.py | 62 ++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/src/ctapipe/image/cleaning.py b/src/ctapipe/image/cleaning.py index c129539add0..1377005dcdf 100644 --- a/src/ctapipe/image/cleaning.py +++ b/src/ctapipe/image/cleaning.py @@ -465,6 +465,7 @@ class ImageCleaner(TelescopeComponent): ``ImageCleaner.from_name()`` to construct an instance of a particular algorithm """ + @abstractmethod def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Calls the relevant functions to identify pixels with signal @@ -482,12 +483,6 @@ def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: np.ndarray boolean mask of pixels passing cleaning """ - mask = self.clean_image( - tel_id=tel_id, - image=event.dl1.tel[tel_id].image, - arrival_times=event.dl1.tel[tel_id].peak_time, - ) - return mask @abstractmethod def clean_image( @@ -537,13 +532,21 @@ class TailcutsImageCleaner(ImageCleaner): "removed.", ).tag(config=True) - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Apply standard picture-boundary cleaning. See `ImageCleaner.__call__()` """ + mask = self.clean_image( + tel_id=tel_id, + image=event.dl1.tel[tel_id].image, + ) + return mask + + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: + return tailcuts_clean( self.subarray.tel[tel_id].camera.geometry, image, @@ -559,13 +562,21 @@ class MARSImageCleaner(TailcutsImageCleaner): 1st-pass MARS-like Image cleaner (See `ctapipe.image.mars_cleaning_1st_pass`) """ - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Apply MARS-style image cleaning. See `ImageCleaner.__call__()` """ + mask = self.clean_image( + tel_id=tel_id, + image=event.dl1.tel[tel_id].image, + ) + return mask + + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: + return mars_cleaning_1st_pass( self.subarray.tel[tel_id].camera.geometry, image, @@ -586,10 +597,20 @@ class FACTImageCleaner(TailcutsImageCleaner): default_value=5.0, help="arrival time limit for neighboring " "pixels, in ns" ).tag(config=True) + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" + + mask = self.clean_image( + tel_id=tel_id, + image=event.dl1.tel[tel_id].image, + arrival_times=event.dl1.tel[tel_id].peak_time, + ) + return mask + def clean_image( self, tel_id: int, image: np.ndarray, arrival_times=None ) -> np.ndarray: - """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" + return fact_image_cleaning( geom=self.subarray.tel[tel_id].camera.geometry, image=image, @@ -615,13 +636,22 @@ class TimeConstrainedImageCleaner(TailcutsImageCleaner): help="arrival time limit for neighboring " "boundary pixels, in ns", ).tag(config=True) - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: + def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: """ Apply MAGIC-like image cleaning with timing information. See `ImageCleaner.__call__()` """ + mask = self.clean_image( + tel_id=tel_id, + image=event.dl1.tel[tel_id].image, + arrival_times=event.dl1.tel[tel_id].peak_time, + ) + return mask + + def clean_image( + self, tel_id: int, image: np.ndarray, arrival_times=None + ) -> np.ndarray: + return time_constrained_clean( self.subarray.tel[tel_id].camera.geometry, image, From 4317c302f068da48bb8888db89cf5afc7d81949d Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 20 Feb 2024 10:36:38 +0100 Subject: [PATCH 09/10] Adding event to __call__ function - Switching from low-level cleaning method approach to just adding event as keyword argument to the __call__ function - backwards compatible now --- docs/changes/2511.api.rst | 9 +- src/ctapipe/image/cleaning.py | 115 +++++++----------- src/ctapipe/image/image_processor.py | 7 +- src/ctapipe/image/reducer.py | 3 +- .../tests/test_image_cleaner_component.py | 7 +- 5 files changed, 58 insertions(+), 83 deletions(-) diff --git a/docs/changes/2511.api.rst b/docs/changes/2511.api.rst index 7e986cf6e66..438966dd31f 100644 --- a/docs/changes/2511.api.rst +++ b/docs/changes/2511.api.rst @@ -1,8 +1,3 @@ -Change the ``ImageCleaner`` API from __call__(self, tel_id, image, peak_time) -to __call__(self, tel_id: int, event: ArrayEventContainer) so cleaning -algorithms can now access relevant information for methods +Adding event as keyword argument to the ``ImageCleaner`` API +so cleaning algorithms can now access relevant information for methods that e.g. require monitoring information. - -The __call__ function now internally uses an abstract -clean_image(self, tel_id, image, peak_time) method -defined by each ImageCleaner subclass. diff --git a/src/ctapipe/image/cleaning.py b/src/ctapipe/image/cleaning.py index 1377005dcdf..8f7994b20b5 100644 --- a/src/ctapipe/image/cleaning.py +++ b/src/ctapipe/image/cleaning.py @@ -28,7 +28,8 @@ import numpy as np -from ..containers import ArrayEventContainer +from ctapipe.containers import ArrayEventContainer + from ..core import TelescopeComponent from ..core.traits import ( BoolTelescopeParameter, @@ -466,30 +467,16 @@ class ImageCleaner(TelescopeComponent): """ @abstractmethod - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: - """ - Calls the relevant functions to identify pixels with signal - and reject those with pure noise. - - Parameters - ---------- - tel_id: int - which telescope id in the subarray is being used (determines - which cut is used) - event: `ctapipe.containers.ArrayEventContainer` - - Returns - ------- - np.ndarray - boolean mask of pixels passing cleaning - """ - - @abstractmethod - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times: np.ndarray = None + def __call__( + self, + tel_id: int, + image: np.ndarray, + arrival_times: np.ndarray = None, + *, + event: ArrayEventContainer = None, ) -> np.ndarray: """ - Abstract cleaning method to be defined by an ImageCleaner subclass. + Identify pixels with signal, and reject those with pure noise. Parameters ---------- @@ -500,12 +487,16 @@ def clean_image( image pixel data corresponding to the camera geometry arrival_times: np.ndarray image of arrival time (not used in this method) + event: `ctapipe.containers.ArrayEventContainer` + ArrayEventContainer to make use of additional parameters + e.g. monitoring data. Returns ------- np.ndarray boolean mask of pixels passing cleaning """ + pass class TailcutsImageCleaner(ImageCleaner): @@ -532,21 +523,18 @@ class TailcutsImageCleaner(ImageCleaner): "removed.", ).tag(config=True) - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def __call__( + self, + tel_id: int, + image: np.ndarray, + arrival_times: np.ndarray = None, + *, + event: ArrayEventContainer = None, + ) -> np.ndarray: """ Apply standard picture-boundary cleaning. See `ImageCleaner.__call__()` """ - mask = self.clean_image( - tel_id=tel_id, - image=event.dl1.tel[tel_id].image, - ) - return mask - - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: - return tailcuts_clean( self.subarray.tel[tel_id].camera.geometry, image, @@ -562,21 +550,18 @@ class MARSImageCleaner(TailcutsImageCleaner): 1st-pass MARS-like Image cleaner (See `ctapipe.image.mars_cleaning_1st_pass`) """ - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def __call__( + self, + tel_id: int, + image: np.ndarray, + arrival_times: np.ndarray = None, + *, + event: ArrayEventContainer = None, + ) -> np.ndarray: """ Apply MARS-style image cleaning. See `ImageCleaner.__call__()` """ - mask = self.clean_image( - tel_id=tel_id, - image=event.dl1.tel[tel_id].image, - ) - return mask - - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: - return mars_cleaning_1st_pass( self.subarray.tel[tel_id].camera.geometry, image, @@ -597,19 +582,15 @@ class FACTImageCleaner(TailcutsImageCleaner): default_value=5.0, help="arrival time limit for neighboring " "pixels, in ns" ).tag(config=True) - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: - """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" - - mask = self.clean_image( - tel_id=tel_id, - image=event.dl1.tel[tel_id].image, - arrival_times=event.dl1.tel[tel_id].peak_time, - ) - return mask - - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None + def __call__( + self, + tel_id: int, + image: np.ndarray, + arrival_times: np.ndarray = None, + *, + event: ArrayEventContainer = None, ) -> np.ndarray: + """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" return fact_image_cleaning( geom=self.subarray.tel[tel_id].camera.geometry, @@ -636,22 +617,18 @@ class TimeConstrainedImageCleaner(TailcutsImageCleaner): help="arrival time limit for neighboring " "boundary pixels, in ns", ).tag(config=True) - def __call__(self, tel_id: int, event: ArrayEventContainer) -> np.ndarray: + def __call__( + self, + tel_id: int, + image: np.ndarray, + arrival_times: np.ndarray = None, + *, + event: ArrayEventContainer = None, + ) -> np.ndarray: """ Apply MAGIC-like image cleaning with timing information. See `ImageCleaner.__call__()` """ - mask = self.clean_image( - tel_id=tel_id, - image=event.dl1.tel[tel_id].image, - arrival_times=event.dl1.tel[tel_id].peak_time, - ) - return mask - - def clean_image( - self, tel_id: int, image: np.ndarray, arrival_times=None - ) -> np.ndarray: - return time_constrained_clean( self.subarray.tel[tel_id].camera.geometry, image, diff --git a/src/ctapipe/image/image_processor.py b/src/ctapipe/image/image_processor.py index fca88aa22dd..5b7d254da78 100644 --- a/src/ctapipe/image/image_processor.py +++ b/src/ctapipe/image/image_processor.py @@ -218,7 +218,12 @@ def _process_telescope_event(self, event): if self.apply_image_modifier.tel[tel_id]: dl1_camera.image = self.modify(tel_id=tel_id, image=dl1_camera.image) - dl1_camera.image_mask = self.clean(tel_id=tel_id, event=event) + dl1_camera.image_mask = self.clean( + tel_id=tel_id, + image=dl1_camera.image, + arrival_times=dl1_camera.peak_time, + event=event, + ) dl1_camera.parameters = self._parameterize_image( tel_id=tel_id, diff --git a/src/ctapipe/image/reducer.py b/src/ctapipe/image/reducer.py index 63a19568f3e..c79f1602216 100644 --- a/src/ctapipe/image/reducer.py +++ b/src/ctapipe/image/reducer.py @@ -1,6 +1,7 @@ """ Algorithms for the data volume reduction. """ + from abc import abstractmethod import numpy as np @@ -191,7 +192,7 @@ def select_pixels(self, waveforms, tel_id=None, selected_gain_channel=None): ) # 1) Step: TailcutCleaning at first - mask = self.cleaner.clean_image(tel_id, dl1.image) + mask = self.cleaner(tel_id, dl1.image) pixels_above_boundary_thresh = ( dl1.image >= self.cleaner.boundary_threshold_pe.tel[tel_id] ) diff --git a/src/ctapipe/image/tests/test_image_cleaner_component.py b/src/ctapipe/image/tests/test_image_cleaner_component.py index ff837e49160..1b8ef34274b 100644 --- a/src/ctapipe/image/tests/test_image_cleaner_component.py +++ b/src/ctapipe/image/tests/test_image_cleaner_component.py @@ -7,7 +7,7 @@ @pytest.mark.parametrize("method", ImageCleaner.non_abstract_subclasses().keys()) -def test_image_cleaner(method, prod5_mst_nectarcam, example_event, reference_location): +def test_image_cleaner(method, prod5_mst_nectarcam, reference_location): """Test that we can construct and use a component-based ImageCleaner""" config = Config( @@ -45,10 +45,7 @@ def test_image_cleaner(method, prod5_mst_nectarcam, example_event, reference_loc image[31:40] = 8.0 times = np.linspace(-5, 10, image.shape[0]) - example_event.dl1.tel[1].image = image - example_event.dl1.tel[1].peak_time = times - - mask = clean(tel_id=1, event=example_event) + mask = clean(tel_id=1, image=image, arrival_times=times) # we're not testing the algorithm here, just that it does something (for the # algorithm tests, see test_cleaning.py From 8a05e27400adabb5095567e873e053744de9064f Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 12 Mar 2024 16:56:05 +0100 Subject: [PATCH 10/10] Switching to MonitoringCameraContainer --- docs/changes/2511.api.rst | 6 +++--- src/ctapipe/image/cleaning.py | 18 +++++++++--------- src/ctapipe/image/image_processor.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/changes/2511.api.rst b/docs/changes/2511.api.rst index 438966dd31f..e847d754d76 100644 --- a/docs/changes/2511.api.rst +++ b/docs/changes/2511.api.rst @@ -1,3 +1,3 @@ -Adding event as keyword argument to the ``ImageCleaner`` API -so cleaning algorithms can now access relevant information for methods -that e.g. require monitoring information. +Adding monitoring: MonitoringCameraContainer as keyword argument to +the ``ImageCleaner`` API so cleaning algorithms can now access +relevant information for methods that e.g. require monitoring information. diff --git a/src/ctapipe/image/cleaning.py b/src/ctapipe/image/cleaning.py index 8f7994b20b5..c5845090254 100644 --- a/src/ctapipe/image/cleaning.py +++ b/src/ctapipe/image/cleaning.py @@ -28,7 +28,7 @@ import numpy as np -from ctapipe.containers import ArrayEventContainer +from ctapipe.containers import MonitoringCameraContainer from ..core import TelescopeComponent from ..core.traits import ( @@ -473,7 +473,7 @@ def __call__( image: np.ndarray, arrival_times: np.ndarray = None, *, - event: ArrayEventContainer = None, + monitoring: MonitoringCameraContainer = None, ) -> np.ndarray: """ Identify pixels with signal, and reject those with pure noise. @@ -487,9 +487,9 @@ def __call__( image pixel data corresponding to the camera geometry arrival_times: np.ndarray image of arrival time (not used in this method) - event: `ctapipe.containers.ArrayEventContainer` - ArrayEventContainer to make use of additional parameters - e.g. monitoring data. + monitoring: `ctapipe.containers.MonitoringCameraContainer` + MonitoringCameraContainer to make use of additional parameters + from monitoring data e.g. pedestal std. Returns ------- @@ -529,7 +529,7 @@ def __call__( image: np.ndarray, arrival_times: np.ndarray = None, *, - event: ArrayEventContainer = None, + monitoring: MonitoringCameraContainer = None, ) -> np.ndarray: """ Apply standard picture-boundary cleaning. See `ImageCleaner.__call__()` @@ -556,7 +556,7 @@ def __call__( image: np.ndarray, arrival_times: np.ndarray = None, *, - event: ArrayEventContainer = None, + monitoring: MonitoringCameraContainer = None, ) -> np.ndarray: """ Apply MARS-style image cleaning. See `ImageCleaner.__call__()` @@ -588,7 +588,7 @@ def __call__( image: np.ndarray, arrival_times: np.ndarray = None, *, - event: ArrayEventContainer = None, + monitoring: MonitoringCameraContainer = None, ) -> np.ndarray: """Apply FACT-style image cleaning. see ImageCleaner.__call__()""" @@ -623,7 +623,7 @@ def __call__( image: np.ndarray, arrival_times: np.ndarray = None, *, - event: ArrayEventContainer = None, + monitoring: MonitoringCameraContainer = None, ) -> np.ndarray: """ Apply MAGIC-like image cleaning with timing information. See `ImageCleaner.__call__()` diff --git a/src/ctapipe/image/image_processor.py b/src/ctapipe/image/image_processor.py index 5b7d254da78..bb810283a71 100644 --- a/src/ctapipe/image/image_processor.py +++ b/src/ctapipe/image/image_processor.py @@ -222,7 +222,7 @@ def _process_telescope_event(self, event): tel_id=tel_id, image=dl1_camera.image, arrival_times=dl1_camera.peak_time, - event=event, + monitoring=event.mon.tel[tel_id], ) dl1_camera.parameters = self._parameterize_image(