Skip to content

Commit

Permalink
Implement CEP-2: new event container structure
Browse files Browse the repository at this point in the history
  • Loading branch information
@maxnoe
maxnoe committed Oct 31, 2023
1 parent 1164e6c commit bd7d777
Show file tree
Hide file tree
Showing 59 changed files with 1,281 additions and 1,214 deletions.
60 changes: 35 additions & 25 deletions ctapipe/calib/camera/calibrator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@
import numpy as np
from numba import float32, float64, guvectorize, int64

from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus
from ctapipe.containers import (
DL0TelescopeContainer,
DL1TelescopeContainer,
PixelStatus,
TelescopeEventContainer,
)
from ctapipe.core import TelescopeComponent
from ctapipe.core.traits import (
BoolTelescopeParameter,
Expand Down Expand Up @@ -153,8 +158,8 @@ def __init__(
parent=self,
)

def _check_r1_empty(self, waveforms):
if waveforms is None:
def _check_r1_empty(self, r1):
if r1 is None or r1.waveform is None:
if not self._r1_empty_warn:
warnings.warn(
"Encountered an event with no R1 data. "
Expand All @@ -165,8 +170,8 @@ def _check_r1_empty(self, waveforms):
else:
return False

def _check_dl0_empty(self, waveforms):
if waveforms is None:
def _check_dl0_empty(self, dl0):
if dl0 is None or dl0.waveform is None:
if not self._dl0_empty_warn:
warnings.warn(
"Encountered an event with no DL0 data. "
Expand All @@ -177,12 +182,13 @@ def _check_dl0_empty(self, waveforms):
else:
return False

def _calibrate_dl0(self, event, tel_id):
r1 = event.r1.tel[tel_id]

if self._check_r1_empty(r1.waveform):
def r1_to_dl0(self, tel_event: TelescopeEventContainer):
if self._check_r1_empty(tel_event.r1):
return

tel_id = tel_event.index.tel_id
r1 = tel_event.r1

signal_pixels = self.data_volume_reducer(
r1.waveform,
tel_id=tel_id,
Expand All @@ -198,7 +204,8 @@ def _calibrate_dl0(self, event, tel_id):
# unset dvr bits for removed pixels
dl0_pixel_status[~signal_pixels] &= ~np.uint8(PixelStatus.DVR_STATUS)

event.dl0.tel[tel_id] = DL0CameraContainer(
# FIXME: trigger information?
tel_event.dl0 = DL0TelescopeContainer(
event_type=r1.event_type,
event_time=r1.event_time,
waveform=dl0_waveform,
Expand All @@ -208,22 +215,24 @@ def _calibrate_dl0(self, event, tel_id):
calibration_monitoring_id=r1.calibration_monitoring_id,
)

def _calibrate_dl1(self, event, tel_id):
waveforms = event.dl0.tel[tel_id].waveform
if self._check_dl0_empty(waveforms):
def dl0_to_dl1(self, tel_event: TelescopeEventContainer):
if self._check_dl0_empty(tel_event.dl0):
return

tel_id = tel_event.index.tel_id
dl0 = tel_event.dl0
waveforms = dl0.waveform
n_pixels, n_samples = waveforms.shape

selected_gain_channel = event.dl0.tel[tel_id].selected_gain_channel
selected_gain_channel = dl0.selected_gain_channel
broken_pixels = _get_invalid_pixels(
n_pixels,
event.mon.tel[tel_id].pixel_status,
tel_event.mon.pixel_status,
selected_gain_channel,
)

dl1_calib = event.calibration.tel[tel_id].dl1
time_shift = event.calibration.tel[tel_id].dl1.time_shift
dl1_calib = tel_event.calibration.dl1
time_shift = dl1_calib.time_shift
readout = self.subarray.tel[tel_id].camera.readout

# subtract any remaining pedestal before extraction
Expand All @@ -239,7 +248,7 @@ def _calibrate_dl1(self, event, tel_id):
# - Read into dl1 container directly?
# - Don't do anything if dl1 container already filled
# - Update on SST review decision
dl1 = DL1CameraContainer(
dl1 = DL1TelescopeContainer(
image=waveforms[..., 0].astype(np.float32),
peak_time=np.zeros(n_pixels, dtype=np.float32),
is_valid=True,
Expand Down Expand Up @@ -283,7 +292,7 @@ def _calibrate_dl1(self, event, tel_id):
)

# store the results in the event structure
event.dl1.tel[tel_id] = dl1
tel_event.dl1 = dl1

def __call__(self, event):
"""
Expand All @@ -294,13 +303,14 @@ def __call__(self, event):
Parameters
----------
event : container
A `~ctapipe.containers.ArrayEventContainer` event container
A `~ctapipe.containers.SubarrayEventContainer` event container
"""
# TODO: How to handle different calibrations depending on tel_id?
tel = event.r1.tel or event.dl0.tel or event.dl1.tel
for tel_id in tel.keys():
self._calibrate_dl0(event, tel_id)
self._calibrate_dl1(event, tel_id)
for tel_event in event.tel.values():
self.calibrate_tel_event(tel_event)

def calibrate_tel_event(self, tel_event):
self.r1_to_dl0(tel_event)
self.dl0_to_dl1(tel_event)


def shift_waveforms(waveforms, time_shift_samples):
Expand Down
28 changes: 14 additions & 14 deletions ctapipe/calib/camera/flatfield.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import numpy as np
from astropy import units as u

from ctapipe.containers import DL1CameraContainer
from ctapipe.containers import DL1TelescopeContainer
from ctapipe.core import Component
from ctapipe.core.traits import Int, List, Unicode
from ctapipe.image.extractor import ImageExtractor
Expand Down Expand Up @@ -108,7 +108,7 @@ def calculate_relative_gain(self, event):
Parameters
----------
event: ctapipe.containers.ArrayEventContainer
event: ctapipe.containers.SubarrayEventContainer
Returns: True if the mon.tel[tel_id].flatfield is updated,
False otherwise
Expand Down Expand Up @@ -169,7 +169,7 @@ def __init__(self, **kwargs):
self.arrival_times = None # arrival time per event in sample
self.sample_masked_pixels = None # masked pixels per event in sample

def _extract_charge(self, event) -> DL1CameraContainer:
def _extract_charge(self, event) -> DL1TelescopeContainer:
"""
Extract the charge and the time from a calibration event
Expand All @@ -182,11 +182,11 @@ def _extract_charge(self, event) -> DL1CameraContainer:
DL1CameraContainer
"""

waveforms = event.r1.tel[self.tel_id].waveform
selected_gain_channel = event.r1.tel[self.tel_id].selected_gain_channel
waveforms = event.tel[self.tel_id].r1.waveform
selected_gain_channel = event.tel[self.tel_id].r1.selected_gain_channel
broken_pixels = _get_invalid_pixels(
n_pixels=waveforms.shape[-2],
pixel_status=event.mon.tel[self.tel_id].pixel_status,
pixel_status=event.tel[self.tel_id].mon.pixel_status,
selected_gain_channel=selected_gain_channel,
)
# Extract charge and time
Expand All @@ -195,7 +195,7 @@ def _extract_charge(self, event) -> DL1CameraContainer:
waveforms, self.tel_id, selected_gain_channel, broken_pixels
)
else:
return DL1CameraContainer(image=0, peak_pos=0, is_valid=False)
return DL1TelescopeContainer(image=0, peak_pos=0, is_valid=False)

def calculate_relative_gain(self, event):
"""
Expand All @@ -209,23 +209,23 @@ def calculate_relative_gain(self, event):
"""

# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
container = event.mon.tel[self.tel_id].flatfield
waveform = event.tel[self.tel_id].r1.waveform
container = event.tel[self.tel_id].mon.flatfield

# re-initialize counter
if self.n_events_seen == self.sample_size:
self.n_events_seen = 0

# real data
trigger_time = event.trigger.time
trigger_time = event.dl0.trigger.time
if event.meta["origin"] != "hessio":
hardware_or_pedestal_mask = np.logical_or(
event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels,
event.mon.tel[self.tel_id].pixel_status.pedestal_failing_pixels,
event.tel[self.tel_id].mon.pixel_status.hardware_failing_pixels,
event.tel[self.tel_id].mon.pixel_status.pedestal_failing_pixels,
)
pixel_mask = np.logical_or(
hardware_or_pedestal_mask,
event.mon.tel[self.tel_id].pixel_status.flatfield_failing_pixels,
event.tel[self.tel_id].mon.pixel_status.flatfield_failing_pixels,
)

else: # patches for MC data
Expand All @@ -237,7 +237,7 @@ def calculate_relative_gain(self, event):

# extract the charge of the event and
# the peak position (assumed as time for the moment)
dl1: DL1CameraContainer = self._extract_charge(event)
dl1: DL1TelescopeContainer = self._extract_charge(event)

if not dl1.is_valid:
return False
Expand Down
28 changes: 14 additions & 14 deletions ctapipe/calib/camera/pedestals.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import numpy as np
from astropy import units as u

from ctapipe.containers import DL1CameraContainer
from ctapipe.containers import DL1TelescopeContainer
from ctapipe.core import Component
from ctapipe.core.traits import Int, List, Unicode
from ctapipe.image.extractor import ImageExtractor
Expand Down Expand Up @@ -134,7 +134,7 @@ def calculate_pedestals(self, event):
Parameters
----------
event: ctapipe.containers.ArrayEventContainer
event: ctapipe.containers.SubarrayEventContainer
Returns: True if the mon.tel[tel_id].pedestal is updated,
False otherwise
Expand Down Expand Up @@ -197,24 +197,24 @@ def __init__(self, **kwargs):
self.charges = None # charge per event in sample
self.sample_masked_pixels = None # pixels tp be masked per event in sample

def _extract_charge(self, event) -> DL1CameraContainer:
def _extract_charge(self, event) -> DL1TelescopeContainer:
"""
Extract the charge and the time from a pedestal event
Parameters
----------
event: ArrayEventContainer
event: SubarrayEventContainer
general event container
Returns
-------
DL1CameraContainer
DL1TelescopeContainer
"""
waveforms = event.r1.tel[self.tel_id].waveform
selected_gain_channel = event.r1.tel[self.tel_id].selected_gain_channel
waveforms = event.tel[self.tel_id].r1.waveform
selected_gain_channel = event.tel[self.tel_id].r1.selected_gain_channel
broken_pixels = _get_invalid_pixels(
n_pixels=waveforms.shape[-2],
pixel_status=event.mon.tel[self.tel_id].pixel_status,
pixel_status=event.tel[self.tel_id].mon.pixel_status,
selected_gain_channel=selected_gain_channel,
)

Expand All @@ -224,7 +224,7 @@ def _extract_charge(self, event) -> DL1CameraContainer:
waveforms, self.tel_id, selected_gain_channel, broken_pixels
)
else:
return DL1CameraContainer(image=0, peak_pos=0, is_valid=False)
return DL1TelescopeContainer(image=0, peak_pos=0, is_valid=False)

def calculate_pedestals(self, event):
"""
Expand All @@ -238,17 +238,17 @@ def calculate_pedestals(self, event):
"""
# initialize the np array at each cycle
waveform = event.r1.tel[self.tel_id].waveform
container = event.mon.tel[self.tel_id].pedestal
waveform = event.tel[self.tel_id].r1.waveform
container = event.tel[self.tel_id].mon.pedestal

# re-initialize counter
if self.n_events_seen == self.sample_size:
self.n_events_seen = 0

# real data
trigger_time = event.trigger.time
trigger_time = event.dl0.trigger.time
if event.meta["origin"] != "hessio":
pixel_mask = event.mon.tel[self.tel_id].pixel_status.hardware_failing_pixels
pixel_mask = event.tel[self.tel_id].mon.pixel_status.hardware_failing_pixels
else: # patches for MC data
pixel_mask = np.zeros(waveform.shape[1], dtype=bool)

Expand All @@ -258,7 +258,7 @@ def calculate_pedestals(self, event):

# extract the charge of the event and
# the peak position (assumed as time for the moment)
dl1: DL1CameraContainer = self._extract_charge(event)
dl1: DL1TelescopeContainer = self._extract_charge(event)

if not dl1.is_valid:
return False
Expand Down
Loading

0 comments on commit bd7d777

Please sign in to comment.