Skip to content

Commit

Permalink
Merge pull request #2375 from cta-observatory/only_triggers
Browse files Browse the repository at this point in the history 
Remove dump-triggers, enable ctapipe-process to store only trigger and simulation data
  • Loading branch information
@kosack
kosack authored Jul 6, 2023
2 parents e3705aa + ea996fe commit c44226a
Show file tree
Hide file tree
Showing 7 changed files with 75 additions and 208 deletions.
99 changes: 47 additions & 52 deletions ctapipe/io/datawriter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from .eventsource import EventSource
from .hdf5tableio import HDF5TableWriter
from .simteleventsource import SimTelEventSource
from .tableio import FixedPointColumnTransform, TableWriter, TelListToMaskTransform
from .tableio import FixedPointColumnTransform, TelListToMaskTransform

__all__ = ["DataWriter", "DATA_MODEL_VERSION", "write_reference_metadata_headers"]

Expand Down Expand Up @@ -298,15 +298,11 @@ def __call__(self, event: ArrayEventContainer):
Write a single event to the output file.
"""
self._at_least_one_event = True
self.log.debug("WRITING EVENT %s", event.index)

# Write subarray event data
self._write_subarray_pointing(event, writer=self._writer)
self._write_subarray_pointing(event)
self._write_trigger(event)

self.log.debug("WRITING EVENT %s", event.index)
self._writer.write(
table_name="dl1/event/subarray/trigger",
containers=[event.index, event.trigger],
)
if event.simulation is not None and event.simulation.shower is not None:
self._writer.write(
table_name="simulation/event/subarray/shower",
Expand All @@ -322,21 +318,21 @@ def __call__(self, event: ArrayEventContainer):
)

if self.write_waveforms:
self._write_r1_telescope_events(self._writer, event)
self._write_r1_telescope_events(event)

if self.write_raw_waveforms:
self._write_r0_telescope_events(self._writer, event)
self._write_r0_telescope_events(event)

# write telescope event data
self._write_dl1_telescope_events(self._writer, event)
self._write_dl1_telescope_events(event)

# write DL2 info if requested
if self.write_showers:
self._write_dl2_telescope_events(self._writer, event)
self._write_dl2_stereo_event(self._writer, event)
self._write_dl2_telescope_events(event)
self._write_dl2_stereo_event(event)

if self.write_muon_parameters:
self._write_muon_telescope_events(self._writer, event)
self._write_muon_telescope_events(event)

def finish(self):
"""called after all events are done"""
Expand All @@ -358,7 +354,6 @@ def finish(self):
)

self._write_context_metadata_headers()

self._writer.close()

@property
Expand Down Expand Up @@ -415,8 +410,9 @@ def _setup_output_path(self):
self.write_muon_parameters,
]
if not any(writable_things):
raise ToolConfigurationError(
"DataWriter configured to write no information"
self.log.warning(
"No processing results were selected for writing"
", only writing trigger and simulation information"
)

def _setup_writer(self):
Expand Down Expand Up @@ -507,13 +503,13 @@ def _setup_writer(self):
self._writer = writer
self.log.debug("Writer initialized: %s", self._writer)

def _write_subarray_pointing(self, event: ArrayEventContainer, writer: TableWriter):
def _write_subarray_pointing(self, event: ArrayEventContainer):
"""store subarray pointing info in a monitoring table"""
pnt = event.pointing
current_pointing = (pnt.array_azimuth, pnt.array_altitude)
if current_pointing != self._last_pointing:
pnt.prefix = ""
writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt])
self._writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt])
self._last_pointing = current_pointing

def _write_scheduling_and_observation_blocks(self):
Expand Down Expand Up @@ -616,49 +612,50 @@ def table_name(self, tel_id):
"""construct dataset table names depending on chosen split method"""
return f"tel_{tel_id:03d}"

def _write_r1_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_trigger(self, event: ArrayEventContainer):
"""
Write trigger information
"""
self._writer.write(
table_name="dl1/event/subarray/trigger",
containers=[event.index, event.trigger],
)

for tel_id, trigger in event.trigger.tel.items():
self._writer.write(
"dl1/event/telescope/trigger", (_get_tel_index(event, tel_id), trigger)
)

def _write_r1_telescope_events(self, event: ArrayEventContainer):
for tel_id, r1_tel in event.r1.tel.items():

tel_index = _get_tel_index(event, tel_id)
table_name = self.table_name(tel_id)

r1_tel.prefix = ""
writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel])
self._writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel])

def _write_r0_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_r0_telescope_events(self, event: ArrayEventContainer):
for tel_id, r0_tel in event.r0.tel.items():

tel_index = _get_tel_index(event, tel_id)
table_name = self.table_name(tel_id)

r0_tel.prefix = ""
writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel])
self._writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel])

def _write_dl1_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_dl1_telescope_events(self, event: ArrayEventContainer):
"""
add entries to the event/telescope tables for each telescope in a single
event
"""

# write the telescope tables
# trigger info
for tel_id, trigger in event.trigger.tel.items():
writer.write(
"dl1/event/telescope/trigger", [_get_tel_index(event, tel_id), trigger]
)

# pointing info
for tel_id, pnt in event.pointing.tel.items():
current_pointing = (pnt.azimuth, pnt.altitude)
if current_pointing != self._last_pointing_tel[tel_id]:
pnt.prefix = ""
writer.write(
self._writer.write(
f"dl1/monitoring/telescope/pointing/tel_{tel_id:03d}",
[event.trigger.tel[tel_id], pnt],
)
Expand All @@ -674,7 +671,7 @@ def _write_dl1_telescope_events(
table_name = self.table_name(tel_id)

if self.write_parameters:
writer.write(
self._writer.write(
table_name=f"dl1/event/telescope/parameters/{table_name}",
containers=[tel_index, *dl1_camera.parameters.values()],
)
Expand All @@ -685,14 +682,14 @@ def _write_dl1_telescope_events(
"DataWriter.write_images is True but event does not contain image"
)

writer.write(
self._writer.write(
table_name=f"dl1/event/telescope/images/{table_name}",
containers=[tel_index, dl1_camera],
)

if self._is_simulation:
# always write this, so that at least the sum is included
writer.write(
self._writer.write(
f"simulation/event/telescope/images/{table_name}",
[tel_index, event.simulation.tel[tel_id]],
)
Expand All @@ -705,7 +702,7 @@ def _write_dl1_telescope_events(
true_parameters = event.simulation.tel[tel_id].true_parameters
# only write the available containers, no peak time related
# features for true image available.
writer.write(
self._writer.write(
f"simulation/event/telescope/parameters/{table_name}",
[
tel_index,
Expand All @@ -717,21 +714,17 @@ def _write_dl1_telescope_events(
],
)

def _write_muon_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_muon_telescope_events(self, event: ArrayEventContainer):

for tel_id, muon in event.muon.tel.items():
table_name = self.table_name(tel_id)
tel_index = _get_tel_index(event, tel_id)
writer.write(
self._writer.write(
f"dl1/event/telescope/muon/{table_name}",
[tel_index, muon.ring, muon.parameters, muon.efficiency],
)

def _write_dl2_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_dl2_telescope_events(self, event: ArrayEventContainer):
"""
write per-telescope DL2 shower information.
Expand All @@ -749,9 +742,11 @@ def _write_dl2_telescope_events(
f"dl2/event/telescope/{container_name}/{algorithm}/{table_name}"
)

writer.write(table_name=name, containers=[tel_index, container])
self._writer.write(
table_name=name, containers=[tel_index, container]
)

def _write_dl2_stereo_event(self, writer: TableWriter, event: ArrayEventContainer):
def _write_dl2_stereo_event(self, event: ArrayEventContainer):
"""
write per-telescope DL2 shower information to e.g.
`/dl2/event/stereo/{geometry,energy,classification}/<algorithm_name>`
Expand All @@ -762,7 +757,7 @@ def _write_dl2_stereo_event(self, writer: TableWriter, event: ArrayEventContaine
# note this will only write info if the particular algorithm
# generated it (otherwise the algorithm map is empty, and no
# data will be written)
writer.write(
self._writer.write(
table_name=f"dl2/event/subarray/{container_name}/{algorithm}",
containers=[event.index, container],
)
Expand Down
142 changes: 0 additions & 142 deletions ctapipe/tools/dump_triggers.py
View file

This file was deleted.

1 change: 1 addition & 0 deletions ctapipe/tools/process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ def start(self):
"""
Process events
"""
self.log.info("applying calibration: %s", self.should_calibrate)
self.log.info("(re)compute DL1: %s", self.should_compute_dl1)
self.log.info("(re)compute DL2: %s", self.should_compute_dl2)
self.log.info(
Expand Down
Loading

0 comments on commit c44226a

Please sign in to comment.