Merge branch 'cta-observatory:main' into irf-tool

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.9"
       - uses: pre-commit/[email protected]
         with:
           extra_args: --files $(git diff origin/main --name-only)
@@ -33,10 +33,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-latest
-            python-version: "3.8"
-            install-method: mamba
-
           - os: ubuntu-latest
             python-version: "3.9"
             install-method: mamba
@@ -47,15 +43,19 @@ jobs:
             extra-args: ["codecov"]
 
           - os: ubuntu-latest
-            python-version: "3.10"
+            python-version: "3.11"
+            install-method: mamba
+
+          - os: ubuntu-latest
+            python-version: "3.11"
             install-method: pip
 
           - os: macos-latest
-            python-version: "3.10"
+            python-version: "3.11"
             install-method: mamba
 
           - os: macos-latest
-            python-version: "3.8"
+            python-version: "3.9"
             install-method: pip
 
     defaults:
@@ -84,7 +84,10 @@ jobs:
 
       - name: mamba setup
         if: matrix.install-method == 'mamba'
-        uses: mamba-org/provision-with-micromamba@v14
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          cache-downloads: true
 
       - name: Python setup
         if: matrix.install-method == 'pip'
@@ -143,7 +146,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install doc dependencies
         run: |

.gitignore

@@ -12,6 +12,7 @@ __pycache__
 ctapipe/_version_cache.py
 ctapipe/_version.py
 
+
 # Ignore .c files by default to avoid including generated code. If you want to
 # add a non-generated .c extension, use `git add -f filename.c`.
 *.c
@@ -25,6 +26,7 @@ MANIFEST
 # Sphinx
 docs/api
 docs/_build
+docs/auto_examples
 
 # Editors and IDEs
 
@@ -86,7 +88,4 @@ distribute-*.tar.gz
 target
 .mypy_cache
 
-examples/notebooks/*.html
-examples/notebooks/*.png
-
 provenance.log

.mailmap

@@ -88,3 +88,6 @@ Orel Gueta <[email protected]> orelgueta <[email protected]
 Rune Michael Dominik <[email protected]>
 
 Henning Ptaszyk <[email protected]>
+
+
+Tomas Bylund <[email protected]> Tomas Bylund <[email protected]>

.readthedocs.yml

@@ -4,8 +4,9 @@ build:
   os: ubuntu-22.04
   apt_packages:
     - ffmpeg
+    - graphviz
   tools:
-    python: "3.8"
+    python: "3.9"
 
 python:
   install:

CHANGES.rst

@@ -1,3 +1,163 @@
+ctapipe v0.20.0 (2023-09-11)
+============================
+
+
+API Changes
+-----------
+
+- The ``ctapipe-dump-triggers`` tool was removed, since it wrote a custom data format
+  not compatble with e.g. the output of the ``DataWriter`` and ``ctapipe-process``.
+  If you only want to store trigger and simulation information from simulated / DL0
+  input files into the ctapipe format HDF5 files, you can now use
+  ``ctapipe-process -i <input> -o <output> --no-write-parameters``. [`#2375 <https://github.com/cta-observatory/ctapipe/pull/2375>`__]
+
+- Change the fill value for invalid telescope ids in ``SubarrayDescription.tel_index_array``
+  from ``-1`` to ``np.iinfo(int).minval`` to prevent ``-1`` being used as an index resulting in the last element being used for invalid telescope ids. [`#2376 <https://github.com/cta-observatory/ctapipe/pull/2376>`__]
+
+- Remove ``EventSource.from_config``, simply use ``EventSource(config=config)`` or 
+  ``EventSource(parent=parent)``. [`#2384 <https://github.com/cta-observatory/ctapipe/pull/2384>`__]
+
+
+Data Model Changes
+------------------
+
+- Added missing fields defined in the CTAO R1 and DL0 data models to the corresponding containers. [`#2338 <https://github.com/cta-observatory/ctapipe/pull/2338>`__]
+
+- Remove the ``injection_height`` field from the ``SimulationConfigContainer``,
+  this field was always empty and is never filled by ``sim_telarray``.
+
+  Add the corresponding ``starting_grammage`` field to the ``SimulatedShowerContainer``,
+  where it is actually available. [`#2343 <https://github.com/cta-observatory/ctapipe/pull/2343>`__]
+
+- Added new fields to the ``MuonEfficiencyContainer`` - ``is_valid`` to check if fit converged successfully, ``parameters_at_limit`` to check if parameters were fitted close to a bound and ``likelihood_value`` which represents cost function value atthe minimum. These fields were added to the output of the ``MuonIntensityFitter``. [`#2381 <https://github.com/cta-observatory/ctapipe/pull/2381>`__]
+
+
+New Features
+------------
+
+- Remove writing the full provenance information to the log  and instead simply refer the reader to the actual provenance file. [`#2328 <https://github.com/cta-observatory/ctapipe/pull/2328>`__]
+
+- Add support for including r1 and r0 waveforms in the ``ctapipe-merge`` tool. [`#2386 <https://github.com/cta-observatory/ctapipe/pull/2386>`__]
+
+
+Bug Fixes
+---------
+
+- The ```HillasIntersection``` method used to fail when individual events were reconstructed to originate from a FoV offset of more than 90 degrees.
+  This is now fixed by returning an INVALID container for a reconstructed offset of larger than 45 degrees. [`#2265 <https://github.com/cta-observatory/ctapipe/pull/2265>`__]
+
+
+Maintenance
+-----------
+
+- Drop support for python 3.8 in accordance with the NEP 29 schedule. [`#2342 <https://github.com/cta-observatory/ctapipe/pull/2342>`__]
+
+- * Switched to ``PyData`` theme for docs
+  * Updated ``Sphinx`` to version 6.2.1
+  * Updated front page of docs [`#2373 <https://github.com/cta-observatory/ctapipe/pull/2373>`__]
+
+
+
+ctapipe 0.19.3 (2023-06-20)
+===========================
+
+This is a bugfix release fixing a number of bugs, mainly one preventing the processing of divergent pointing
+prod6 data due to a bug in ``SoftwareTrigger``, see below for details.
+
+
+Bug Fixes
+---------
+
+- Fix peak time units of FlashCamExtractor (See https://github.com/cta-observatory/ctapipe/issues/2336) [`#2337 <https://github.com/cta-observatory/ctapipe/pull/2337>`__]
+
+- Fix shape of mask returned by ``NullDataVolumeReducer``. [`#2340 <https://github.com/cta-observatory/ctapipe/pull/2340>`__]
+
+- Fix definition of the ``--dl2-subarray`` flag of ``ctapipe-merge``. [`#2341 <https://github.com/cta-observatory/ctapipe/pull/2341>`__]
+
+- Fix ``ctapipe-train-disp-reconstructor --help`` raising an exception. [`#2352 <https://github.com/cta-observatory/ctapipe/pull/2352>`__]
+
+- Correctly fill ``reference_location`` for ``SubarrayDescription.tel_coords``. [`#2354 <https://github.com/cta-observatory/ctapipe/pull/2354>`__]
+
+- Fix ``SoftwareTrigger`` not removing all parts of a removed telescope event
+  from the array event leading to invalid files produced by ``DataWriter``. [`#2357 <https://github.com/cta-observatory/ctapipe/pull/2357>`__]
+
+- Fix that the pixel picker of the matplotlib ``CameraDisplay`` triggers
+  also for clicks on other ``CameraDisplay`` instances in the same figure. [`#2358 <https://github.com/cta-observatory/ctapipe/pull/2358>`__]
+
+
+New Features
+------------
+
+- Add support for Hillas parameters in ``TelescopeFrame`` to
+  ``CameraDisplay.overlay_moments`` and make sure that the
+  label text does not overlap with the ellipse. [`#2347 <https://github.com/cta-observatory/ctapipe/pull/2347>`__]
+
+- Add support for using ``ctapipe.image.toymodel`` features in ``TelescopeFrame``. [`#2349 <https://github.com/cta-observatory/ctapipe/pull/2349>`__]
+
+
+Maintenance
+-----------
+
+- Improve docstring and validation of parameters of ``CameraGeometry``. [`#2361 <https://github.com/cta-observatory/ctapipe/pull/2361>`__]
+
+
+
+ctapipe v0.19.2 (2023-05-17)
+============================
+
+This release contains a critical bugfix for the ``FlashCamExtractor`` that resulted
+in non-sensical peak time values in DL1, see below.
+
+Bug Fixes
+---------
+
+- Fix a bug in the peak_time estimation of ``FlashCamExtractor`` (See issue `#2332 <https://github.com/cta-observatory/ctapipe/issues/2332>`_) [`#2333 <https://github.com/cta-observatory/ctapipe/pull/2333>`__]
+
+
+ctapipe v0.19.1 (2023-05-11)
+============================
+
+This release is a small bugfix release for v0.19.0, that also includes a new feature enabling computing different
+telescope multiplicities in the machine learning feature generation.
+
+Thanks to the release of numba 0.57 and some minor fixes, ctapipe is now also compatible with Python 3.11.
+
+Bug Fixes
+---------
+
+- Fix ``ApplyModels.overwrite``. [`#2311 <https://github.com/cta-observatory/ctapipe/pull/2311>`__]
+
+- Fix for config files not being included as inputs in provenance log. [`#2312 <https://github.com/cta-observatory/ctapipe/pull/2312>`__]
+
+- Fix calculation of the neighbor matrix of ``CameraGeometry`` for empty and single-pixel geometries. [`#2317 <https://github.com/cta-observatory/ctapipe/pull/2317>`__]
+
+- Fix HDF5Writer not working on windows due to using pathlib for hdf5 dataset names. [`#2319 <https://github.com/cta-observatory/ctapipe/pull/2319>`__]
+
+- Fix StereoTrigger assuming the wrong data type for ``tels_with_trigger``, resulting in
+  it not working for actual events read from an EventSource. [`#2320 <https://github.com/cta-observatory/ctapipe/pull/2320>`__]
+
+- Allow disabling the cross validation (by setting ``CrossValidator.n_cross_validations = 0``)
+  for the train tools. [`#2310 <https://github.com/cta-observatory/ctapipe/pull/2310>`__]
+
+
+New Features
+------------
+
+- Add ``SubarrayDescription.mulitplicity`` method that can compute
+  telescope multiplicity for a given telescope boolean mask, either for
+  all telescope or a given telescope type.
+
+  Enable adding additional keyword arguments to ``FeatureGenerator``.
+
+  Pass the ``SubarrayDescription`` to ``FeatureGenerator`` in sklearn classes. [`#2308 <https://github.com/cta-observatory/ctapipe/pull/2308>`__]
+
+
+Maintenance
+-----------
+
+- Add support for python 3.11. [`#2107 <https://github.com/cta-observatory/ctapipe/pull/2107>`__]
+
+
 ctapipe v0.19.0 (2023-03-30)
 ============================
 

README.rst

@@ -71,11 +71,11 @@ or via::
 
   pip install ctapipe
 
-**Note**: to install a specific version of ctapipe take look at the documentation `here <https://cta-observatory.github.io/ctapipe/getting_started_users>`__.
+**Note**: to install a specific version of ctapipe take look at the documentation `here <https://ctapipe.readthedocs.org/en/latest/getting_started_users/>`__.
 
 **Note**: ``mamba`` is a C++ reimplementation of conda and can be found `here <https://github.com/mamba-org/mamba>`__.
 
 Note this is *pre-alpha* software and is not yet stable enough for end-users (expect large API changes until the first stable 1.0 release).
 
 Developers should follow the development install instructions found in the
-`documentation <https://cta-observatory.github.io/ctapipe/getting_started>`__.
+`documentation <https://ctapipe.readthedocs.org/en/latest/getting_started/>`__.

ctapipe/calib/__init__.py

@@ -1,7 +1,10 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 """
-Calibration
+Module for calibration code
 """
 from .camera import CameraCalibrator, GainSelector
 
-__all__ = ["CameraCalibrator", "GainSelector"]
+__all__ = [
+    "CameraCalibrator",
+    "GainSelector",
+]

ctapipe/calib/camera/__init__.py

@@ -3,10 +3,10 @@
 Camera calibration module.
 """
 
-from .calibrator import CameraCalibrator
-from .gainselection import GainSelector
+from .calibrator import CameraCalibrator  # noqa: F401
+from .gainselection import GainSelector  # noqa: F401
 
 __all__ = [
-    "CameraCalibrator",
-    "GainSelector",
+    # "CameraCalibrator",
+    # "GainSelector",
 ]

ctapipe/calib/camera/calibrator.py

@@ -9,7 +9,7 @@
 import numpy as np
 from numba import float32, float64, guvectorize, int64
 
-from ctapipe.containers import DL1CameraContainer
+from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus
 from ctapipe.core import TelescopeComponent
 from ctapipe.core.traits import (
     BoolTelescopeParameter,
@@ -178,19 +178,35 @@ def _check_dl0_empty(self, waveforms):
             return False
 
     def _calibrate_dl0(self, event, tel_id):
-        waveforms = event.r1.tel[tel_id].waveform
-        selected_gain_channel = event.r1.tel[tel_id].selected_gain_channel
-        if self._check_r1_empty(waveforms):
+        r1 = event.r1.tel[tel_id]
+
+        if self._check_r1_empty(r1.waveform):
             return
 
-        reduced_waveforms_mask = self.data_volume_reducer(
-            waveforms, tel_id=tel_id, selected_gain_channel=selected_gain_channel
+        signal_pixels = self.data_volume_reducer(
+            r1.waveform,
+            tel_id=tel_id,
+            selected_gain_channel=r1.selected_gain_channel,
         )
 
-        waveforms_copy = waveforms.copy()
-        waveforms_copy[~reduced_waveforms_mask] = 0
-        event.dl0.tel[tel_id].waveform = waveforms_copy
-        event.dl0.tel[tel_id].selected_gain_channel = selected_gain_channel
+        dl0_waveform = r1.waveform.copy()
+        dl0_waveform[~signal_pixels] = 0
+
+        dl0_pixel_status = r1.pixel_status.copy()
+        # set dvr pixel bit in pixel_status for pixels kept by DVR
+        dl0_pixel_status[signal_pixels] |= PixelStatus.DVR_STORED_AS_SIGNAL
+        # unset dvr bits for removed pixels
+        dl0_pixel_status[~signal_pixels] &= ~np.uint8(PixelStatus.DVR_STATUS)
+
+        event.dl0.tel[tel_id] = DL0CameraContainer(
+            event_type=r1.event_type,
+            event_time=r1.event_time,
+            waveform=dl0_waveform,
+            selected_gain_channel=r1.selected_gain_channel,
+            pixel_status=dl0_pixel_status,
+            first_cell_id=r1.first_cell_id,
+            calibration_monitoring_id=r1.calibration_monitoring_id,
+        )
 
     def _calibrate_dl1(self, event, tel_id):
         waveforms = event.dl0.tel[tel_id].waveform

ctapipe/compat.py

@@ -0,0 +1,17 @@
+"""
+Module for python version compatibility
+"""
+import sys
+
+__all__ = [
+    "StrEnum",
+]
+
+
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from enum import Enum
+
+    class StrEnum(str, Enum):
+        """Compatibility backfill of StrEnum for python < 3.11"""