diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07eac48a..145ebee5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,12 @@ name: CI -on: [push, pull_request] +on: + push: + branches: + - master + tags: + - '**' + pull_request: env: NUMBA_NUM_THREADS: 1 @@ -12,11 +18,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8"] - ctapipe-version: ["v0.12.0", ] + python-version: ["3.8", "3.9", "3.10"] + ctapipe-version: ["v0.17.0", ] + + defaults: + run: + shell: bash -exl {0} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -35,15 +45,19 @@ jobs: environment-file: environment.yml - name: Install - shell: bash -l {0} env: CTAPIPE_VERSION: ${{ matrix.ctapipe-version }} run: | + pip install -e . # we install ctapipe using pip to be able to select any commit, e.g. the current master pip install pytest-cov "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION" - pip install -e . git describe --tags + - name: Test Plugin + run: | + # check the LSTEventSource is available for LST + python eventsource_subclasses.py | grep LSTEventSource + - name: Download test data env: TEST_DATA_USER: ${{ secrets.test_data_user }} @@ -52,10 +66,7 @@ jobs: ./download_test_data.sh - name: Tests - shell: bash -l {0} run: | - # github actions starts a new shell for each "step", so we need to - # activate our env again pytest --cov=ctapipe_io_lst --cov-report=xml - uses: codecov/codecov-action@v1 diff --git a/.gitignore b/.gitignore index db76441a..794c6064 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,8 @@ test_data __pycache__ # ignore version cache file (generated automatically when setup.py is run) -ctapipe_io_lst/_version.py -ctapipe_io_lst/_version_cache.py +src/ctapipe_io_lst/_version.py +src/ctapipe_io_lst/_version_cache.py # Ignore .c files by default to avoid including generated code. If you want to # add a non-generated .c extension, use `git add -f filename.c`. diff --git a/MANIFEST.in b/MANIFEST.in index 85ed0a85..193ce6f3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,4 @@ -include README.rst -include setup.cfg - -prune ctapipe_io_lst/_dev_version +prune src/ctapipe_io_lst/_dev_version prune .github exclude .gitignore exclude .codacy.yml diff --git a/ctapipe_io_lst/constants.py b/ctapipe_io_lst/constants.py deleted file mode 100644 index f002b4d4..00000000 --- a/ctapipe_io_lst/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -import numpy as np - -N_GAINS = 2 -N_MODULES = 265 -N_PIXELS_MODULE = 7 -N_PIXELS = N_MODULES * N_PIXELS_MODULE -N_CAPACITORS_CHANNEL = 1024 -# 4 drs4 channels are cascaded for each pixel -N_CAPACITORS_PIXEL = 4 * N_CAPACITORS_CHANNEL -N_SAMPLES = 40 -HIGH_GAIN = 0 -LOW_GAIN = 1 -CLOCK_FREQUENCY_KHZ = 133e3 - -# we have 8 channels per module, but only 7 are used. -N_CHANNELS_MODULE = 8 - -# First capacitor order according Dragon v5 board data format -CHANNEL_ORDER_HIGH_GAIN = [0, 0, 1, 1, 2, 2, 3] -CHANNEL_ORDER_LOW_GAIN = [4, 4, 5, 5, 6, 6, 7] - -PIXEL_INDEX = np.arange(N_PIXELS) diff --git a/ctapipe_io_lst/resources/LSTCam-003.camgeom.fits.gz b/ctapipe_io_lst/resources/LSTCam-003.camgeom.fits.gz deleted file mode 100644 index 1c03a48f..00000000 Binary files a/ctapipe_io_lst/resources/LSTCam-003.camgeom.fits.gz and /dev/null differ diff --git a/ctapipe_io_lst/resources/LSTCam-004.camgeom.fits.gz b/ctapipe_io_lst/resources/LSTCam-004.camgeom.fits.gz deleted file mode 100644 index 58d6e988..00000000 Binary files a/ctapipe_io_lst/resources/LSTCam-004.camgeom.fits.gz and /dev/null differ diff --git a/ctapipe_io_lst/tests/resources/calibration.Run2462.0000.hdf5 b/ctapipe_io_lst/tests/resources/calibration.Run2462.0000.hdf5 deleted file mode 100644 index 113ec284..00000000 Binary files a/ctapipe_io_lst/tests/resources/calibration.Run2462.0000.hdf5 and /dev/null differ diff --git a/ctapipe_io_lst/tests/resources/drs4_pedestal.Run2460.0000.fits.gz b/ctapipe_io_lst/tests/resources/drs4_pedestal.Run2460.0000.fits.gz deleted file mode 100644 index d863e4e6..00000000 Binary files a/ctapipe_io_lst/tests/resources/drs4_pedestal.Run2460.0000.fits.gz and /dev/null differ diff --git a/environment.yml b/environment.yml index de3dc181..77ad52f2 100644 --- a/environment.yml +++ b/environment.yml @@ -3,16 +3,16 @@ channels: - conda-forge - default dependencies: - - astropy>=4.2 - - python=3.8 # nail the python version, so conda does not try upgrading / dowgrading - - ctapipe=0.12 + - astropy=5.2 + - python=3.10 # nail the python version, so conda does not try upgrading / dowgrading + - ctapipe=0.17 - eventio - corsikaio - protozfits=2.0 - zeromq - ipython - numba - - numpy>=1.20,<1.23 + - numpy=1.22 - numpydoc - pytest - pyyaml diff --git a/eventsource_subclasses.py b/eventsource_subclasses.py new file mode 100644 index 00000000..3ddaeac2 --- /dev/null +++ b/eventsource_subclasses.py @@ -0,0 +1,5 @@ +from ctapipe.io import EventSource +from ctapipe.core import non_abstract_children + +for cls in non_abstract_children(EventSource): + print(cls.__name__) diff --git a/pyproject.toml b/pyproject.toml index 16a5dd9f..99157694 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ [build-system] -requires = ["setuptools >= 40.6.0", "wheel", "setuptools_scm[toml]>=3.4"] +requires = ["setuptools >= 64.0.3", "setuptools_scm[toml]>=3.4"] build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "src/ctapipe_io_lst/_version.py" diff --git a/setup.cfg b/setup.cfg index d53d488c..6e063460 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,17 +1,62 @@ [metadata] name = ctapipe_io_lst description = ctapipe plugin for reading LST prototype files -long-description = file: README.md -long-description-content-type = text/markdown; charset=UTF-8; variant=GFM -author = LST Consortium -author_email = cassol@cppm.in2p3.fr +long_description = file: README.md +long_description_content_type = text/markdown; charset=UTF-8; variant=GFM +author = CTA LST Project +author_email = maximilian.linhoff@tu-dortmund.de license = BSD 3-clause +project_urls = + Bug Tracker = https://github.com/cta-observatory/ctapipe_io_lst/issues + Source Code = https://github.com/cta-observatory/ctapipe_io_lst + +classifiers = + Development Status :: 4 - Beta + License :: OSI Approved :: MIT License + Intended Audience :: Science/Research + Topic :: Scientific/Engineering :: Astronomy + Topic :: Scientific/Engineering :: Physics + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + + +[options] +packages = find: +package_dir = + = src +python_requires = >=3.8 +zip_safe = False +install_requires= + astropy~=5.2 + ctapipe~=0.17.0 + protozfits~=2.0 + numpy>=1.20 + +[options.package_data] +* = resources/* + +[options.packages.find] +where = src +exclude = + ctapipe_io_lst._dev_version + +[options.extras_require] +tests = + pytest +dev = + setuptools_scm[toml] +all = + %(tests)s + %(dev)s + + [tool:pytest] minversion = 3.0 addopts = -v - [aliases] test = pytest diff --git a/setup.py b/setup.py index d98342d8..8bf1ba93 100755 --- a/setup.py +++ b/setup.py @@ -1,18 +1,2 @@ -from setuptools import setup, find_packages -import os - -setup( - packages=find_packages(exclude=["ctapipe_io_lst._dev_version"]), - use_scm_version={"write_to": os.path.join("ctapipe_io_lst", "_version.py")}, - install_requires=[ - 'astropy~=4.2', - 'ctapipe~=0.12', - 'protozfits~=2.0', - 'setuptools_scm', - 'numpy>=1.20,<1.23' - ], - package_data={ - 'ctapipe_io_lst': ['resources/*'], - }, - tests_require=['pytest'], -) +from setuptools import setup +setup() diff --git a/ctapipe_io_lst/__init__.py b/src/ctapipe_io_lst/__init__.py similarity index 87% rename from ctapipe_io_lst/__init__.py rename to src/ctapipe_io_lst/__init__.py index a215d6db..dd00635a 100644 --- a/ctapipe_io_lst/__init__.py +++ b/src/ctapipe_io_lst/__init__.py @@ -2,19 +2,20 @@ """ EventSource for LSTCam protobuf-fits.fz-files. """ +from ctapipe.instrument.subarray import EarthLocation import numpy as np from astropy import units as u from pkg_resources import resource_filename -import os -from os import listdir from ctapipe.core import Provenance from ctapipe.instrument import ( + ReflectorShape, TelescopeDescription, SubarrayDescription, CameraDescription, CameraReadout, CameraGeometry, OpticsDescription, + SizeType, ) from enum import IntFlag, auto from astropy.time import Time @@ -23,10 +24,13 @@ from ctapipe.io.datalevels import DataLevel from ctapipe.core.traits import Bool, Float, Enum, Path from ctapipe.containers import ( - PixelStatusContainer, EventType, R0CameraContainer, R1CameraContainer, + CoordinateFrameType, PixelStatusContainer, EventType, PointingMode, R0CameraContainer, R1CameraContainer, + SchedulingBlockContainer, ObservationBlockContainer, ) from ctapipe.coordinates import CameraFrame +from ctapipe_io_lst.ground_frame import ground_frame_from_earth_location + from .multifiles import MultiFiles from .containers import LSTArrayEventContainer, LSTServiceContainer, LSTEventContainer from .version import __version__ @@ -41,7 +45,7 @@ TIB_DTYPE, ) from .constants import ( - HIGH_GAIN, N_GAINS, N_PIXELS, N_SAMPLES + HIGH_GAIN, LST_LOCATIONS, N_GAINS, N_PIXELS, N_SAMPLES, LST1_LOCATION, REFERENCE_LOCATION, ) @@ -87,12 +91,16 @@ class PixelStatus(IntFlag): BOTH_GAINS_STORED = HIGH_GAIN_STORED | LOW_GAIN_STORED +#: LST Optics Description OPTICS = OpticsDescription( - 'LST', + name='LST', + size_type=SizeType.LST, + n_mirrors=1, + n_mirror_tiles=198, + reflector_shape=ReflectorShape.PARABOLIC, equivalent_focal_length=u.Quantity(28, u.m), - num_mirrors=1, + effective_focal_length=u.Quantity(29.30565, u.m), mirror_area=u.Quantity(386.73, u.m**2), - num_mirror_tiles=198, ) @@ -112,14 +120,14 @@ def get_channel_info(pixel_status): return (pixel_status & 0b1100) >> 2 -def load_camera_geometry(version=4): +def load_camera_geometry(): ''' Load camera geometry from bundled resources of this repo ''' f = resource_filename( - 'ctapipe_io_lst', f'resources/LSTCam-{version:03d}.camgeom.fits.gz' + 'ctapipe_io_lst', 'resources/LSTCam.camgeom.fits.gz' ) Provenance().add_input_file(f, role="CameraGeometry") cam = CameraGeometry.from_table(f) - cam.frame = CameraFrame(focal_length=OPTICS.equivalent_focal_length) + cam.frame = CameraFrame(focal_length=OPTICS.effective_focal_length) return cam @@ -247,6 +255,31 @@ class LSTEventSource(EventSource): ) ).tag(config=True) + reference_position_lon = Float( + default_value=REFERENCE_LOCATION.lon.deg, + help=( + "Longitude of the reference location for telescope GroundFrame coordinates.", + " Default is the roughly area weighted average of LST-1, MAGIC-1 and MAGIC-2.", + ) + ).tag(config=True) + + reference_position_lat = Float( + default_value=REFERENCE_LOCATION.lat.deg, + help=( + "Latitude of the reference location for telescope GroundFrame coordinates.", + " Default is the roughly area weighted average of LST-1, MAGIC-1 and MAGIC-2.", + ) + ).tag(config=True) + + reference_position_height = Float( + default_value=REFERENCE_LOCATION.height.to_value(u.m), + help=( + "Height of the reference location for telescope GroundFrame coordinates.", + " Default is current MC obslevel.", + ) + ).tag(config=True) + + classes = [PointingSource, EventTimeCalculator, LSTR0Corrections] def __init__(self, input_url=None, **kwargs): @@ -278,28 +311,63 @@ def __init__(self, input_url=None, **kwargs): self.input_url, parent=self, ) - self.geometry_version = 4 - self.camera_config = self.multi_file.camera_config + self.run_id = self.camera_config.configuration_id self.tel_id = self.camera_config.telescope_id - self._subarray = self.create_subarray(self.geometry_version, self.tel_id) + self.run_start = Time(self.camera_config.date, format='unix') + + reference_location = EarthLocation( + lon=self.reference_position_lon * u.deg, + lat=self.reference_position_lat * u.deg, + height=self.reference_position_height * u.m, + ) + self._subarray = self.create_subarray(self.tel_id, reference_location) self.r0_r1_calibrator = LSTR0Corrections( subarray=self._subarray, parent=self ) self.time_calculator = EventTimeCalculator( subarray=self.subarray, - run_id=self.camera_config.configuration_id, + run_id=self.run_id, expected_modules_id=self.camera_config.lstcam.expected_modules_id, parent=self, ) self.pointing_source = PointingSource(subarray=self.subarray, parent=self) self.lst_service = self.fill_lst_service_container(self.tel_id, self.camera_config) + target_info = {} + pointing_mode = PointingMode.UNKNOWN + if self.pointing_information: + target = self.pointing_source.get_target(tel_id=self.tel_id, time=self.run_start) + if target is not None: + target_info["subarray_pointing_lon"] = target["ra"] + target_info["subarray_pointing_lat"] = target["dec"] + target_info["subarray_pointing_frame"] = CoordinateFrameType.ICRS + pointing_mode = PointingMode.TRACK + + self._scheduling_blocks = { + self.run_id: SchedulingBlockContainer( + sb_id=np.uint64(self.run_id), + producer_id=f"LST-{self.tel_id}", + pointing_mode=pointing_mode, + ) + } + + self._observation_blocks = { + self.run_id: ObservationBlockContainer( + obs_id=np.uint64(self.run_id), + sb_id=np.uint64(self.run_id), + producer_id=f"LST-{self.tel_id}", + actual_start_time=self.run_start, + **target_info + ) + } + self.read_pedestal_ids() + + if self.use_flatfield_heuristic is None: - date_of_run = Time(self.camera_config.date, format='unix') - self.use_flatfield_heuristic = date_of_run < NO_FF_HEURISTIC_DATE + self.use_flatfield_heuristic = self.run_start < NO_FF_HEURISTIC_DATE self.log.info(f"Changed `use_flatfield_heuristic` to {self.use_flatfield_heuristic}") @property @@ -313,7 +381,15 @@ def is_simulation(self): @property def obs_ids(self): # currently no obs id is available from the input files - return [self.camera_config.configuration_id, ] + return list(self.observation_blocks) + + @property + def observation_blocks(self): + return self._observation_blocks + + @property + def scheduling_blocks(self): + return self._scheduling_blocks @property def datalevels(self): @@ -322,42 +398,50 @@ def datalevels(self): return (DataLevel.R0, ) @staticmethod - def create_subarray(geometry_version, tel_id=1): + def create_subarray(tel_id=1, reference_location=None): """ Obtain the subarray from the EventSource Returns ------- ctapipe.instrument.SubarrayDescription """ + if reference_location is None: + reference_location = REFERENCE_LOCATION - # camera info from LSTCam-[geometry_version].camgeom.fits.gz file - camera_geom = load_camera_geometry(version=geometry_version) + camera_geom = load_camera_geometry() # get info on the camera readout: daq_time_per_sample, pulse_shape_time_step, pulse_shapes = read_pulse_shapes() camera_readout = CameraReadout( - 'LSTCam', - 1 / daq_time_per_sample, - pulse_shapes, - pulse_shape_time_step, + name='LSTCam', + n_pixels=N_PIXELS, + n_channels=N_GAINS, + n_samples=N_SAMPLES, + sampling_rate=(1 / daq_time_per_sample).to(u.GHz), + reference_pulse_shape=pulse_shapes, + reference_pulse_sample_width=pulse_shape_time_step, ) - camera = CameraDescription('LSTCam', camera_geom, camera_readout) + camera = CameraDescription(name='LSTCam', geometry=camera_geom, readout=camera_readout) lst_tel_descr = TelescopeDescription( - name='LST', tel_type='LST', optics=OPTICS, camera=camera + name='LST', optics=OPTICS, camera=camera ) tel_descriptions = {tel_id: lst_tel_descr} - # LSTs telescope position taken from MC from the moment - tel_positions = {tel_id: [50., 50., 16] * u.m} + xyz = ground_frame_from_earth_location( + LST_LOCATIONS[tel_id], + reference_location, + ).cartesian.xyz + tel_positions = {tel_id: xyz} subarray = SubarrayDescription( name=f"LST-{tel_id} subarray", tel_descriptions=tel_descriptions, tel_positions=tel_positions, + reference_location=LST1_LOCATION, ) return subarray diff --git a/ctapipe_io_lst/_dev_version/__init__.py b/src/ctapipe_io_lst/_dev_version/__init__.py similarity index 100% rename from ctapipe_io_lst/_dev_version/__init__.py rename to src/ctapipe_io_lst/_dev_version/__init__.py diff --git a/ctapipe_io_lst/anyarray_dtypes.py b/src/ctapipe_io_lst/anyarray_dtypes.py similarity index 100% rename from ctapipe_io_lst/anyarray_dtypes.py rename to src/ctapipe_io_lst/anyarray_dtypes.py diff --git a/ctapipe_io_lst/calibration.py b/src/ctapipe_io_lst/calibration.py similarity index 97% rename from ctapipe_io_lst/calibration.py rename to src/ctapipe_io_lst/calibration.py index 6708a7d3..4c0b897b 100644 --- a/ctapipe_io_lst/calibration.py +++ b/src/ctapipe_io_lst/calibration.py @@ -12,7 +12,7 @@ ) from ctapipe.calib.camera.gainselection import ThresholdGainSelector -from ctapipe.containers import MonitoringContainer +from ctapipe.containers import FlatFieldContainer, MonitoringCameraContainer, MonitoringContainer, PedestalContainer, PixelStatusContainer, WaveformCalibrationContainer from ctapipe.io import HDF5TableReader, read_table from .containers import LSTArrayEventContainer from traitlets import Enum @@ -339,33 +339,25 @@ def _read_calibration_file(path): """ Read the correction from hdf5 calibration file """ - mon = MonitoringContainer() with tables.open_file(path) as f: - tel_ids = [ - int(key[4:]) for key in f.root._v_children.keys() + tel_groups = [ + key for key in f.root._v_children.keys() if key.startswith('tel_') ] - for tel_id in tel_ids: + mon = MonitoringContainer() + + for base in tel_groups: with HDF5TableReader(path) as h5_table: - base = f'/tel_{tel_id}' # read the calibration data - table = base + '/calibration' - next(h5_table.read(table, mon.tel[tel_id].calibration)) - - # read pedestal data - table = base + '/pedestal' - next(h5_table.read(table, mon.tel[tel_id].pedestal)) - - # read flat-field data - table = base + '/flatfield' - next(h5_table.read(table, mon.tel[tel_id].flatfield)) - - # read the pixel_status container - table = base + '/pixel_status' - next(h5_table.read(table, mon.tel[tel_id].pixel_status)) - + tel_id = int(base[4:]) + mon.tel[tel_id] = MonitoringCameraContainer( + calibration=next(h5_table.read(f'/{base}/calibration', WaveformCalibrationContainer)), + pedestal=next(h5_table.read(f'/{base}/pedestal', PedestalContainer)), + flatfield=next(h5_table.read(f'/{base}/flatfield', FlatFieldContainer)), + pixel_status=next(h5_table.read(f"/{base}/pixel_status", PixelStatusContainer)), + ) return mon @staticmethod diff --git a/src/ctapipe_io_lst/constants.py b/src/ctapipe_io_lst/constants.py new file mode 100644 index 00000000..fb7d4838 --- /dev/null +++ b/src/ctapipe_io_lst/constants.py @@ -0,0 +1,45 @@ +import numpy as np +import astropy.units as u +from astropy.coordinates import EarthLocation + +N_GAINS = 2 +N_MODULES = 265 +N_PIXELS_MODULE = 7 +N_PIXELS = N_MODULES * N_PIXELS_MODULE +N_CAPACITORS_CHANNEL = 1024 +# 4 drs4 channels are cascaded for each pixel +N_CAPACITORS_PIXEL = 4 * N_CAPACITORS_CHANNEL +N_SAMPLES = 40 +HIGH_GAIN = 0 +LOW_GAIN = 1 +CLOCK_FREQUENCY_KHZ = 133e3 + +# we have 8 channels per module, but only 7 are used. +N_CHANNELS_MODULE = 8 + +# First capacitor order according Dragon v5 board data format +CHANNEL_ORDER_HIGH_GAIN = [0, 0, 1, 1, 2, 2, 3] +CHANNEL_ORDER_LOW_GAIN = [4, 4, 5, 5, 6, 6, 7] + +PIXEL_INDEX = np.arange(N_PIXELS) + +#: location of lst-1 as `~astropy.coordinates.EarthLocation` +#: Taken from Abelardo's Coordinates of LST-1 & MAGIC presentation +#: https://redmine.cta-observatory.org/attachments/65827 +LST1_LOCATION = EarthLocation( + lon=-17.89149701 * u.deg, + lat=28.76152611 * u.deg, + # height of central pin + distance from pin to elevation axis + height=2184 * u.m + 15.883 * u.m +) + +#: Area averaged position of LST-1, MAGIC-1 and MAGIC-2 (using 23**2 and 17**2 m2) +REFERENCE_LOCATION = EarthLocation( + lon=-17.890879 * u.deg, + lat=28.761579 * u.deg, + height=2199 * u.m, # MC obs-level +) + +LST_LOCATIONS = { + 1: LST1_LOCATION, +} diff --git a/ctapipe_io_lst/containers.py b/src/ctapipe_io_lst/containers.py similarity index 100% rename from ctapipe_io_lst/containers.py rename to src/ctapipe_io_lst/containers.py diff --git a/ctapipe_io_lst/event_time.py b/src/ctapipe_io_lst/event_time.py similarity index 100% rename from ctapipe_io_lst/event_time.py rename to src/ctapipe_io_lst/event_time.py diff --git a/src/ctapipe_io_lst/ground_frame.py b/src/ctapipe_io_lst/ground_frame.py new file mode 100644 index 00000000..a9f281de --- /dev/null +++ b/src/ctapipe_io_lst/ground_frame.py @@ -0,0 +1,34 @@ +"""Conversions between ground frame and earth location + +This will be included in ctapipe 0.19, remove when upgrading. +""" +from astropy.coordinates import AltAz, ITRS, CartesianRepresentation +from ctapipe.coordinates import GroundFrame + +def _altaz_to_earthlocation(altaz): + local_itrs = altaz.transform_to(ITRS(location=altaz.location)) + itrs = ITRS(local_itrs.cartesian + altaz.location.get_itrs().cartesian) + return itrs.earth_location + + +def _earthlocation_to_altaz(location, reference_location): + # See + # https://docs.astropy.org/en/stable/coordinates/common_errors.html#altaz-calculations-for-earth-based-objects + # for why this is necessary and we cannot just do + # `get_itrs().transform_to(AltAz())` + itrs_cart = location.get_itrs().cartesian + itrs_ref_cart = reference_location.get_itrs().cartesian + local_itrs = ITRS(itrs_cart - itrs_ref_cart, location=reference_location) + return local_itrs.transform_to(AltAz(location=reference_location)) + +def ground_frame_to_earth_location(ground_frame, reference_location): + # in astropy, x points north, y points east, so we need a minus for y. + cart = CartesianRepresentation(ground_frame.x, -ground_frame.y, ground_frame.z) + altaz = AltAz(cart, location=reference_location) + return _altaz_to_earthlocation(altaz) + +def ground_frame_from_earth_location(location, reference_location): + altaz = _earthlocation_to_altaz(location, reference_location) + x, y, z = altaz.cartesian.xyz + # in astropy, x points north, y points east, so we need a minus for y. + return GroundFrame(x=x, y=-y, z=z) diff --git a/ctapipe_io_lst/multifiles.py b/src/ctapipe_io_lst/multifiles.py similarity index 100% rename from ctapipe_io_lst/multifiles.py rename to src/ctapipe_io_lst/multifiles.py diff --git a/ctapipe_io_lst/pointing.py b/src/ctapipe_io_lst/pointing.py similarity index 100% rename from ctapipe_io_lst/pointing.py rename to src/ctapipe_io_lst/pointing.py diff --git a/src/ctapipe_io_lst/resources/LSTCam.camgeom.fits.gz b/src/ctapipe_io_lst/resources/LSTCam.camgeom.fits.gz new file mode 100644 index 00000000..6e612250 Binary files /dev/null and b/src/ctapipe_io_lst/resources/LSTCam.camgeom.fits.gz differ diff --git a/ctapipe_io_lst/resources/no_corrections_pulse_LST.dat b/src/ctapipe_io_lst/resources/no_corrections_pulse_LST.dat similarity index 100% rename from ctapipe_io_lst/resources/no_corrections_pulse_LST.dat rename to src/ctapipe_io_lst/resources/no_corrections_pulse_LST.dat diff --git a/ctapipe_io_lst/resources/oversampled_pulse_LST_8dynode_pix6_20200204.dat b/src/ctapipe_io_lst/resources/oversampled_pulse_LST_8dynode_pix6_20200204.dat similarity index 100% rename from ctapipe_io_lst/resources/oversampled_pulse_LST_8dynode_pix6_20200204.dat rename to src/ctapipe_io_lst/resources/oversampled_pulse_LST_8dynode_pix6_20200204.dat diff --git a/src/ctapipe_io_lst/tests/__init__.py b/src/ctapipe_io_lst/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ctapipe_io_lst/tests/resources/first_caps.hdf5 b/src/ctapipe_io_lst/tests/resources/first_caps.hdf5 similarity index 100% rename from ctapipe_io_lst/tests/resources/first_caps.hdf5 rename to src/ctapipe_io_lst/tests/resources/first_caps.hdf5 diff --git a/ctapipe_io_lst/tests/test_calib.py b/src/ctapipe_io_lst/tests/test_calib.py similarity index 94% rename from ctapipe_io_lst/tests/test_calib.py rename to src/ctapipe_io_lst/tests/test_calib.py index 36a7deb5..7e848a42 100644 --- a/ctapipe_io_lst/tests/test_calib.py +++ b/src/ctapipe_io_lst/tests/test_calib.py @@ -14,13 +14,14 @@ test_data = Path(os.getenv('LSTCHAIN_TEST_DATA', 'test_data')) test_r0_path = test_data / 'real/R0/20200218/LST-1.1.Run02008.0000_first50.fits.fz' test_r0_calib_path = test_data / 'real/R0/20200218/LST-1.1.Run02006.0004.fits.fz' -test_calib_path = test_data / 'real/monitoring/PixelCalibration/LevelA/calibration/20200218/v0.8.2.post2.dev48+gb1343281/calibration_filters_52.Run02006.0000.h5' -test_drs4_pedestal_path = test_data / 'real/monitoring/PixelCalibration/LevelA/drs4_baseline/20200218/v0.8.2.post2.dev48+gb1343281/drs4_pedestal.Run02005.0000.h5' -test_time_calib_path = test_data / 'real/monitoring/PixelCalibration/LevelA/drs4_time_sampling_from_FF/20191124/v0.8.2.post2.dev48+gb1343281/time_calibration.Run01625.0000.h5' test_missing_module_path = test_data / 'real/R0/20210215/LST-1.1.Run03669.0000_first50.fits.fz' test_r0_gainselected_path = test_data / 'real/R0/20200218/LST-1.1.Run02008.0000_first50_gainselected.fits.fz' - +calib_version = "ctapipe-v0.17" +calib_path = test_data / 'real/monitoring/PixelCalibration/Cat-A/' +test_calib_path = calib_path / f'calibration/20200218/{calib_version}/calibration_filters_52.Run02006.0000.h5' +test_drs4_pedestal_path = calib_path / f'drs4_baseline/20200218/{calib_version}/drs4_pedestal.Run02005.0000.h5' +test_time_calib_path = calib_path / f'drs4_time_sampling_from_FF/20191124/{calib_version}/time_calibration.Run01625.0000.h5' def test_get_first_capacitor(): @@ -84,7 +85,7 @@ def test_init(): from ctapipe_io_lst import LSTEventSource from ctapipe_io_lst.calibration import LSTR0Corrections - subarray = LSTEventSource.create_subarray(geometry_version=4) + subarray = LSTEventSource.create_subarray() r0corr = LSTR0Corrections(subarray) assert r0corr.last_readout_time.keys() == {1, } diff --git a/ctapipe_io_lst/tests/test_event_time.py b/src/ctapipe_io_lst/tests/test_event_time.py similarity index 97% rename from ctapipe_io_lst/tests/test_event_time.py rename to src/ctapipe_io_lst/tests/test_event_time.py index 46485c15..c073e203 100644 --- a/ctapipe_io_lst/tests/test_event_time.py +++ b/src/ctapipe_io_lst/tests/test_event_time.py @@ -119,7 +119,7 @@ def test_ucts_jumps(): lst.svc.module_ids = np.arange(N_MODULES) lst.evt.module_status = np.ones(N_MODULES) - subarray = LSTEventSource.create_subarray(geometry_version=4, tel_id=1) + subarray = LSTEventSource.create_subarray(tel_id=1) true_time_s = int(Time.now().unix) @@ -213,7 +213,7 @@ def test_extract_reference_values(caplog): from ctapipe_io_lst.event_time import EventTimeCalculator, CENTRAL_MODULE from ctapipe_io_lst import LSTEventSource - subarray = LSTEventSource.create_subarray(geometry_version=4, tel_id=1) + subarray = LSTEventSource.create_subarray(tel_id=1) # no reference values given and extract_reference = False should raise with pytest.raises(ValueError): @@ -269,7 +269,7 @@ def test_no_reference_values_no_ucts(caplog): from ctapipe_io_lst.event_time import EventTimeCalculator, CENTRAL_MODULE from ctapipe_io_lst import LSTEventSource - subarray = LSTEventSource.create_subarray(geometry_version=4, tel_id=1) + subarray = LSTEventSource.create_subarray(tel_id=1) # test ucts reference extaction works time_calculator = EventTimeCalculator( diff --git a/ctapipe_io_lst/tests/test_lsteventsource.py b/src/ctapipe_io_lst/tests/test_lsteventsource.py similarity index 91% rename from ctapipe_io_lst/tests/test_lsteventsource.py rename to src/ctapipe_io_lst/tests/test_lsteventsource.py index ca6d21a4..34a01c4c 100644 --- a/ctapipe_io_lst/tests/test_lsteventsource.py +++ b/src/ctapipe_io_lst/tests/test_lsteventsource.py @@ -8,7 +8,7 @@ import pytest import tables -from ctapipe.containers import EventType +from ctapipe.containers import CoordinateFrameType, EventType, PointingMode from ctapipe.calib.camera.gainselection import ThresholdGainSelector from ctapipe_io_lst.constants import N_GAINS, N_PIXELS_MODULE, N_SAMPLES, N_PIXELS @@ -103,6 +103,14 @@ def test_subarray(): assert source.lst_service.telescope_id == 1 assert source.lst_service.num_modules == 265 + position = source.subarray.positions[1] + mc_position = [-6.336, 60.405, 12.5] * u.m + + # mc uses slightly different reference location and z is off + # so only test x/y distance + distance = np.linalg.norm(mc_position[:2] - position[:2]) + assert distance < 0.5 * u.m + with tempfile.NamedTemporaryFile(suffix='.h5') as f: subarray.to_hdf(f.name) @@ -214,6 +222,19 @@ def test_pointing_info(): test_r0_dir / 'LST-1.1.Run02008.0000_first50.fits.fz', config=Config(config), ) as source: + + sb = source.scheduling_blocks[2008] + assert sb.sb_id == 2008 + assert sb.pointing_mode is PointingMode.TRACK + + obs = source.observation_blocks[2008] + assert u.isclose(obs.subarray_pointing_lon, 83.6296 * u.deg) + assert u.isclose(obs.subarray_pointing_lat, 22.0144 * u.deg) + assert obs.subarray_pointing_frame is CoordinateFrameType.ICRS + assert obs.producer_id == "LST-1" + assert obs.obs_id == 2008 + assert obs.sb_id == 2008 + for e in source: assert u.isclose(e.pointing.array_ra, 83.6296 * u.deg) assert u.isclose(e.pointing.array_dec, 22.0144 * u.deg) diff --git a/ctapipe_io_lst/tests/test_multifile.py b/src/ctapipe_io_lst/tests/test_multifile.py similarity index 100% rename from ctapipe_io_lst/tests/test_multifile.py rename to src/ctapipe_io_lst/tests/test_multifile.py diff --git a/ctapipe_io_lst/tests/test_pointing.py b/src/ctapipe_io_lst/tests/test_pointing.py similarity index 97% rename from ctapipe_io_lst/tests/test_pointing.py rename to src/ctapipe_io_lst/tests/test_pointing.py index 088ad5e4..bfa4b37b 100644 --- a/ctapipe_io_lst/tests/test_pointing.py +++ b/src/ctapipe_io_lst/tests/test_pointing.py @@ -26,7 +26,7 @@ def test_interpolation(): from ctapipe_io_lst.pointing import PointingSource from ctapipe_io_lst import LSTEventSource - subarray = LSTEventSource.create_subarray(geometry_version=4) + subarray = LSTEventSource.create_subarray() pointing_source = PointingSource( subarray=subarray, drive_report_path=test_drive_report, @@ -101,7 +101,7 @@ def test_targets(): capella = Time(1582069585, format="unix") after_last_tracking = Time(1582070175, format="unix") - subarray = LSTEventSource.create_subarray(geometry_version=4, tel_id=1) + subarray = LSTEventSource.create_subarray(tel_id=1) # test explicitly giving path and with using drive_report_path test_kwargs = [ diff --git a/ctapipe_io_lst/tests/test_stage1.py b/src/ctapipe_io_lst/tests/test_stage1.py similarity index 90% rename from ctapipe_io_lst/tests/test_stage1.py rename to src/ctapipe_io_lst/tests/test_stage1.py index e4d67f68..5246379b 100644 --- a/ctapipe_io_lst/tests/test_stage1.py +++ b/src/ctapipe_io_lst/tests/test_stage1.py @@ -12,12 +12,15 @@ test_data = Path(os.getenv('LSTCHAIN_TEST_DATA', 'test_data')) test_r0_path = test_data / 'real/R0/20200218/LST-1.1.Run02008.0000_first50.fits.fz' -test_calib_path = test_data / 'real/monitoring/PixelCalibration/LevelA/calibration/20200218/v0.8.2.post2.dev48+gb1343281/calibration_filters_52.Run02006.0000.h5' -test_drs4_pedestal_path = test_data / 'real/monitoring/PixelCalibration/LevelA/drs4_baseline/20200218/v0.8.2.post2.dev48+gb1343281/drs4_pedestal.Run02005.0000.h5' -test_time_calib_path = test_data / 'real/monitoring/PixelCalibration/LevelA/drs4_time_sampling_from_FF/20191124/v0.8.2.post2.dev48+gb1343281/time_calibration.Run01625.0000.h5' test_drive_report = test_data / 'real/monitoring/DrivePositioning/DrivePosition_log_20200218.txt' test_run_summary = test_data / 'real/monitoring/RunSummary/RunSummary_20200218.ecsv' +calib_version = "ctapipe-v0.17" +calib_path = test_data / 'real/monitoring/PixelCalibration/Cat-A/' +test_calib_path = calib_path / f'calibration/20200218/{calib_version}/calibration_filters_52.Run02006.0000.h5' +test_drs4_pedestal_path = calib_path / f'drs4_baseline/20200218/{calib_version}/drs4_pedestal.Run02005.0000.h5' +test_time_calib_path = calib_path / f'drs4_time_sampling_from_FF/20191124/{calib_version}/time_calibration.Run01625.0000.h5' + def test_stage1(tmp_path): """Test the ctapipe stage1 tool can read in LST real data using the event source""" @@ -62,12 +65,11 @@ def test_stage1(tmp_path): tool = ProcessorTool() output = tmp_path / "test_dl1.h5" - ret = run_tool(tool, argv=[ + run_tool(tool, argv=[ f'--input={test_r0_path}', f'--output={output}', f'--config={config_path}', - ]) - assert ret == 0 + ], raises=True) # test our custom default works assert tool.event_source.r0_r1_calibrator.gain_selector.threshold == 3500 @@ -137,12 +139,11 @@ def test_no_ff_tagging(tmp_path): tool = ProcessorTool() output = tmp_path / "test_dl1.h5" - ret = run_tool(tool, argv=[ + run_tool(tool, argv=[ f'--input={test_r0_path}', f'--output={output}', f'--config={config_path}', - ]) - assert ret == 0 + ], raises=True) # test our custom default works assert tool.event_source.r0_r1_calibrator.gain_selector.threshold == 3500 diff --git a/ctapipe_io_lst/tests/test_version.py b/src/ctapipe_io_lst/tests/test_version.py similarity index 100% rename from ctapipe_io_lst/tests/test_version.py rename to src/ctapipe_io_lst/tests/test_version.py diff --git a/ctapipe_io_lst/version.py b/src/ctapipe_io_lst/version.py similarity index 100% rename from ctapipe_io_lst/version.py rename to src/ctapipe_io_lst/version.py