From 688170ebef8700edafa916106687abbab2497764 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Tue, 13 Aug 2024 13:56:48 -0700 Subject: [PATCH] Move region/time tests to right place so that each backend can be tested --- .../lsst/daf/butler/tests/butler_queries.py | 77 ++++++++++++++++++ tests/test_simpleButler.py | 81 ------------------- 2 files changed, 77 insertions(+), 81 deletions(-) diff --git a/python/lsst/daf/butler/tests/butler_queries.py b/python/lsst/daf/butler/tests/butler_queries.py index e758aba707..13513a9a91 100644 --- a/python/lsst/daf/butler/tests/butler_queries.py +++ b/python/lsst/daf/butler/tests/butler_queries.py @@ -33,10 +33,12 @@ import unittest from abc import ABC, abstractmethod from collections.abc import Iterable, Sequence +from operator import attrgetter from typing import ClassVar from uuid import UUID import astropy.time +from lsst.sphgeom import Region from .._butler import Butler from .._dataset_type import DatasetType @@ -1058,6 +1060,81 @@ def do_query(constraint: Predicate) -> list[int]: ): x.exposure.observation_type.as_boolean() + def test_dataset_region_queries(self) -> None: + """Test region queries for datasets.""" + # Import data to play with. + butler = self.make_butler("base.yaml", "ci_hsc-subset.yaml") + + run = "HSC/runs/ci_hsc/20240806T180642Z" + with butler._query() as query: + # Return everything. + results = query.datasets("calexp", collections=run) + # Sort by data coordinate. + refs = sorted(results.with_dimension_records(), key=attrgetter("dataId")) + self.assertEqual(len(refs), 33) + + # Use a region from the first visit. + first_visit_region = refs[0].dataId.visit.region # type: ignore + + # Get a visit detector region from the first ref. + with butler._query() as query: + data_id = refs[0].dataId.mapping + records = list(query.dimension_records("visit_detector_region").where(**data_id)) # type: ignore + self.assertEqual(len(records), 1) + + for pos, count in ( + ("CIRCLE 320. -0.25 10.", 33), # Match everything. + ("CIRCLE 321.0 -0.4 0.01", 1), # Should be small region on 1 detector. + ("CIRCLE 321.1 -0.35 0.02", 2), + ("CIRCLE 321.1 -0.48 0.05", 1), # Center off the region. + ("CIRCLE 321.0 -0.5 0.01", 0), # No overlap. + (first_visit_region.to_ivoa_pos(), 33), # Visit region overlaps everything. + (records[0].region.to_ivoa_pos(), 17), # Some overlap. + ): + with butler._query() as query: + results = query.datasets("calexp", collections=run) + results = results.where( + "instrument = 'HSC' AND visit_detector_region.region OVERLAPS(POS)", + bind={"POS": Region.from_ivoa_pos(pos)}, + ) + refs = list(results) + self.assertEqual(len(refs), count, f"POS={pos} REFS={refs}") + + def test_dataset_time_queries(self) -> None: + """Test region queries for datasets.""" + # Import data to play with. + butler = self.make_butler("base.yaml", "ci_hsc-subset.yaml") + + # Some times from the test data. + v_903334_pre = astropy.time.Time("2013-01-01T12:00:00", scale="tai", format="isot") + v_903334_mid = astropy.time.Time("2013-06-17T13:29:20", scale="tai", format="isot") + v_904014_pre = astropy.time.Time("2013-11-01T12:00:00", scale="tai", format="isot") + v_904014_post = astropy.time.Time("2013-12-21T12:00:00", scale="tai", format="isot") + + with butler._query() as query: + run = "HSC/runs/ci_hsc/20240806T180642Z" + results = query.datasets("calexp", collections=run) + + # Use a time during the middle of a visit. + v_903334 = results.where( + "instrument = 'HSC' and visit.timespan OVERLAPS(ts)", bind={"ts": v_903334_mid} + ) + self.assertEqual(len(list(v_903334)), 4) + + # Timespan covering first half of the data. + first_half = results.where( + "instrument = 'HSC' and visit.timespan OVERLAPS(t1, t2)", + bind={"t1": v_903334_pre, "t2": v_904014_pre}, + ) + self.assertEqual(len(list(first_half)), 17) + + # Query using a timespan object. + with_ts = results.where( + "instrument = 'HSC' and visit.timespan OVERLAPS(ts)", + bind={"ts": Timespan(v_904014_pre, v_904014_post)}, + ) + self.assertEqual(len(list(with_ts)), 16) + def _get_exposure_ids_from_dimension_records(dimension_records: Iterable[DimensionRecord]) -> list[int]: output = [] diff --git a/tests/test_simpleButler.py b/tests/test_simpleButler.py index 2afa65d00d..0ef43f5e5f 100644 --- a/tests/test_simpleButler.py +++ b/tests/test_simpleButler.py @@ -32,7 +32,6 @@ import re import tempfile import unittest -from operator import attrgetter from typing import Any try: @@ -56,7 +55,6 @@ from lsst.daf.butler.registry import RegistryConfig, RegistryDefaults, _RegistryFactory from lsst.daf.butler.tests import DatastoreMock from lsst.daf.butler.tests.utils import TestCaseMixin, makeTestTempDir, removeTestTempDir -from lsst.sphgeom import Region try: from lsst.daf.butler.tests.server import create_test_server @@ -798,85 +796,6 @@ def makeButler(self, writeable: bool = False) -> Butler: DatastoreMock.apply(butler) return butler - def test_region_queries(self): - """Test region queries for datasets.""" - # Import data to play with. - butler = self.makeButler(writeable=True) - butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) - butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "ci_hsc-subset.yaml")) - - run = "HSC/runs/ci_hsc/20240806T180642Z" - with butler._query() as query: - # Return everything. - results = query.datasets("calexp", collections=run) - # Sort by data coordinate. - refs = sorted(results.with_dimension_records(), key=attrgetter("dataId")) - self.assertEqual(len(refs), 33) - - # Use a region from the first visit. - first_visit_region = refs[0].dataId.visit.region - - # Get a visit detector region from the first ref. - with butler._query() as query: - results = query.dimension_records("visit_detector_region").where(**refs[0].dataId.mapping) - records = list(results) - self.assertEqual(len(records), 1) - - for pos, count in ( - ("CIRCLE 320. -0.25 10.", 33), # Match everything. - ("CIRCLE 321.0 -0.4 0.01", 1), # Should be small region on 1 detector. - ("CIRCLE 321.1 -0.35 0.02", 2), - ("CIRCLE 321.1 -0.48 0.05", 1), # Center off the region. - ("CIRCLE 321.0 -0.5 0.01", 0), # No overlap. - (first_visit_region.to_ivoa_pos(), 33), # Visit region overlaps everything. - (records[0].region.to_ivoa_pos(), 17), # Some overlap. - ): - with butler._query() as query: - results = query.datasets("calexp", collections=run) - results = results.where( - "instrument = 'HSC' AND visit_detector_region.region OVERLAPS(POS)", - bind={"POS": Region.from_ivoa_pos(pos)}, - ) - refs = list(results) - self.assertEqual(len(refs), count, f"POS={pos} REFS={refs}") - - def test_time_queries(self): - """Test region queries for datasets.""" - # Import data to play with. - butler = self.makeButler(writeable=True) - butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "base.yaml")) - butler.import_(filename=os.path.join(TESTDIR, "data", "registry", "ci_hsc-subset.yaml")) - - # Some times from the test data. - v_903334_pre = astropy.time.Time("2013-01-01T12:00:00", scale="tai", format="isot") - v_903334_mid = astropy.time.Time("2013-06-17T13:29:20", scale="tai", format="isot") - v_904014_pre = astropy.time.Time("2013-11-01T12:00:00", scale="tai", format="isot") - v_904014_post = astropy.time.Time("2013-12-21T12:00:00", scale="tai", format="isot") - - with butler._query() as query: - run = "HSC/runs/ci_hsc/20240806T180642Z" - results = query.datasets("calexp", collections=run) - - # Use a time during the middle of a visit. - v_903334 = results.where( - "instrument = 'HSC' and visit.timespan OVERLAPS(ts)", bind={"ts": v_903334_mid} - ) - self.assertEqual(len(list(v_903334)), 4) - - # Timespan covering first half of the data. - first_half = results.where( - "instrument = 'HSC' and visit.timespan OVERLAPS(t1, t2)", - bind={"t1": v_903334_pre, "t2": v_904014_pre}, - ) - self.assertEqual(len(list(first_half)), 17) - - # Query using a timespan object. - with_ts = results.where( - "instrument = 'HSC' and visit.timespan OVERLAPS(ts)", - bind={"ts": Timespan(v_904014_pre, v_904014_post)}, - ) - self.assertEqual(len(list(with_ts)), 16) - class NameKeyCollectionManagerDirectSimpleButlerTestCase(DirectSimpleButlerTestCase, unittest.TestCase): """Run tests against DirectButler implementation using the