Skip to content

Commit

Permalink
Merge pull request #24 from lsst/tickets/DM-41326
Browse files Browse the repository at this point in the history 
DM-41326: Avoid using DimensionGraph->DimensionGroup transition tools.
  • Loading branch information
@TallJimbo
TallJimbo authored Jul 25, 2024
2 parents 98f1666 + de985f0 commit 700c84e
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 93 deletions.
132 changes: 44 additions & 88 deletions python/lsst/ci/middleware/data/input-dataset-types.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,23 @@
{
"name": "_mock_isr_metadata",
"storageClass": "PropertySet",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"detector",
"physical_filter",
"exposure"
]
},
],
"parentStorageClass": null,
"isCalibration": false
},
{
"name": "_mock_objectTable",
"storageClass": "_mock_ArrowTable",
"dimensions": {
"names": [
"dimensions": [
"tract",
"patch"
]
},
],
"parentStorageClass": null,
"isCalibration": false
}
Expand All @@ -32,15 +28,13 @@
{
"name": "_mock_raw",
"storageClass": "_mock_Exposure",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"detector",
"physical_filter",
"exposure"
]
},
],
"parentStorageClass": null,
"isCalibration": false
}
Expand All @@ -49,186 +43,156 @@
{
"name": "_mock_brighterFatterKernel",
"storageClass": "_mock_BrighterFatterKernel",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_transmission_filter",
"storageClass": "_mock_TransmissionCurve",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"physical_filter"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_dark",
"storageClass": "_mock_ExposureF",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_flat",
"storageClass": "_mock_ExposureF",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"detector",
"physical_filter"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_yBackground",
"storageClass": "_mock_StrayLightData",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"detector",
"physical_filter"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_bfKernel",
"storageClass": "_mock_NumpyArray",
"dimensions": {
"names": [
"dimensions": [
"instrument"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_crosstalk",
"storageClass": "_mock_CrosstalkCalib",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_transmission_atmosphere",
"storageClass": "_mock_TransmissionCurve",
"dimensions": {
"names": [
"dimensions": [
"instrument"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_transmission_sensor",
"storageClass": "_mock_TransmissionCurve",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_linearizer",
"storageClass": "_mock_Linearizer",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_transmission_optics",
"storageClass": "_mock_TransmissionCurve",
"dimensions": {
"names": [
"dimensions": [
"instrument"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_defects",
"storageClass": "_mock_Defects",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_bias",
"storageClass": "_mock_ExposureF",
"dimensions": {
"names": [
"dimensions": [
"instrument",
"detector"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_fringe",
"storageClass": "_mock_ExposureF",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"detector",
"physical_filter"
]
},
],
"parentStorageClass": null,
"isCalibration": true
},
{
"name": "_mock_sky",
"storageClass": "_mock_ExposureF",
"dimensions": {
"names": [
"dimensions": [
"band",
"instrument",
"detector",
"physical_filter"
]
},
],
"parentStorageClass": null,
"isCalibration": true
}
Expand All @@ -237,11 +201,9 @@
{
"name": "_mock_camera",
"storageClass": "_mock_Camera",
"dimensions": {
"names": [
"dimensions": [
"instrument"
]
},
],
"parentStorageClass": null,
"isCalibration": true
}
Expand All @@ -250,22 +212,18 @@
{
"name": "_mock_gaia_dr2_20200414",
"storageClass": "_mock_SimpleCatalog",
"dimensions": {
"names": [
"dimensions": [
"htm7"
]
},
],
"parentStorageClass": null,
"isCalibration": false
},
{
"name": "_mock_ps1_pv3_3pi_20170110",
"storageClass": "_mock_SimpleCatalog",
"dimensions": {
"names": [
"dimensions": [
"htm7"
]
},
],
"parentStorageClass": null,
"isCalibration": false
}
Expand All @@ -274,14 +232,12 @@
{
"name": "_mock_brightObjectMask",
"storageClass": "_mock_ObjectMaskCatalog",
"dimensions": {
"names": [
"dimensions": [
"band",
"skymap",
"tract",
"patch"
]
},
],
"parentStorageClass": null,
"isCalibration": false
}
Expand Down
6 changes: 2 additions & 4 deletions python/lsst/ci/middleware/mock_dataset_maker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,8 @@ def make_datasets(self, dataset_type: DatasetType, run: str = MISC_INPUT_RUN) ->
return
if not self.butler.registry.registerDatasetType(dataset_type):
return
if (data_ids := self.cached_data_ids.get(dataset_type.dimensions.as_group())) is None:
remaining_skypix_dimensions, dimensions = self._split_dimensions(
dataset_type.dimensions.as_group()
)
if (data_ids := self.cached_data_ids.get(dataset_type.dimensions)) is None:
remaining_skypix_dimensions, dimensions = self._split_dimensions(dataset_type.dimensions)
data_ids = self._get_bounded_data_ids(dimensions)
while remaining_skypix_dimensions:
skypix_dimension = remaining_skypix_dimensions.pop()
Expand Down
4 changes: 3 additions & 1 deletion python/lsst/ci/middleware/repo_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,9 @@ def _fill_visit_definition_elements(self, butler: Butler, visit_ids: list[int],
for element_name in ["exposure", "visit_definition", "visit_system", "visit_system_membership"]:
getattr(self, element_name).extend(
d.records[element_name]
for d in data_ids.subset(butler.dimensions[element_name].graph, unique=True).expanded()
for d in data_ids.subset(
butler.dimensions[element_name].minimal_group, unique=True
).expanded()
)

def _fill_visit(self, butler: Butler, visit_ids: list[int], instrument: str) -> None:
Expand Down

0 comments on commit 700c84e

Please sign in to comment.