Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PREOPS-4646: feature- Support addition of opsim data to an archive for use by schedview-prenight #20

Merged
merged 19 commits into from
Jan 16, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add func for querying sim archive metadata
ehneilsen committed Jan 5, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 0983f8de2289b6aca4ed890ce9348c1dabaeb5e9
65 changes: 65 additions & 0 deletions rubin_scheduler/sim_archive/sim_archive.py
Original file line number Diff line number Diff line change
@@ -277,3 +277,68 @@ def check_opsim_archive_resource(archive_uri):
results[file_info["name"]] = file_info["md5"] == hashlib.md5(content).hexdigest()

return results


def _build_archived_sim_label(base_uri, metadata_resource, metadata):
label_base = metadata_resource.dirname().geturl().removeprefix(base_uri).rstrip("/").lstrip("/")

# If a label is supplied by the metadata, use it
if "label" in metadata:
label = f"{label_base} {metadata['label']}"
return label

try:
sim_dates = metadata["simulated_dates"]
start_date = sim_dates["start"]
end_date = sim_dates["end"]
label = f"{label_base} of {start_date}"
if end_date != start_date:
label = f"{label} to {end_date}"
except KeyError:
label = label_base

if "scheduler_version" in metadata:
label = f"{label} with {metadata['scheduler_version']}"

return label


def read_archived_sim_metadata(base_uri, latest=None, num_nights=5):
"""Read metadata for a time range of archived opsim output.
Parameters:
----------
base_uri : `str`
The base URI of the archive resource to be checked.
latest : `str`, optional
The date of the latest simulation whose metadata should be loaded.
This is the date on which the simulations was added to the archive,
not necessarily the date on which the simulation was run, or any
of the dates simulated.
Default is today.
num_nights : `int`
The number of nights of the date window to load.
Returns:
-------
sim_metadata: `dict`
A dictionary of metadata for simulations in the date range.
"""
latest_mjd = int(Time.now().mjd if latest is None else Time(latest).mjd)
earliest_mjd = int(latest_mjd - num_nights)

all_metadata = {}
for mjd in range(earliest_mjd, latest_mjd + 1):
iso_date = Time(mjd, format="mjd").iso[:10]
date_resource = ResourcePath(base_uri).join(iso_date, forceDirectory=True)
if date_resource.exists():
for base_dir, found_dirs, found_files in date_resource.walk(file_filter=r".*sim_metadata.yaml"):
for found_file in found_files:
found_resource = ResourcePath(base_dir).join(found_file)
these_metadata = yaml.safe_load(found_resource.read().decode("utf-8"))
these_metadata["label"] = _build_archived_sim_label(
base_uri, found_resource, these_metadata
)
all_metadata[str(found_resource.dirname())] = these_metadata

return all_metadata
7 changes: 7 additions & 0 deletions tests/sim_archive/test_sim_archive.py
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@
from rubin_scheduler.sim_archive.sim_archive import (
check_opsim_archive_resource,
make_sim_archive_dir,
read_archived_sim_metadata,
transfer_archive_dir,
)
from rubin_scheduler.utils import survey_start_mjd
@@ -79,3 +80,9 @@ def test_sim_archive(self):
)
for value in archive_check.values():
self.assertTrue(value)

# Read back the metadata
archive_metadata = read_archived_sim_metadata(test_resource_uri)
base = sim_archive_uri.dirname().geturl().removeprefix(test_resource_uri).rstrip("/").lstrip("/")
expected_label = f"{base} test"
self.assertEqual(archive_metadata[sim_archive_uri.geturl()]["label"], expected_label)