From 49df15cc305b23f9ce4e46ddd348740da6cc9141 Mon Sep 17 00:00:00 2001 From: Andy Salnikov Date: Fri, 3 Nov 2023 11:50:29 -0700 Subject: [PATCH 1/3] Update pre-commit config --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbfb184..b793031 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.10.1 hooks: - id: black # It is recommended to specify the latest version of Python # supported by your project here, or alternatively use # pre-commit's default_language_version, see # https://pre-commit.com/#top_level-default_language_version - language_version: python3.10 + language_version: python3.11 - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: @@ -21,6 +21,6 @@ repos: name: isort (python) - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.0.277 + rev: v0.1.3 hooks: - id: ruff From 1bc4c438af45ce9cb9e1f5d0f4690507d1e35a51 Mon Sep 17 00:00:00 2001 From: Andy Salnikov Date: Fri, 3 Nov 2023 11:51:31 -0700 Subject: [PATCH 2/3] Update imports for new location of SqlRegistry class --- migrations/obscore-config/4fe28ef5030f.py | 2 +- .../lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/obscore-config/4fe28ef5030f.py b/migrations/obscore-config/4fe28ef5030f.py index 85bacce..bbb712c 100644 --- a/migrations/obscore-config/4fe28ef5030f.py +++ b/migrations/obscore-config/4fe28ef5030f.py @@ -10,7 +10,7 @@ import yaml from alembic import context, op -from lsst.daf.butler.registries.sql import SqlRegistry +from lsst.daf.butler.registry.sql_registry import SqlRegistry from lsst.daf.butler_migrate.butler_attributes import ButlerAttributes from lsst.daf.butler_migrate.registry import make_registry from lsst.utils import doImportType diff --git a/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py b/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py index 64cbb16..51bf3f9 100644 --- a/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py +++ b/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py @@ -29,9 +29,9 @@ from lsst.daf.butler import Butler, Config, DatasetAssociation, DatasetId, DatasetRef, SkyPixDimension from lsst.daf.butler.datastores.fileDatastore import FileDatastore -from lsst.daf.butler.registries.sql import SqlRegistry from lsst.daf.butler.registry import CollectionType from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase +from lsst.daf.butler.registry.sql_registry import SqlRegistry from lsst.daf.butler.transfers import RepoExportContext from lsst.resources import ResourcePath from lsst.utils.introspection import get_class_of From baadca2cbad0511efa50504bf812e268c883b079 Mon Sep 17 00:00:00 2001 From: Andy Salnikov Date: Fri, 3 Nov 2023 12:36:03 -0700 Subject: [PATCH 3/3] Fix mypy complaintS --- python/lsst/daf/butler_migrate/registry.py | 6 +-- .../script/rewrite_sqlite_registry.py | 37 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/lsst/daf/butler_migrate/registry.py b/python/lsst/daf/butler_migrate/registry.py index ac70680..63ba0ea 100644 --- a/python/lsst/daf/butler_migrate/registry.py +++ b/python/lsst/daf/butler_migrate/registry.py @@ -21,10 +21,10 @@ from __future__ import annotations -from lsst.daf.butler import Registry +from lsst.daf.butler.registry.sql_registry import SqlRegistry -def make_registry(repository: str, writeable: bool = True) -> Registry: +def make_registry(repository: str, writeable: bool = True) -> SqlRegistry: """Make Registry instance. Parameters @@ -34,4 +34,4 @@ def make_registry(repository: str, writeable: bool = True) -> Registry: writeable : `bool`, optional If `True` (default) create a read-write connection to the database. """ - return Registry.fromConfig(config=repository, writeable=writeable) + return SqlRegistry.fromConfig(config=repository, writeable=writeable) diff --git a/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py b/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py index 51bf3f9..a5f6d11 100644 --- a/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py +++ b/python/lsst/daf/butler_migrate/script/rewrite_sqlite_registry.py @@ -29,6 +29,7 @@ from lsst.daf.butler import Butler, Config, DatasetAssociation, DatasetId, DatasetRef, SkyPixDimension from lsst.daf.butler.datastores.fileDatastore import FileDatastore +from lsst.daf.butler.direct_butler import DirectButler from lsst.daf.butler.registry import CollectionType from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase from lsst.daf.butler.registry.sql_registry import SqlRegistry @@ -56,11 +57,12 @@ def rewrite_sqlite_registry(source: str) -> None: """ # Create the source butler early so we can ask it questions # without assuming things. - source_butler = Butler(source, writeable=False) - assert isinstance(source_butler.registry, SqlRegistry), "Expecting SqlRegistry instance" + source_butler = Butler.from_config(source, writeable=False) + assert isinstance(source_butler, DirectButler) + assert isinstance(source_butler._registry, SqlRegistry), "Expecting SqlRegistry instance" # Check that we are really working with a SQLite database. - if not isinstance(source_butler.registry._db, SqliteDatabase): + if not isinstance(source_butler._registry._db, SqliteDatabase): raise RuntimeError("This command can only be used on SQLite registries.") # The source butler knows where its config came from. @@ -120,15 +122,16 @@ def rewrite_sqlite_registry(source: str) -> None: ) # Create destination butler - dest_butler = Butler(dest_config, writeable=True) - assert isinstance(dest_butler.registry, SqlRegistry), "Expecting SqlRegistry instance" - assert isinstance(dest_butler.registry._db, SqliteDatabase), "Expecting SqliteDatabase instance" + dest_butler = Butler.from_config(dest_config, writeable=True) + assert isinstance(dest_butler, DirectButler) + assert isinstance(dest_butler._registry, SqlRegistry), "Expecting SqlRegistry instance" + assert isinstance(dest_butler._registry._db, SqliteDatabase), "Expecting SqliteDatabase instance" transfer_everything(source_butler, dest_butler) # Obtain the name of the sqlite file at the destination. - assert dest_butler.registry._db.filename is not None, "Expecting non-None filename from registry" - dest_registry_uri = ResourcePath(dest_butler.registry._db.filename) + assert dest_butler._registry._db.filename is not None, "Expecting non-None filename from registry" + dest_registry_uri = ResourcePath(dest_butler._registry._db.filename) # Finished with writing to the destination butler so # delete the variable to ensure we can't do any more. @@ -138,8 +141,8 @@ def rewrite_sqlite_registry(source: str) -> None: # and move the existing registry to a backup. # Relocate the source registry first - assert source_butler.registry._db.filename is not None, "Expecting non-None filename from registry" - source_registry_uri = ResourcePath(source_butler.registry._db.filename) + assert source_butler._registry._db.filename is not None, "Expecting non-None filename from registry" + source_registry_uri = ResourcePath(source_butler._registry._db.filename) new_basename = "original_" + source_registry_uri.basename() backup_registry_uri = source_registry_uri.updatedFile(new_basename) os.rename(source_registry_uri.ospath, backup_registry_uri.ospath) @@ -160,16 +163,16 @@ def rewrite_sqlite_registry(source: str) -> None: print(f"Successfully rewrote registry for butler at {source_config_uri}") -def transfer_everything(source_butler: Butler, dest_butler: Butler) -> None: +def transfer_everything(source_butler: DirectButler, dest_butler: DirectButler) -> None: """Transfer all content from one butler to another butler. Assumes that both registries have a common dimension universe. Parameters ---------- - source_butler : `~lsst.daf.butler.Butler` + source_butler : `~lsst.daf.butler.direct_butler.DirectButler` Butler to use as a source of information. - dest_butler : `~lsst.daf.butler.Butler` + dest_butler : `~lsst.daf.butler.direct_butler.DirectButler` Butler to receive all the content. """ # Read all the datasets we are going to transfer, removing duplicates. @@ -237,14 +240,14 @@ def create_associations( ) -def transfer_non_datasets(source_butler: Butler, dest_butler: Butler) -> None: +def transfer_non_datasets(source_butler: DirectButler, dest_butler: DirectButler) -> None: """Transfer everything that isn't related to datasets. Parameters ---------- - source_butler : `~lsst.daf.butler.Butler` + source_butler : `~lsst.daf.butler.direct_butler.DirectButler` Butler to extract information from. - dest_butler : `~lsst.daf.butler.Butler` + dest_butler : `~lsst.daf.butler.direct_butler.DirectButler` Destination butler. """ # Use a string buffer to save on file I/O. This might result in twice @@ -256,7 +259,7 @@ def transfer_non_datasets(source_butler: Butler, dest_butler: Butler) -> None: BackendClass = get_class_of(source_butler._config["repo_transfer_formats", "yaml", "export"]) backend = BackendClass(yamlBuffer) exporter = RepoExportContext( - source_butler.registry, source_butler.datastore, backend, directory=None, transfer=None + source_butler._registry, source_butler.datastore, backend, directory=None, transfer=None ) # Export all the collections.