Merge pull request #862 from lsst/tickets/DM-39915

DM-39915: Make butler.datastore an internal property

.github/workflows/build_docs.yaml

@@ -16,7 +16,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
           cache: "pip"
@@ -45,8 +45,11 @@ jobs:
       - name: Build and install
         run: pip install --no-deps -v .
 
+      - name: Install graphviz
+        run: sudo apt-get install graphviz
+
       - name: Install documenteer
-        run: pip install 'documenteer[pipelines]<0.8'
+        run: pip install 'documenteer[pipelines]>=0.8'
 
       - name: Build documentation
         working-directory: ./doc

.github/workflows/do_not_merge.yaml

@@ -0,0 +1,41 @@
+name: "Check commits can be merged"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  do-not-merge-checker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check that there are no commits that should not be merged
+        uses: gsactions/commit-message-checker@v2
+        with:
+          excludeDescription: "true"  # optional: this excludes the description body of a pull request
+          excludeTitle: "true"  # optional: this excludes the title of a pull request
+          checkAllCommitMessages: "true"  # optional: this checks all commits associated with a pull request
+          accessToken: ${{ secrets.GITHUB_TOKEN }}  # github access token is only required if checkAllCommitMessages is true
+          # Check for message indicating that there is a commit that should
+          # not be merged.
+          pattern: ^(?!DO NOT MERGE)
+          flags: "i"
+          error: |
+            "This step failed because there is a commit containing the text
+            'DO NOT MERGE'. Remove this commit from the branch before merging
+            or change the commit summary."
+
+      - uses: actions/checkout@v3
+
+      - name: Check requirements.txt for branches
+        shell: bash
+        run: |
+          FILE=requirements.txt
+          MATCH=tickets/DM-
+          if grep -q $MATCH $FILE
+          then
+            echo "Ticket branches found in $FILE:"
+            grep -n $MATCH $FILE
+            exit 1
+          fi

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
@@ -23,6 +23,6 @@ repos:
         name: isort (python)
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.275
+    rev: v0.0.277
     hooks:
       - id: ruff

doc/changes/DM-39915.feature.rst

@@ -0,0 +1,6 @@
+Added new APIs to support the deprecation of ``LimitedButler.datastore``:
+* ``LimitedButler.get_datastore_roots`` can be used to retrieve any root URIs associated with attached datastores.
+  If a datastore does not support the concept it will return `None` for its root URI.
+* ``LimitedButler.get_datastore_names`` can be used to retrieve the names of the internal datastores.
+* ``LimitedButler.get_many_uris`` allows for the bulk retrieval of URIs from a list of refs.
+* Also made ``getURI`` and ``getURIs`` available for ``LimitedButler``.

doc/changes/DM-39915.removal.rst

@@ -0,0 +1,2 @@
+The semi-public ``Butler.datastore`` property has now been deprecated.
+The ``LimitedButler`` API has been expanded such that there is no longer any need for anyone to access the datastore class directly.

doc/lsst.daf.butler/concreteStorageClasses.rst

@@ -26,18 +26,20 @@ Retrieving this dataset via:
 .. code-block:: python
 
    butler.get(
-       "deepCoadd_obj", ...,
+       "deepCoadd_obj",
+       ...,
        parameters={
-           "columns": {"dataset": "meas",
-                       "filter": ["HSC-R", "HSC-I"],
-                       "column": ["base_SdssShape_xx", "base_SdssShape_yy"]}
-       }
+           "columns": {
+               "dataset": "meas",
+               "filter": ["HSC-R", "HSC-I"],
+               "column": ["base_SdssShape_xx", "base_SdssShape_yy"],
+           }
+       },
    )
 
 is equivalent to (but potentially much more efficient than):
 
 .. code-block:: python
 
    full = butler.get("deepCoadd_obj", ...)
-   full.loc[:, ["meas", ["HSC-R", "HSC-I"],
-                ["base_SdssShape_xx", "base_SdssShape_yy"]]]
+   full.loc[:, ["meas", ["HSC-R", "HSC-I"], ["base_SdssShape_xx", "base_SdssShape_yy"]]]

doc/lsst.daf.butler/writing-subcommands.rst

@@ -44,18 +44,21 @@ Example of a command or subcommand definition:
 
     import click
 
+
     @click.group()
     def git():
         """An example git-style interface."""
         pass
 
+
     # Notice this uses "@git" instead of "@click", this adds the command to the
     # git group.
     @git.command()
     def pull():
         """An example 'pull' subcommand."""
         print("pull!")
 
+
     if __name__ == "__main__":
         git()
 
@@ -133,18 +136,21 @@ An example of a subcommand that uses options:
 
     import click
 
+
     @click.group()
     def git():
         """An example git-style interface."""
         pass
 
+
     @git.command()
     @click.option("-m", "--message", help="commit message")
     @click.option("-a", "--all", help="commit all changed files", is_flag=True)
     def commit(all, message):
         """An example 'commit' subcommand."""
         print(f"commit. all: {all}, message: {message}")
 
+
     if __name__ == "__main__":
         git()
 
@@ -183,11 +189,13 @@ An example of a subcommand that uses arguments:
 
     import click
 
+
     @click.group()
     def git():
         """An example git-style interface."""
         pass
 
+
     @git.command()
     @click.argument("branch")
     def checkout(branch):
@@ -199,6 +207,7 @@ An example of a subcommand that uses arguments:
         """
         print(f"checkout branch {branch}")
 
+
     if __name__ == "__main__":
         git()
 
@@ -257,24 +266,29 @@ An example implementation of ``git checkout`` that uses MWArgumentDecorator and
     # as the argument name in the command function where it is used. (This is
     # available for any click.option)
     new_branch_option = MWOptionDecorator(
-        "-b", "make_new_branch",
+        "-b",
+        "make_new_branch",
         help="create and checkout a new branch",
-        is_flag=True)  # is_flag makes the option take no values, uses a bool
-                       # which is true if the option is passed and false by
-                       # default.
+        # is_flag makes the option take no values, uses a bool
+        # which is true if the option is passed and false by default.
+        is_flag=True,
+    )
+
 
     @click.group()
     def git():
         """An example git-style interface."""
         pass
 
+
     @git.command()
     @branch_argument()
     @new_branch_option()
     def checkout(branch, make_new_branch):
         """An example 'checkout' subcommand."""
         print(f"checkout branch {branch}, make new:{make_new_branch}")
 
+
     if __name__ == "__main__":
         git()
 
@@ -311,13 +325,13 @@ Defines an Option Group decorator:
    :name: option-group-example
 
     class pipeline_build_options(OptionGroup):  # noqa: N801
-    """Decorator to add options to a command function for building a pipeline.
-    """
+        """Decorator to add options to a command function for building a pipeline."""
 
-    def __init__(self):
-        self.decorators = [
-            ctrlMpExecOpts.pipeline_option(),
-            ctrlMpExecOpts.task_option()]
+        def __init__(self):
+            self.decorators = [
+                ctrlMpExecOpts.pipeline_option(),
+                ctrlMpExecOpts.task_option(),
+            ]
 
 Uses an Option Group decorator:
 
@@ -440,19 +454,19 @@ It's easy to create a new kind of command by copying the template below and maki
 
     from lsst.daf.butler.cli.butler import LoaderCLI
 
+
     # Change the class name to better describe your command.
     class ButlerCLI(LoaderCLI):
-
         # Replace this value with the import path to your `cmd` module.
         localCmdPkg = "lsst.daf.butler.cli.cmd"
 
         # Replace this value with the manifest environment variable described
         # above.
         pluginEnvVar = "DAF_BUTLER_PLUGINS"
 
+
     # Change ``cls=ButlerCLI`` to be the same as your new class name above.
-    @click.command(cls=ButlerCLI,
-                   context_settings=dict(help_option_names=["-h", "--help"]))
+    @click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
     # You can remove log_level_option if you do not support it. You can add
     # other command options here. (Subcommand options are declared elsewhere).
     @log_level_option()

pyproject.toml

@@ -139,7 +139,6 @@ convention = "numpy"
 add-ignore = ["D107", "D105", "D102", "D100", "D200", "D205", "D400", "D104"]
 
 [tool.coverage.report]
-show_missing = true
 exclude_lines = [
     "pragma: no cover",
     "raise AssertionError",
@@ -170,7 +169,6 @@ ignore = [
     "D200",
     "D205",
     "D400",
-    "E402", # Module level import not at top of file (against Rubin style)
 ]
 line-length = 110
 select = [