Merge pull request #40 from lsst-dm/tickets/DM-44546

Ignore documentation string mismatch in dimension validation

.github/workflows/build_docs.yaml

@@ -25,25 +25,21 @@ jobs:
       - name: Install sqlite
         run: sudo apt-get install sqlite libyaml-dev
 
-      - name: Set the VIRTUAL_ENV variable for uv to work
-        run: |
-          echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV
-
       - name: Update pip/wheel infrastructure
         run: |
           python -m pip install --upgrade pip
           pip install uv
-          uv pip install wheel
+          uv pip install --system wheel
 
       - name: Install dependencies
         run: |
-          uv pip install -r requirements.txt
+          uv pip install --system -r requirements.txt
 
       - name: Build and install
-        run: uv pip install --no-deps -v -e .
+        run: uv pip install --system --no-deps -v -e .
 
       - name: Install documenteer
-        run: uv pip install 'documenteer[pipelines]==0.8.2'
+        run: uv pip install --system 'documenteer[pipelines]==0.8.2'
 
       - name: Build documentation
         env:

migrations/dimensions-config/1fae088c80b6.py

@@ -84,8 +84,6 @@ def _validate_initial_dimension_universe(ctx: MigrationContext) -> None:
             ctx.attributes.validate_dimensions_json(5)
         except ValueError as e:
             e.add_note(
-                "Repositories originally created at dimension universe 1 or earlier may have incorrect"
-                " documentation strings.\n"
                 "Re-run butler migrate with the flag '--options allow_dimension_universe_mismatch=1' to"
                 " bypass this check.\n"
                 "This will overwrite any customizations made to the dimension universe."

python/lsst/daf/butler_migrate/butler_attributes.py

@@ -219,7 +219,7 @@ def validate_dimensions_json(self, expected_universe_version: int) -> None:
         expected_json = load_historical_dimension_universe_json(expected_universe_version)
         actual_json = self._load_dimensions_json()
         diff = compare_json_strings(expected_json, actual_json)
-        if diff is not None:
+        if diff is not None and not _is_expected_dimensions_json_mismatch(expected_json, actual_json):
             err = ValueError(
                 "dimensions.json stored in database does not match expected"
                 f" daf_butler universe version {expected_universe_version}."
@@ -241,3 +241,37 @@ def replace_dimensions_json(self, universe_version: int) -> None:
         """
         dimensions = load_historical_dimension_universe_json(universe_version)
         self.update(_DIMENSIONS_JSON_KEY, dimensions)
+
+
+def _is_expected_dimensions_json_mismatch(expected_json: str, actual_json: str) -> bool:
+    # Return `True` if the two dimension universe JSON strings differ only in
+    # ways expected because of the history of this data.  Older repositories
+    # that have been previously migrated have some documentation strings that
+    # don't match the current dimension universe because of how dimension
+    # universes were patched in earlier migrations.
+
+    diff = compare_json_strings(
+        _strip_doc_properties_from_json(expected_json), _strip_doc_properties_from_json(actual_json)
+    )
+    return diff is None
+
+
+def _strip_doc_properties_from_json(json_string: str) -> str:
+    # Remove any properties named 'doc' from objects in the given JSON string.
+    dictionary = json.loads(json_string)
+    stripped = _strip_doc_properties_from_dict(dictionary)
+    return json.dumps(stripped)
+
+
+def _strip_doc_properties_from_dict(dictionary: dict[str, object]) -> dict[str, object]:
+    # Recursively remove any properties named 'doc' from the given dict or any
+    # dicts in its values.
+    stripped: dict[str, object] = {}
+    for key, value in dictionary.items():
+        if key != "doc":
+            if isinstance(value, dict):
+                stripped[key] = _strip_doc_properties_from_dict(value)
+            else:
+                stripped[key] = value
+
+    return stripped

tests/test_dimensions_json.py

@@ -328,6 +328,28 @@ def test_validate_dimensions_json(self) -> None:
             attribs.replace_dimensions_json(universe)
             attribs.validate_dimensions_json(universe)
 
+    def test_ignore_expected_dimension_json_mismatch(self) -> None:
+        original = '{"a": 1, "b": {"doc": "good"}}'
+        self.assertTrue(butler_attributes._is_expected_dimensions_json_mismatch(original, original))
+        # Mismatched doc but everything else OK
+        self.assertTrue(
+            butler_attributes._is_expected_dimensions_json_mismatch(original, '{"a": 1, "b": {"doc": "bad"}}')
+        )
+        # Order doesn't matter
+        self.assertTrue(
+            butler_attributes._is_expected_dimensions_json_mismatch(original, '{"b": {"doc": "bad"}, "a": 1}')
+        )
+        # Mismatched non-doc value
+        self.assertFalse(
+            butler_attributes._is_expected_dimensions_json_mismatch(
+                original, '{"a": 2, "b": {"doc": "good"}}'
+            )
+        )
+        # Mismatched value and doc
+        self.assertFalse(
+            butler_attributes._is_expected_dimensions_json_mismatch(original, '{"a": 2, "b": {"doc": "bad"}}')
+        )
+
 
 class SQLiteDimensionsJsonTestCase(DimensionsJsonTestCase, unittest.TestCase):
     """Test using SQLite backend."""