Merge pull request #39 from candleindark/enh

Organize validation reports to dicts

src/dandisets_linkml_status_tools/cli/__init__.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated
 
@@ -9,13 +10,15 @@
 from pydantic2linkml.cli.tools import LogLevel
 
 from dandisets_linkml_status_tools.models import (
-    ASSET_PYDANTIC_REPORT_LIST_ADAPTER,
+    ASSET_VALIDATION_REPORTS_ADAPTER,
     DANDI_METADATA_LIST_ADAPTER,
-    DANDISET_PYDANTIC_REPORT_LIST_ADAPTER,
+    DANDISET_VALIDATION_REPORTS_ADAPTER,
     AssetValidationReport,
+    AssetValidationReportsType,
     Config,
     DandiMetadata,
     DandisetValidationReport,
+    DandisetValidationReportsType,
 )
 from dandisets_linkml_status_tools.tools import (
     compile_dandiset_linkml_translation_report,
@@ -151,9 +154,9 @@ def manifests(
         reports_dir_path / ASSET_PYDANTIC_VALIDATION_REPORTS_FILE_NAME
     )
 
-    def append_dandiset_validation_report() -> None:
+    def add_dandiset_validation_report() -> None:
         """
-        Append a `DandisetValidationReport` object to `dandiset_validation_reports`
+        Add a `DandisetValidationReport` object to `dandiset_validation_reports`
         if the current dandiset version directory contains a dandiset metadata file.
         """
         dandiset_metadata_file_path = version_dir / DANDISET_FILE_NAME
@@ -171,23 +174,25 @@ def append_dandiset_validation_report() -> None:
 
         dandiset_metadata = dandiset_metadata_file_path.read_text()
         pydantic_validation_errs = pydantic_validate(dandiset_metadata, model)
+
         # noinspection PyTypeChecker
-        dandiset_validation_reports.append(
+        dandiset_validation_reports[dandiset_identifier][dandiset_version] = (
             DandisetValidationReport(
                 dandiset_identifier=dandiset_identifier,
                 dandiset_version=dandiset_version,
                 pydantic_validation_errs=pydantic_validation_errs,
             )
         )
+
         logger.info(
             "Dandiset %s:%s: Generated and added a dandiset validation report",
             dandiset_identifier,
             dandiset_version,
         )
 
-    def extend_asset_validation_reports() -> None:
+    def add_asset_validation_reports() -> None:
         """
-        Extend `asset_validation_reports` with `AssetValidationReport` objects if the
+        Add `AssetValidationReport` objects to `asset_validation_reports` if the
         current dandiset version directory contains an assets metadata file.
         """
         assets_metadata_file_path = version_dir / ASSETS_FILE_NAME
@@ -218,12 +223,13 @@ def extend_asset_validation_reports() -> None:
             )
             raise RuntimeError(msg) from e
 
+        reports_of_specific_dandiset_version: list[AssetValidationReport] = []
         for asset_metadata in assets_metadata_python:
             asset_id = asset_metadata.get("id")
             asset_path = asset_metadata.get("path")
             pydantic_validation_errs = pydantic_validate(asset_metadata, model)
             # noinspection PyTypeChecker
-            asset_validation_reports.append(
+            reports_of_specific_dandiset_version.append(
                 AssetValidationReport(
                     dandiset_identifier=dandiset_identifier,
                     dandiset_version=dandiset_version,
@@ -232,14 +238,21 @@ def extend_asset_validation_reports() -> None:
                     pydantic_validation_errs=pydantic_validation_errs,
                 )
             )
+
+        # Add the asset validation reports of the current dandiset version to
+        # the asset_validation_reports dictionary
+        asset_validation_reports[dandiset_identifier][
+            dandiset_version
+        ] = reports_of_specific_dandiset_version
+
         logger.info(
             "Dandiset %s:%s: Generated and added asset validation reports",
             dandiset_identifier,
             dandiset_version,
         )
 
-    dandiset_validation_reports: list[DandisetValidationReport] = []
-    asset_validation_reports: list[AssetValidationReport] = []
+    dandiset_validation_reports: DandisetValidationReportsType = defaultdict(dict)
+    asset_validation_reports: AssetValidationReportsType = defaultdict(dict)
     for dandiset_dir in get_direct_subdirs(manifest_path):
         # === In a dandiset directory ===
         dandiset_identifier = dandiset_dir.name
@@ -248,8 +261,8 @@ def extend_asset_validation_reports() -> None:
             # === In a dandiset version directory ===
             dandiset_version = version_dir.name
 
-            append_dandiset_validation_report()
-            extend_asset_validation_reports()
+            add_dandiset_validation_report()
+            add_asset_validation_reports()
 
     # Ensure directory for reports exists
     logger.info("Creating report directory: %s", reports_dir_path)
@@ -259,12 +272,12 @@ def extend_asset_validation_reports() -> None:
     write_reports(
         dandiset_pydantic_validation_reports_file_path,
         dandiset_validation_reports,
-        DANDISET_PYDANTIC_REPORT_LIST_ADAPTER,
+        DANDISET_VALIDATION_REPORTS_ADAPTER,
     )
 
     # Write the asset Pydantic validation reports to a file
     write_reports(
         asset_pydantic_validation_reports_file_path,
         asset_validation_reports,
-        ASSET_PYDANTIC_REPORT_LIST_ADAPTER,
+        ASSET_VALIDATION_REPORTS_ADAPTER,
     )

src/dandisets_linkml_status_tools/models.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from collections.abc import Sequence
 from datetime import datetime
 from pathlib import Path
@@ -6,7 +7,14 @@
 from dandi.dandiapi import VersionStatus
 from jsonschema import ValidationError
 from linkml.validator.report import ValidationResult
-from pydantic import AfterValidator, BaseModel, Json, PlainSerializer, TypeAdapter
+from pydantic import (
+    AfterValidator,
+    BaseModel,
+    Field,
+    Json,
+    PlainSerializer,
+    TypeAdapter,
+)
 from pydantic2linkml.cli.tools import LogLevel
 from typing_extensions import TypedDict  # Required for Python < 3.12 by Pydantic
 
@@ -231,10 +239,20 @@ def dandiset_schema_version(self) -> str:
     linkml_validation_errs: LinkmlValidationErrsType = []
 
 
+DandisetValidationReportsType: TypeAlias = defaultdict[
+    str, Annotated[dict[str, DandisetValidationReport], Field(default_factory=dict)]
+]
+AssetValidationReportsType: TypeAlias = defaultdict[
+    str, Annotated[dict[str, list[AssetValidationReport]], Field(default_factory=dict)]
+]
+ValidationReportsType: TypeAlias = (
+    DandisetValidationReportsType | AssetValidationReportsType
+)
+
 # Type adapters for various types (this section should be at the end of this file)
 DANDI_METADATA_ADAPTER = TypeAdapter(DandiMetadata)
 PYDANTIC_VALIDATION_ERRS_ADAPTER = TypeAdapter(PydanticValidationErrsType)
 LINKML_VALIDATION_ERRS_ADAPTER = TypeAdapter(LinkmlValidationErrsType)
 DANDI_METADATA_LIST_ADAPTER = TypeAdapter(list[DandiMetadata])
-DANDISET_PYDANTIC_REPORT_LIST_ADAPTER = TypeAdapter(list[DandisetValidationReport])
-ASSET_PYDANTIC_REPORT_LIST_ADAPTER = TypeAdapter(list[AssetValidationReport])
+DANDISET_VALIDATION_REPORTS_ADAPTER = TypeAdapter(DandisetValidationReportsType)
+ASSET_VALIDATION_REPORTS_ADAPTER = TypeAdapter(AssetValidationReportsType)

src/dandisets_linkml_status_tools/tools.py

@@ -29,7 +29,7 @@
     JsonschemaValidationErrorType,
     LinkmlValidationErrsType,
     PydanticValidationErrsType,
-    ValidationReport,
+    ValidationReportsType,
 )
 
 try:
@@ -99,14 +99,15 @@ def pydantic_validate(data: DandiMetadata | str, model: type[BaseModel]) -> str:
 
 
 def write_reports(
-    file_path: Path, reports: list[ValidationReport], type_adapter: TypeAdapter
+    file_path: Path, reports: ValidationReportsType, type_adapter: TypeAdapter
 ) -> None:
     """
-    Write a given list of validation reports to a specified file
+    Write a given collection of validation reports to a specified file
 
     :param file_path: The path specifying the file to write the reports to
-    :param reports: The list of validation reports to write
-    :param type_adapter: The type adapter to use for serializing the list of reports
+    :param reports: The collection of validation reports to write
+    :param type_adapter: The type adapter to use for serializing the collection of
+        reports
     """
     file_path.write_bytes(type_adapter.dump_json(reports, indent=2))