feat: Store portal title and URL

OpenDataServices/oc4ids-registry#20

Author: jarofgreen

migrations/versions/cde761a59c2f_add_portals.py

@@ -0,0 +1,32 @@
+"""add_portals
+
+Revision ID: cde761a59c2f
+Revises: b21b5de6ee2d
+Create Date: 2025-07-08 07:51:48.954914
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'cde761a59c2f'
+down_revision: Union[str, None] = 'b21b5de6ee2d'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('dataset', sa.Column('portal_url', sa.String(), nullable=True))
+    op.add_column('dataset', sa.Column('portal_title', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('dataset', 'portal_title')
+    op.drop_column('dataset', 'portal_url')
+    # ### end Alembic commands ###

oc4ids_datastore_pipeline/database.py

@@ -36,6 +36,8 @@ class Dataset(Base):
     csv_url: Mapped[Optional[str]] = mapped_column(String, nullable=True)
     xlsx_url: Mapped[Optional[str]] = mapped_column(String, nullable=True)
     updated_at: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True))
+    portal_url: Mapped[Optional[str]] = mapped_column(String, nullable=True)
+    portal_title: Mapped[Optional[str]] = mapped_column(String, nullable=True)
 
 
 def get_engine() -> Engine:

oc4ids_datastore_pipeline/pipeline.py

@@ -111,6 +111,8 @@ def save_dataset_metadata(
     json_url: Optional[str],
     csv_url: Optional[str],
     xlsx_url: Optional[str],
+    portal_title: Optional[str],
+    portal_url: Optional[str],
 ) -> None:
     logger.info(f"Saving metadata for dataset {dataset_id}")
     try:
@@ -127,6 +129,8 @@ def save_dataset_metadata(
             license_url=license_url,
             license_title=license_title,
             license_title_short=license_title_short,
+            portal_title=portal_title,
+            portal_url=portal_url,
             json_url=json_url,
             csv_url=csv_url,
             xlsx_url=xlsx_url,
@@ -157,6 +161,8 @@ def process_dataset(dataset_id: str, registry_metadata: dict[str, str]) -> None:
         json_url=json_public_url,
         csv_url=csv_public_url,
         xlsx_url=xlsx_public_url,
+        portal_title=registry_metadata["portal_title"],
+        portal_url=registry_metadata["portal_url"],
     )
     logger.info(f"Processed dataset {dataset_id}")
 

oc4ids_datastore_pipeline/registry.py

@@ -24,6 +24,8 @@ def fetch_registered_datasets() -> dict[str, dict[str, str]]:
             registered_datasets[key] = {
                 "source_url": r_data_json["fields"]["url"]["value"],
                 "country": r_data_json["fields"]["country"]["value"],
+                "portal_title": r_data_json["fields"]["portal_title"]["value"],
+                "portal_url": r_data_json["fields"]["portal_url"]["value"],
             }
         registered_datasets_count = len(registered_datasets)
         logger.info(f"Fetched URLs for {registered_datasets_count} datasets")

tests/test_registry.py

@@ -28,6 +28,8 @@ def test_fetch_registered_datasets(mocker: MockerFixture) -> None:
             "fields": {
                 "url": {"value": "https://test_dataset.json"},
                 "country": {"value": "ab"},
+                "portal_title": {"value": "Our Portal"},
+                "portal_url": {"value": "https://our.portal"},
             }
         },
     ]
@@ -37,7 +39,12 @@ def test_fetch_registered_datasets(mocker: MockerFixture) -> None:
     result = fetch_registered_datasets()
 
     assert result == {
-        "test_dataset": {"source_url": "https://test_dataset.json", "country": "ab"}
+        "test_dataset": {
+            "source_url": "https://test_dataset.json",
+            "country": "ab",
+            "portal_title": "Our Portal",
+            "portal_url": "https://our.portal",
+        }
     }