Merge pull request #78 from opendata-swiss/feat/harvest-and-export-re…

…source-documentation-field feat: Harvest and export distribution field 'documentation'
opendata-swiss · Sep 19, 2023 · 8944ced · 8944ced
2 parents 82ada5d + dace014
commit 8944ced
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 3 deletions.
diff --git a/ckanext/dcatapchharvest/profiles.py b/ckanext/dcatapchharvest/profiles.py
@@ -522,6 +522,11 @@ def parse_dataset(self, dataset_dict, dataset_ref):  # noqa
                     resource_dict.get('format')):
                 resource_dict['media_type'] = resource_dict['format']
 
+            # Documentation
+            resource_dict['documentation'] = self._object_value_list(
+                distribution, FOAF.page
+            )
+
             # Timestamp fields
             for key, predicate in (
                     ('issued', DCT.issued),
@@ -778,7 +783,6 @@ def graph_from_dataset(self, dataset_dict, dataset_ref):  # noqa
 
             #  Lists
             items = [
-                ('documentation', FOAF.page, None, Literal),
                 ('language', DCT.language, None, Literal),
                 ('conforms_to', DCT.conformsTo, None, Literal),
             ]
@@ -810,6 +814,13 @@ def graph_from_dataset(self, dataset_dict, dataset_ref):  # noqa
             elif download_url:
                 g.add((distribution, DCAT.accessURL, URIRef(download_url)))
 
+            # Documentation
+            documentation = resource_dict.get('documentation', [])
+            for link in documentation:
+                doc = URIRef(link)
+                g.add((doc, RDF.type, FOAF.Document))
+                g.add((distribution, FOAF.page, doc))
+
             # Format
             if resource_dict.get('format'):
                 g.add((

diff --git a/ckanext/dcatapchharvest/tests/fixtures/1901.xml b/ckanext/dcatapchharvest/tests/fixtures/1901.xml
@@ -16,6 +16,12 @@
         <dct:identifier>346265-fr@bundesamt-fur-statistik-bfs</dct:identifier>
         <dct:format>HTML</dct:format>
         <dct:rights>NonCommercialAllowed-CommercialWithPermission-ReferenceRequired</dct:rights>
+        <foaf:page>
+          <foaf:Document rdf:about="https://example.com/documentation-distribution-1"/>
+        </foaf:page>
+        <foaf:page>
+          <foaf:Document rdf:about="https://example.com/documentation-distribution-2"/>
+        </foaf:page>
       </dcat:Distribution>
     </dcat:distribution>
     <dct:language>de</dct:language>

diff --git a/ckanext/dcatapchharvest/tests/fixtures/dataset.json b/ckanext/dcatapchharvest/tests/fixtures/dataset.json
@@ -94,8 +94,18 @@
     "it": ""
   },
   "documentation": [
-    "http://dataset.info.org/doc1",
-    "http://dataset.info.org/doc2"
+    "https://example.com/documentation-dataset-1",
+    "https://example.com/documentation-dataset-2"
+  ],
+  "resources": [
+    {
+      "id": "e2c50e70-67ad-4f86-bb1b-3f93867eadaa",
+      "package_id": "4b6fe9ca-dc77-4cec-92a4-55c6624a5bd6",
+      "documentation": [
+        "https://example.com/documentation-resource-1",
+        "https://example.com/documentation-resource-2"
+      ]
+    }
   ],
   "extras": [
     {

diff --git a/ckanext/dcatapchharvest/tests/test_dcatap_ch_parse.py b/ckanext/dcatapchharvest/tests/test_dcatap_ch_parse.py
@@ -114,6 +114,12 @@ def test_dataset_all_fields(self):
         eq_(resource['url'], u'https://www.bfs.admin.ch/asset/fr/hs-b-00.01-jb-1901')
         assert 'download_url' not in resource, "download_url not available on resource"
 
+        # Lists
+        eq_(
+            sorted(resource['documentation']),
+            ['https://example.com/documentation-distribution-1', 'https://example.com/documentation-distribution-2']
+        )
+
         # Distribution URI
         eq_(resource['uri'], u'https://opendata.swiss/dataset/7451e012-64b2-4bbc-af20-a0e2bc61b585/resource/c8ec6ca0-6923-4cf3-92f2-95a10e6f8e25')
 

diff --git a/ckanext/dcatapchharvest/tests/test_dcatap_ch_serialize.py b/ckanext/dcatapchharvest/tests/test_dcatap_ch_serialize.py
@@ -70,6 +70,14 @@ def test_graph_from_dataset(self):
             for value in values:
                 assert self._triple(g, dataset_ref, item[1], item[2](value))
 
+        # Resources
+        eq_(len([t for t in g.triples((dataset_ref, DCAT.distribution, None))]), len(dataset["resources"]))
+        for resource_dict in dataset.get("resources", []):
+            distribution = URIRef(dh.resource_uri(resource_dict))
+            assert self._triple(g, distribution, RDF.type, DCAT.Distribution)
+            for link in resource_dict.get("documentation", []):
+                assert self._triple(g, distribution, FOAF.page, URIRef(link))
+
     def test_graph_from_dataset_uri(self):
         """Tests that datasets (resources) with a uri from the test system
         have that uri changed to reference the prod system when they are output