Merge branch 'master' of github.com:ckan/ckanext-dcat

Author: amercader

.github/workflows/publish-pypi.yml

@@ -12,9 +12,9 @@ jobs:
     name: Build distribution
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: "3.9"
     - name: Install pypa/build

.github/workflows/test.yml

@@ -6,8 +6,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: '3.9'
       - name: Install requirements
@@ -53,7 +53,7 @@ jobs:
       CKAN_REDIS_URL: redis://redis:6379/1
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Install requirements (common)
       run: |
         pip install -r requirements.txt
@@ -77,4 +77,4 @@ jobs:
         ckan -c test.ini db init
         ckan -c test.ini db pending-migrations --apply
     - name: Run tests
-      run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=term-missing --cov-append --disable-warnings ckanext/dcat/tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=term-missing --cov-append --disable-warnings ckanext/dcat/tests

ckanext/dcat/profiles/euro_dcat_ap_3.py

@@ -6,6 +6,7 @@
     SKOS,
     ADMS,
     RDF,
+    OWL,
 )
 
 from .base import URIRefOrLiteral
@@ -31,6 +32,18 @@ def parse_dataset(self, dataset_dict, dataset_ref):
         dataset_dict = self._parse_dataset_v2_scheming(dataset_dict, dataset_ref)
 
         # DCAT AP v3: hasVersion
+        dataset_dict = self._parse_dataset_v3(dataset_dict, dataset_ref)
+
+        return dataset_dict
+
+    def _parse_dataset_v3(self, dataset_dict, dataset_ref):
+
+        # version
+        value = self._object_value(dataset_ref, DCAT.version)
+        if value:
+            dataset_dict["version"] = value
+
+        # hasVersion
         values = self._object_value_list(dataset_ref, DCAT.hasVersion)
         if values:
             dataset_dict["has_version"] = values
@@ -51,12 +64,6 @@ def graph_from_dataset(self, dataset_dict, dataset_ref):
         # DCAT AP v3 properties also applied to higher versions
         self._graph_from_dataset_v3(dataset_dict, dataset_ref)
 
-        # DCAT AP v3: List triples
-        items = [
-            ("has_version", DCAT.hasVersion, None, URIRefOrLiteral),
-        ]
-        self._add_list_triples_from_dict(dataset_dict, dataset_ref, items)
-
     def graph_from_catalog(self, catalog_dict, catalog_ref):
 
         self._graph_from_catalog_base(catalog_dict, catalog_ref)
@@ -71,6 +78,22 @@ def _graph_from_dataset_v3(self, dataset_dict, dataset_ref):
             self.g.remove((dataset_ref, RDF.type, None))
             self.g.add((dataset_ref, RDF.type, DCAT.DatasetSeries))
 
+        # version own:versionInfo -> dcat:version
+        self.g.remove((dataset_ref, OWL.versionInfo, None))
+        self._add_triple_from_dict(
+            dataset_dict,
+            dataset_ref,
+            DCAT.version,
+            "version",
+            _type=Literal,
+        )
+
+        # hasVersion
+        items = [
+            ("has_version", DCAT.hasVersion, None, URIRefOrLiteral),
+        ]
+        self._add_list_triples_from_dict(dataset_dict, dataset_ref, items)
+
         # byteSize decimal -> nonNegativeInteger
         for subject, predicate, object in self.g.triples((None, DCAT.byteSize, None)):
             if object and object.datatype == XSD.decimal:

ckanext/dcat/profiles/euro_dcat_ap_scheming.py

@@ -11,6 +11,8 @@
     SKOS,
     LOCN,
     RDFS,
+    SCHEMA,
+    TIME,
 )
 
 
@@ -30,6 +32,47 @@ def graph_from_dataset(self, dataset_dict, dataset_ref):
 
         self._graph_from_dataset_v2_scheming(dataset_dict, dataset_ref)
 
+    def _temporal_interval_details(self, interval_ref):
+        details = {}
+
+        start = self._object_value(interval_ref, DCAT.startDate)
+        end = self._object_value(interval_ref, DCAT.endDate)
+
+        if not (start or end):
+            start_nodes = [t for t in self.g.objects(interval_ref, TIME.hasBeginning)]
+            end_nodes = [t for t in self.g.objects(interval_ref, TIME.hasEnd)]
+            if start_nodes:
+                start = self._object_value_multiple_predicate(
+                    start_nodes[0],
+                    [TIME.inXSDDateTimeStamp, TIME.inXSDDateTime, TIME.inXSDDate],
+                )
+            if end_nodes:
+                end = self._object_value_multiple_predicate(
+                    end_nodes[0],
+                    [TIME.inXSDDateTimeStamp, TIME.inXSDDateTime, TIME.inXSDDate],
+                )
+
+        if not (start or end):
+            start = self._object_value(interval_ref, SCHEMA.startDate)
+            end = self._object_value(interval_ref, SCHEMA.endDate)
+
+        if start:
+            details["start"] = start
+        if end:
+            details["end"] = end
+
+        return details
+
+    def _temporal_coverage_details(self, dataset_ref):
+        temporal_coverage = []
+
+        for interval_ref in self.g.objects(dataset_ref, DCT.temporal):
+            details = self._temporal_interval_details(interval_ref)
+            if details:
+                temporal_coverage.append(details)
+
+        return temporal_coverage
+
     def _parse_dataset_v2_scheming(self, dataset_dict, dataset_ref):
         """
         Modify the dataset_dict generated by the euro_dcat_ap and euro_dcat_ap_2
@@ -112,12 +155,14 @@ def _prune_agent_translations(agent_list):
                 new_extras = []
                 new_dict = {}
                 check_name = new_fields_mapping.get(field_name, field_name)
+                subfield_names = [
+                    subfield["field_name"]
+                    for subfield in schema_field["repeating_subfields"]
+                ]
                 for extra in dataset_dict.get("extras", []):
                     if extra["key"].startswith(f"{check_name}_"):
                         subfield = extra["key"][extra["key"].index("_") + 1 :]
-                        if subfield in [
-                            f["field_name"] for f in schema_field["repeating_subfields"]
-                        ]:
+                        if subfield in subfield_names:
                             new_dict[subfield] = extra["value"]
                         else:
                             new_extras.append(extra)
@@ -126,8 +171,22 @@ def _prune_agent_translations(agent_list):
                         new_dict["geom"] = extra["value"]
                     else:
                         new_extras.append(extra)
+                # Legacy profiles may have already promoted namespaced extras such as
+                # `temporal_start` to first-level schema fields before we get here.
+                # Rebuild the repeating structure from those root fields as well.
+                for subfield in subfield_names:
+                    legacy_field_name = f"{check_name}_{subfield}"
+                    value = dataset_dict.get(legacy_field_name)
+                    if value not in (None, "", [], {}):
+                        new_dict.setdefault(subfield, value)
+                if field_name == "temporal_coverage":
+                    temporal_coverage = self._temporal_coverage_details(dataset_ref)
+                    if temporal_coverage:
+                        dataset_dict[field_name] = temporal_coverage
+                        dataset_dict["extras"] = new_extras
+                        continue
                 if new_dict:
-                    dataset_dict[field_name] = [new_dict]
+                    dataset_dict.setdefault(field_name, [new_dict])
                     dataset_dict["extras"] = new_extras
 
         # Contact details

ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_parse.py

@@ -1,8 +1,11 @@
 import pytest
+from rdflib import BNode, Graph, Literal, URIRef
+from rdflib.namespace import RDF
 
 from ckan.tests.helpers import call_action
 
 from ckanext.dcat.processors import RDFParser
+from ckanext.dcat.profiles import DCAT, DCT, XSD
 from ckanext.dcat.tests.utils import BaseParseTest
 
 
@@ -160,3 +163,66 @@ def test_e2e_dcat_to_ckan(self):
         assert resource["access_services"][0]["endpoint_url"] == [
             "http://publications.europa.eu/webapi/rdf/sparql"
         ]
+
+    def test_multiple_temporal_periods_are_parsed_into_temporal_coverage(self):
+        g = Graph()
+
+        dataset_ref = URIRef("http://example.com/datasets/temporal-multi")
+        first_period = BNode()
+        second_period = BNode()
+
+        g.add((dataset_ref, RDF.type, DCAT.Dataset))
+        g.add((dataset_ref, DCT.title, Literal("Dataset with multiple periods")))
+        g.add((dataset_ref, DCT.description, Literal("Test dataset")))
+        g.add((dataset_ref, DCT.temporal, first_period))
+        g.add((dataset_ref, DCT.temporal, second_period))
+        g.add(
+            (
+                first_period,
+                DCAT.startDate,
+                Literal("2021-01-01T00:00:00Z", datatype=XSD.dateTime),
+            )
+        )
+        g.add(
+            (
+                first_period,
+                DCAT.endDate,
+                Literal("2021-12-31T00:00:00Z", datatype=XSD.dateTime),
+            )
+        )
+        g.add(
+            (
+                second_period,
+                DCAT.startDate,
+                Literal("2022-01-01T00:00:00Z", datatype=XSD.dateTime),
+            )
+        )
+        g.add(
+            (
+                second_period,
+                DCAT.endDate,
+                Literal("2022-12-31T00:00:00Z", datatype=XSD.dateTime),
+            )
+        )
+
+        p = RDFParser()
+        p.g = g
+
+        datasets = [d for d in p.datasets()]
+
+        assert len(datasets) == 1
+
+        temporal_coverage = sorted(
+            datasets[0]["temporal_coverage"], key=lambda item: item["start"]
+        )
+
+        assert temporal_coverage == [
+            {
+                "start": "2021-01-01T00:00:00+00:00",
+                "end": "2021-12-31T00:00:00+00:00",
+            },
+            {
+                "start": "2022-01-01T00:00:00+00:00",
+                "end": "2022-12-31T00:00:00+00:00",
+            },
+        ]

ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_serialize.py

@@ -68,7 +68,7 @@ def test_e2e_ckan_to_dcat(self):
         assert self._triple(g, dataset_ref, RDF.type, DCAT.Dataset)
         assert self._triple(g, dataset_ref, DCT.title, dataset["title"])
         assert self._triple(g, dataset_ref, DCT.description, dataset["notes"])
-        assert self._triple(g, dataset_ref, OWL.versionInfo, dataset["version"])
+        assert self._triple(g, dataset_ref, DCAT.version, dataset["version"])
 
         # Standard fields
         assert self._triple(g, dataset_ref, DCT.identifier, dataset["identifier"])

ckanext/dcat/tests/profiles/dcat_us_3/test_dcat_us_3_profile_serialize.py

@@ -69,7 +69,7 @@ def test_e2e_ckan_to_dcat(self):
         assert self._triple(g, dataset_ref, RDF.type, DCAT.Dataset)
         assert self._triple(g, dataset_ref, DCT.title, dataset["title"])
         assert self._triple(g, dataset_ref, DCT.description, dataset["notes"])
-        assert self._triple(g, dataset_ref, OWL.versionInfo, dataset["version"])
+        assert self._triple(g, dataset_ref, DCAT.version, dataset["version"])
 
         # Standard fields
         assert self._triple(g, dataset_ref, DCT.identifier, dataset["identifier"])