chore: Upgrade to dcat 2.1.0 and harvest extension to 1.6.0

GenomicDataInfrastructure · Nov 19, 2024 · 5c5878c · 5c5878c
1 parent d4ea0fb
commit 5c5878c
Show file tree

Hide file tree

Showing 13 changed files with 36 additions and 412 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -43,9 +43,9 @@ jobs:
         pip install -r dev-requirements.txt
         pip install --upgrade pytest-rerunfailures
         pip install -e 'git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]'
-        pip install -e 'git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest[requirements]'
-        pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v2.0.0#egg=ckanext-dcat[requirements]'
-        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v2.0.0/requirements.txt
+        pip install -e 'git+https://github.com/ckan/ckanext-harvest.git@v1.6.0#egg=ckanext-harvest[requirements]'
+        pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v2.1.0#egg=ckanext-dcat[requirements]'
+        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v2.1.0/requirements.txt
         python3 setup.py develop
     - name: Setup extension
       run: |

diff --git a/README.md b/README.md
@@ -107,14 +107,14 @@ Fairdatapoint plugin depends on `ckanext-scheming`, `ckanext-harvester` and `cka
 otherwise run:
 ```commandline
 pip install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-scheming[requirements]'
-pip install -e 'git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest[requirements]'
-pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v2.0.0#egg=ckanext-dcat'
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v2.0.0/requirements.txt
+pip install -e 'git+https://github.com/ckan/ckanext-harvest.git@v1.6.0#egg=ckanext-harvest[requirements]'
+pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v2.1.0#egg=ckanext-dcat'
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v2.1.0/requirements.txt
 ```
 
 ## Tests
 
-To run the tests go to [GDI harvester test information](https://genomicdatainfrastructure.github.io/gdi-userportal-docs/docs/ckan/extension-local-setup-and-testing/)
+To run the tests go to [GDI harvester test information](https://genomicdatainfrastructure.github.io/gdi-userportal-docs/developer-guide/ckan/extension-local-setup-and-testing/)
 
 
 ## Releasing a new version of ckanext-fairdatapoint

diff --git a/ckanext/fairdatapoint/profiles.py b/ckanext/fairdatapoint/profiles.py
@@ -53,70 +53,6 @@ class FAIRDataPointDCATAPProfile(EuropeanDCATAP3Profile):
     def parse_dataset(self, dataset_dict: Dict, dataset_ref: URIRef) -> Dict:
         super(FAIRDataPointDCATAPProfile, self).parse_dataset(dataset_dict, dataset_ref)
 
-        #dataset_dict = self._parse_contact_point(dataset_dict, dataset_ref)
-        dataset_dict = self._parse_creator(dataset_dict, dataset_ref)
-
-       ## dataset_dict = _convert_extras_to_declared_schema_fields(dataset_dict)
-
         dataset_dict['tags'] = validate_tags(dataset_dict['tags'])
 
         return dataset_dict
-
-    def _contact_point_details(self, subject, predicate) -> List:
-        """
-        Overrides RDFProfile._contact_details so uri is taken from hasUID for VCard
-        """
-        contact_list = []
-
-        for agent in self.g.objects(subject, predicate):
-            contact = {
-                'uri': (str(agent) if isinstance(agent, URIRef)
-                        else self._get_vcard_property_value(agent, VCARD.hasUID)),
-                'name': self._get_vcard_property_value(agent, VCARD.hasFN, VCARD.fn),
-                'email': self._without_mailto(self._get_vcard_property_value(agent, VCARD.hasEmail))}
-
-            contact_list.append(contact)
-
-        return contact_list
-
-    def _parse_contact_point(self, dataset_dict: Dict, dataset_ref: URIRef) -> Dict:
-        """
-        ckan-dcat extension implies there can be just one contact point and in case a list is provided by source only
-        last value is taken. Besides it never solves uri from a VCard object. This function parses DCAT.contactPoint 
-        information to a list of `pontact_point` dictionaries and replaces ckan-dcat values
-        """
-        contact_point = self._contact_point_details(subject=dataset_ref, predicate=DCAT.contactPoint)
-        dcat_profile_contact_fields = ['contact_name', 'contact_email', 'contact_uri']
-        if contact_point:
-            dataset_dict['extras'].append({'key': 'contact_point', 'value': contact_point})
-            # Remove the extras contact_ fields if they were parsed by dcat extension
-            dataset_dict['extras'] = \
-                [item for item in dataset_dict['extras'] if item.get('key') not in dcat_profile_contact_fields]
-        return dataset_dict
-
-    def _parse_creator(self, dataset_dict: Dict, dataset_ref: URIRef) -> Dict:
-        graph = self.g
-        creators = []
-        for creator_ref in graph.objects(dataset_ref, DCTERMS.creator):
-            creator = {}
-            creator_identifier = graph.value(creator_ref, DCTERMS.identifier)
-            creator_name = graph.value(creator_ref, FOAF.name)
-
-            if creator_identifier:
-                creator['identifier'] = str(creator_identifier)
-            if creator_name:
-                creator['name'] = str(creator_name)
-            else:
-                # If the creator is a URI, use it as the identifier
-                if isinstance(creator_ref, URIRef):
-                    creator['identifier'] = str(creator_ref)
-                    creator['name'] = str(creator_ref)
-                else:
-                    creator['name'] = str(creator_ref)
-
-            creators.append(creator)
-
-        if len(creators) > 0:
-            dataset_dict['creator'] = creators
-
-        return dataset_dict
diff --git a/ckanext/fairdatapoint/tests/test_data/Project_27866022694497978.ttl b/ckanext/fairdatapoint/tests/test_data/Project_27866022694497978.ttl
@@ -15,7 +15,7 @@
     dcterms:hasVersion <https://repo.metadatacenter.org/template-instances/2836bf1c-76e9-44e7-a65e-80e9ca63025a> ;
     dcterms:identifier "27866022694497978"^^xsd:token ;
     dcterms:isPartOf <https://covid19initiatives.health-ri.nl/p/ProjectOverview?focusarea=http://purl.org/zonmw/generic/10006> ;
-    dcterms:publisher <https://opal.health-ri.nl/pub/> ;
+    dcterms:publisher <https://opal.health-ri.nl/pub> ;
     dcterms:temporal [ a dcterms:PeriodOfTime ;
             dcat:endDate "2025-12-31"^^xsd:date ;
             dcat:startDate "2020-01-01"^^xsd:date ] ;

diff --git a/ckanext/fairdatapoint/tests/test_data/Project_27866022694497978_out.ttl b/ckanext/fairdatapoint/tests/test_data/Project_27866022694497978_out.ttl
@@ -15,7 +15,7 @@
     dcterms:hasVersion <https://repo.metadatacenter.org/template-instances/2836bf1c-76e9-44e7-a65e-80e9ca63025a> ;
     dcterms:identifier "27866022694497978"^^xsd:token ;
     dcterms:isPartOf <https://covid19initiatives.health-ri.nl/p/ProjectOverview?focusarea=http://purl.org/zonmw/generic/10006> ;
-    dcterms:publisher <https://opal.health-ri.nl/pub/> ;
+    dcterms:publisher <https://opal.health-ri.nl/pub> ;
     dcterms:temporal [ a dcterms:PeriodOfTime ;
             dcat:endDate "2025-12-31"^^xsd:date ;
             dcat:startDate "2020-01-01"^^xsd:date ] ;

diff --git a/ckanext/fairdatapoint/tests/test_data/contact_point_multiple_cards.ttl b/ckanext/fairdatapoint/tests/test_data/contact_point_multiple_cards.ttl
diff --git a/ckanext/fairdatapoint/tests/test_data/contact_point_multiple_urls.ttl b/ckanext/fairdatapoint/tests/test_data/contact_point_multiple_urls.ttl
diff --git a/ckanext/fairdatapoint/tests/test_data/contact_point_url.ttl b/ckanext/fairdatapoint/tests/test_data/contact_point_url.ttl
diff --git a/ckanext/fairdatapoint/tests/test_data/contact_point_vcard.ttl b/ckanext/fairdatapoint/tests/test_data/contact_point_vcard.ttl
diff --git a/ckanext/fairdatapoint/tests/test_data/creator_prisma.ttl b/ckanext/fairdatapoint/tests/test_data/creator_prisma.ttl
diff --git a/ckanext/fairdatapoint/tests/test_data/scheming/schemas/gdi_userportal.yaml b/ckanext/fairdatapoint/tests/test_data/scheming/schemas/gdi_userportal.yaml
@@ -8,6 +8,17 @@ about: DCAT-AP 3 compatible schema
 about_url: http://github.com/ckan/ckanext-dcat
 
 dataset_fields:
+
+- field_name: has_version
+  label:
+    en: Has Version
+    nl: Bevat Versie
+  preset: multiple_text
+  help_inline: true
+  help_text:
+    en: "[dct:hasVersion] This property refers to a related Dataset that is a version, edition, or adaptation of the described Dataset."
+    nl: "[dct:hasVersion] Deze eigenschap verwijst naar een gerelateerde Dataset die een versie, editie of aanpassing is van de beschreven Dataset."
+
 - field_name: issued
   label:
     en: Issued Date
@@ -26,16 +37,6 @@ dataset_fields:
     en: "[dct:modified] This property contains the most recent date on which the Dataset was changed or modified."
     nl: "[dct:modified] Deze eigenschap bevat de meest recente datum waarop de Dataset is gewijzigd of gewijzigd."
 
-- field_name: has_version
-  label:
-    en: Has Version
-    nl: Bevat Versie
-  preset: multiple_text
-  help_inline: true
-  help_text:
-    en: "[dct:hasVersion] This property refers to a related Dataset that is a version, edition, or adaptation of the described Dataset."
-    nl: "[dct:hasVersion] Deze eigenschap verwijst naar een gerelateerde Dataset die een versie, editie of aanpassing is van de beschreven Dataset."
-
 - field_name: temporal_start
   label:
     en: Temporal Start Date
@@ -56,36 +57,6 @@ dataset_fields:
     nl: "[dct:temporal] Deze eigenschap verwijst naar een tijdsperiode die door de Dataset wordt gedekt."
   preset: datetime_flex
 
-- field_name: creator
-  label: Creator
-  repeating_subfields:
-
-    - field_name: uri
-      label: Creator URI
-
-    - field_name: name
-      label: Creator Name
-
-    - field_name: email
-      label: Creator Email
-      display_snippet: email.html
-
-    - field_name: url
-      label: Creator URL
-      display_snippet: link.html
-
-    - field_name: type
-      label: Creator Type
-
-    - field_name: identifier
-      label: Creator Identifier
-      help_text:
-        en: Unique identifier for the creator, such as a ROR ID.
-        nl: Unieke identificatie voor de maker, zoals een ROR-ID.
-  help_text:
-    en: Entity responsible for producing the dataset.
-    nl: Entiteit die verantwoordelijk is voor het produceren van de dataset.
-
 resource_fields:
 - field_name: issued
   label:

diff --git a/ckanext/fairdatapoint/tests/test_processors.py b/ckanext/fairdatapoint/tests/test_processors.py
@@ -53,17 +53,10 @@ def test_fdp_record_converter_dataset_dict(self):
                                 license_id="", identifier="27866022694497978",
                                 has_version=[
                                     "https://repo.metadatacenter.org/template-instances/2836bf1c-76e9-44e7-a65e-80e9ca63025a"],
-                                contact=[
-                                    {
-                                        "name": "N.K. De Vries"
-                                    }
-                                ], creator=[{"identifier": "https://orcid.org/0000-0002-0180-3636",
-                                             "name": "https://orcid.org/0000-0002-0180-3636"}],
-                                publisher=[
-                                    {
-                                        "uri": "https://opal.health-ri.nl/pub/"
-                                    }
-                                ], temporal_start='2020-01-01', temporal_end='2025-12-31')
+                                contact=[{'email': '', 'identifier': 'https://orcid.org/0000-0002-4348-707X', 'name': 'N.K. De Vries','uri': ''}
+                                ], creator=[{'email': '', 'identifier': '', 'name': '', 'type': '', 'uri': 'https://orcid.org/0000-0002-0180-3636', 'url': ''}],
+                                publisher=[{'email': '','identifier': '','name': '','type': '','uri': 'https://opal.health-ri.nl/pub', 'url': ''}],
+                                temporal_start='2020-01-01', temporal_end='2025-12-31')
         assert actual_dataset == expected_dataset
 
     def test_fdp_record_converter_catalog_dict(self):
@@ -85,11 +78,17 @@ def test_fdp_record_converter_catalog_dict(self):
             "language": ["http://id.loc.gov/vocabulary/iso639-1/en"],
             "license_id": "",
             "modified": '2023-10-06T10:12:55.614000+00:00',
-            "publisher": [
+            'publisher': [
                 {
-                    "name": "Automatic"
-                }
+                    'email': '',
+                    'identifier': '',
+                    "name": "Automatic",
+                    'type': '',
+                    'uri': '',
+                    'url': '',
+                },
             ],
+
             "resources": [],
             "tags": [],
             "title": "Slovenia National Node"