pygeometa and WCMP2 updates (#521)

* update based on latest pygeometa/WCMP2 updates * force VCard style string for WCMP2 phone * fix * fix * fix * update CI * fix * fix * fix * fix * bump pyoscar * update WCMP2 templates
wmo-im · Sep 22, 2023 · 6d7c8c4 · 6d7c8c4
1 parent 01e8b2d
commit 6d7c8c4
Show file tree

Hide file tree

Showing 13 changed files with 121 additions and 81 deletions.
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
@@ -16,9 +16,11 @@ jobs:
       working-directory: tests
       run: |
         pip3 install -r requirements.txt
+        pip3 install check-jsonschema
     - name: cache schemas 📦
       run: |
         pywis-pubsub schema sync
+        curl https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json
     - name: display Docker and Python versions 📦
       run: |
         docker version
@@ -46,47 +48,62 @@ jobs:
       env:
         TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mwi-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/malawi
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
+        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Italy data 🇮🇹
       env:
         TOPIC_HIERARCHY: ita.roma_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ita-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:ita:roma_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/italy
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
+        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Algeria data 🇩🇿
       env:
         TOPIC_HIERARCHY: dza.alger_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dza-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:dza:dza_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/algeria
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
+        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Romania data 🇷🇴
       env:
         TOPIC_HIERARCHY: rou.rnimh.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/rou-synoptic-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations
         TEST_DATA: /data/wis2box/observations/romania
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
+        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Congo data 🇨🇩
       env:
         TOPIC_HIERARCHY: cog.brazza_met_centre.data.core.weather.surface-based-observations.synop
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cog-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:cog:brazza_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/congo
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA
         python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
+        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: sync stations 🔄
       run: |
         sleep 15

diff --git a/config-templates/metadata-synop.yml.tmpl b/config-templates/metadata-synop.yml.tmpl
@@ -16,23 +16,24 @@ metadata:
 identification:
     language: en
     charset: utf8
-    title: Surface weather observations from $CENTRE_NAME
-    abstract: Surface weather observations from $CENTRE_NAME
+    title: Hourly synoptic observations from fixed-land stations (SYNOP) ($COUNTRY_NAME)
+    abstract: Hourly synoptic observations from fixed-land stations (SYNOP) ($COUNTRY_NAME)
     dates:
         creation: $CREATION_DATE
         publication: $PUBLICATION_DATE
     keywords:
         default:
             keywords:
-                - upper-air weather
+                - surface
+                - land
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: WMO WIS2 Topic Hierarchy
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [$BOUNDING_BOX]
@@ -46,18 +47,18 @@ identification:
 contact:
     pointOfContact: &contact_poc
         organization: $CENTRE_NAME
-        url: NA
-        individualname: NA
-        positionname: NA
-        phone: NA
-        fax: NA
-        address: NA
-        city: NA
-        administrativearea: NA
-        postalcode: NA
+        url: null
+        individualname: null
+        positionname: null
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
         country: $COUNTRY_NAME
         email: $WIS2BOX_EMAIL
-        hoursofservice: NA
+        hoursofservice: null
         contactinstructions: email
 
     distributor: *contact_poc
diff --git a/config-templates/metadata-temp.yml.tmpl b/config-templates/metadata-temp.yml.tmpl
@@ -16,23 +16,25 @@ metadata:
 identification:
     language: en
     charset: utf8
-    title: Upper-air weather observations from $CENTRE_NAME
-    abstract: Upper-air weather observations from $CENTRE_NAME
+    title: Upper-level temperature/humidity/wind reports from fixed-land stations (TEMP) ($COUNTRY_NAME)
+    abstract: Upper-level temperature/humidity/wind reports from fixed-land stations (TEMP) ($COUNTRY_NAME)
     dates:
         creation: $CREATION_DATE
         publication: $PUBLICATION_DATE
     keywords:
         default:
             keywords:
-                - upper-air weather
+                - upper air
+                - humidity
+                - wind
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: WMO WIS2 Topic Hierarchy
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [$BOUNDING_BOX]
@@ -46,18 +48,18 @@ identification:
 contact:
     pointOfContact: &contact_poc
         organization: $CENTRE_NAME
-        url: NA
-        individualname: NA
-        positionname: NA
-        phone: NA
-        fax: NA
-        address: NA
-        city: NA
-        administrativearea: NA
-        postalcode: NA
+        url: null
+        individualname: null
+        positionname: null
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
         country: $COUNTRY_NAME
         email: $WIS2BOX_EMAIL
-        hoursofservice: NA
+        hoursofservice: null
         contactinstructions: email
 
     distributor: *contact_poc
diff --git a/examples/config/surface-weather-observations.yml b/examples/config/surface-weather-observations.yml
@@ -29,12 +29,11 @@ identification:
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
-
+                name: WMO WIS2 Topic Hierarchy
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [-180, -90, 190, 90]
@@ -52,13 +51,13 @@ contact:
         url: https://example.org
         individualname: Firstname Lastname
         positionname: Position Name
-        phone: NA
-        fax: NA
-        address: NA
-        city: NA
-        administrativearea: NA
-        postalcode: NA
-        country: NA
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
+        country: null
         email: [email protected]
         hoursofservice: 0700h - 1500h UTC
         contactinstructions: email

diff --git a/tests/data/metadata/discovery/cog-surface-weather-observations.yml b/tests/data/metadata/discovery/cog-surface-weather-observations.yml
@@ -24,11 +24,11 @@ identification:
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [11.0937728207,-5.03798674888,18.4530652198,3.72819651938]
@@ -58,4 +58,3 @@ contact:
         contactinstructions: email
 
     distributor: *contact_poc
-
diff --git a/tests/data/metadata/discovery/dza-surface-weather-observations.yml b/tests/data/metadata/discovery/dza-surface-weather-observations.yml
@@ -24,11 +24,11 @@ identification:
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [-8.68439978681, 19.0573642034, 11.9995056495, 37.1183806422]
@@ -43,16 +43,16 @@ identification:
 contact:
     pointOfContact: &contact_poc
         organization: Department of Climate Change and Meteorologial Services (DCCMS)
-        url: NA
+        url: https://www.meteo.dz
         individualname: Firstname Lastname
         positionname: Position Name
-        phone: NA
-        fax: NA
-        address: NA
-        city: NA
-        administrativearea: NA
-        postalcode: NA
-        country: NA
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
+        country: null
         email: [email protected]
         hoursofservice: 0700h - 1500h UTC
         contactinstructions: email

diff --git a/tests/data/metadata/discovery/ita-surface-weather-observations.yml b/tests/data/metadata/discovery/ita-surface-weather-observations.yml
@@ -24,11 +24,11 @@ identification:
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [6.74995, 36.61998, 18.48024, 47.11539]
@@ -39,20 +39,21 @@ identification:
               resolution: P1H
     rights: WMO Unified Policy for the International Exchange of Earth System Data
     url: https://example.org/malawi-surface-weather-observations
+    wmo_data_policy: core
 
 contact:
     pointOfContact: &contact_poc
         organization: Department of Climate Change and Meteorologial Services (DCCMS)
         url: http://www.meteoam.it/
         individualname: Firstname Lastname
         positionname: Position Name
-        phone: NA
-        fax: NA
-        address: NA
-        city: NA
-        administrativearea: NA
-        postalcode: NA
-        country: NA
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
+        country: null
         email: [email protected]
         hoursofservice: 0700h - 1500h UTC
         contactinstructions: email

diff --git a/tests/data/metadata/discovery/mwi-surface-weather-observations.yml b/tests/data/metadata/discovery/mwi-surface-weather-observations.yml
@@ -24,11 +24,11 @@ identification:
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [32.6881653175,-16.8012997372,35.7719047381,-9.23059905359]
@@ -46,8 +46,8 @@ contact:
         url: https://www.metmalawi.gov.mw
         individualname: Firstname Lastname
         positionname: Position Name
-        phone: +265-1-822-014
-        fax: +265-1-822-215
+        phone: "+2651822014"
+        fax: "+2651822215"
         address: P.O. Box 1808
         city: Blantyre
         administrativearea: Blantyre District

diff --git a/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml b/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml
@@ -24,11 +24,11 @@ identification:
                 - observations
         wmo:
             keywords:
-                - weatherObservations
+                - weather
             keywords_type: theme
             vocabulary:
-                name: WMO Category Code
-                url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv
     extents:
         spatial:
             - bbox: [20.2201924985,43.6884447292,29.62654341,48.2208812526]
@@ -46,8 +46,8 @@ contact:
         url: https://www.meteoromania.ro
         individualname: Firstname Lastname
         positionname: Position Name
-        phone:  +40 21 318 32 40
-        fax: +40 21 316 31 43
+        phone: "+40213183240"
+        fax: "+40213163143"
         address: Building A, Șoseaua București-Ploiești 9
         city: Bucharest
         administrativearea: Municipality of Bucharest