From 6d7c8c4b0cfd108e6933d38eb00fb200ce8e7ab5 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 21 Sep 2023 21:00:31 -0400 Subject: [PATCH] pygeometa and WCMP2 updates (#521) * update based on latest pygeometa/WCMP2 updates * force VCard style string for WCMP2 phone * fix * fix * fix * update CI * fix * fix * fix * fix * bump pyoscar * update WCMP2 templates --- .github/workflows/tests-docker.yml | 17 ++++++++++ config-templates/metadata-synop.yml.tmpl | 33 +++++++++--------- config-templates/metadata-temp.yml.tmpl | 34 ++++++++++--------- .../config/surface-weather-observations.yml | 21 ++++++------ .../cog-surface-weather-observations.yml | 7 ++-- .../dza-surface-weather-observations.yml | 22 ++++++------ .../ita-surface-weather-observations.yml | 21 ++++++------ .../mwi-surface-weather-observations.yml | 10 +++--- .../rou-synoptic-weather-observations.yml | 10 +++--- wis2box-management/Dockerfile | 4 +-- wis2box-management/requirements.txt | 2 +- .../wis2box/api/backend/elastic.py | 10 ++++++ .../wis2box/metadata/discovery.py | 11 ++++++ 13 files changed, 121 insertions(+), 81 deletions(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index a32e812dd..373ff270e 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -16,9 +16,11 @@ jobs: working-directory: tests run: | pip3 install -r requirements.txt + pip3 install check-jsonschema - name: cache schemas 📦 run: | pywis-pubsub schema sync + curl https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json - name: display Docker and Python versions 📦 run: | docker version @@ -46,47 +48,62 @@ jobs: env: TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mwi-surface-weather-observations.yml + DISCOVERY_METADATA_ID: urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations TEST_DATA: /data/wis2box/observations/malawi run: | python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA + curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID + check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID - name: add Italy data 🇮🇹 env: TOPIC_HIERARCHY: ita.roma_met_centre.data.core.weather.surface-based-observations.synop DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ita-surface-weather-observations.yml + DISCOVERY_METADATA_ID: urn:x-wmo:md:ita:roma_met_centre:surface-weather-observations TEST_DATA: /data/wis2box/observations/italy run: | python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA + curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID + check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID - name: add Algeria data 🇩🇿 env: TOPIC_HIERARCHY: dza.alger_met_centre.data.core.weather.surface-based-observations.synop DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dza-surface-weather-observations.yml + DISCOVERY_METADATA_ID: urn:x-wmo:md:dza:dza_met_centre:surface-weather-observations TEST_DATA: /data/wis2box/observations/algeria run: | python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA + curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID + check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID - name: add Romania data 🇷🇴 env: TOPIC_HIERARCHY: rou.rnimh.data.core.weather.surface-based-observations.synop DISCOVERY_METADATA: /data/wis2box/metadata/discovery/rou-synoptic-weather-observations.yml + DISCOVERY_METADATA_ID: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations TEST_DATA: /data/wis2box/observations/romania run: | python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA + curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID + check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID - name: add Congo data 🇨🇩 env: TOPIC_HIERARCHY: cog.brazza_met_centre.data.core.weather.surface-based-observations.synop DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cog-surface-weather-observations.yml + DISCOVERY_METADATA_ID: urn:x-wmo:md:cog:brazza_met_centre:surface-weather-observations TEST_DATA: /data/wis2box/observations/congo run: | python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA + curl http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID + check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID - name: sync stations 🔄 run: | sleep 15 diff --git a/config-templates/metadata-synop.yml.tmpl b/config-templates/metadata-synop.yml.tmpl index ce0d1a28f..3a8723b3b 100644 --- a/config-templates/metadata-synop.yml.tmpl +++ b/config-templates/metadata-synop.yml.tmpl @@ -16,23 +16,24 @@ metadata: identification: language: en charset: utf8 - title: Surface weather observations from $CENTRE_NAME - abstract: Surface weather observations from $CENTRE_NAME + title: Hourly synoptic observations from fixed-land stations (SYNOP) ($COUNTRY_NAME) + abstract: Hourly synoptic observations from fixed-land stations (SYNOP) ($COUNTRY_NAME) dates: creation: $CREATION_DATE publication: $PUBLICATION_DATE keywords: default: keywords: - - upper-air weather + - surface + - land - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: WMO WIS2 Topic Hierarchy + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [$BOUNDING_BOX] @@ -46,18 +47,18 @@ identification: contact: pointOfContact: &contact_poc organization: $CENTRE_NAME - url: NA - individualname: NA - positionname: NA - phone: NA - fax: NA - address: NA - city: NA - administrativearea: NA - postalcode: NA + url: null + individualname: null + positionname: null + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null country: $COUNTRY_NAME email: $WIS2BOX_EMAIL - hoursofservice: NA + hoursofservice: null contactinstructions: email distributor: *contact_poc diff --git a/config-templates/metadata-temp.yml.tmpl b/config-templates/metadata-temp.yml.tmpl index ab06448e6..de47a8209 100644 --- a/config-templates/metadata-temp.yml.tmpl +++ b/config-templates/metadata-temp.yml.tmpl @@ -16,23 +16,25 @@ metadata: identification: language: en charset: utf8 - title: Upper-air weather observations from $CENTRE_NAME - abstract: Upper-air weather observations from $CENTRE_NAME + title: Upper-level temperature/humidity/wind reports from fixed-land stations (TEMP) ($COUNTRY_NAME) + abstract: Upper-level temperature/humidity/wind reports from fixed-land stations (TEMP) ($COUNTRY_NAME) dates: creation: $CREATION_DATE publication: $PUBLICATION_DATE keywords: default: keywords: - - upper-air weather + - upper air + - humidity + - wind - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: WMO WIS2 Topic Hierarchy + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [$BOUNDING_BOX] @@ -46,18 +48,18 @@ identification: contact: pointOfContact: &contact_poc organization: $CENTRE_NAME - url: NA - individualname: NA - positionname: NA - phone: NA - fax: NA - address: NA - city: NA - administrativearea: NA - postalcode: NA + url: null + individualname: null + positionname: null + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null country: $COUNTRY_NAME email: $WIS2BOX_EMAIL - hoursofservice: NA + hoursofservice: null contactinstructions: email distributor: *contact_poc diff --git a/examples/config/surface-weather-observations.yml b/examples/config/surface-weather-observations.yml index c3b816292..3d440742c 100644 --- a/examples/config/surface-weather-observations.yml +++ b/examples/config/surface-weather-observations.yml @@ -29,12 +29,11 @@ identification: - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv - + name: WMO WIS2 Topic Hierarchy + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [-180, -90, 190, 90] @@ -52,13 +51,13 @@ contact: url: https://example.org individualname: Firstname Lastname positionname: Position Name - phone: NA - fax: NA - address: NA - city: NA - administrativearea: NA - postalcode: NA - country: NA + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null + country: null email: you@example.org hoursofservice: 0700h - 1500h UTC contactinstructions: email diff --git a/tests/data/metadata/discovery/cog-surface-weather-observations.yml b/tests/data/metadata/discovery/cog-surface-weather-observations.yml index 15a6440f3..c1acfd332 100644 --- a/tests/data/metadata/discovery/cog-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/cog-surface-weather-observations.yml @@ -24,11 +24,11 @@ identification: - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [11.0937728207,-5.03798674888,18.4530652198,3.72819651938] @@ -58,4 +58,3 @@ contact: contactinstructions: email distributor: *contact_poc - diff --git a/tests/data/metadata/discovery/dza-surface-weather-observations.yml b/tests/data/metadata/discovery/dza-surface-weather-observations.yml index 53050c312..fa9b2965a 100644 --- a/tests/data/metadata/discovery/dza-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/dza-surface-weather-observations.yml @@ -24,11 +24,11 @@ identification: - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [-8.68439978681, 19.0573642034, 11.9995056495, 37.1183806422] @@ -43,16 +43,16 @@ identification: contact: pointOfContact: &contact_poc organization: Department of Climate Change and Meteorologial Services (DCCMS) - url: NA + url: https://www.meteo.dz individualname: Firstname Lastname positionname: Position Name - phone: NA - fax: NA - address: NA - city: NA - administrativearea: NA - postalcode: NA - country: NA + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null + country: null email: you@example.org hoursofservice: 0700h - 1500h UTC contactinstructions: email diff --git a/tests/data/metadata/discovery/ita-surface-weather-observations.yml b/tests/data/metadata/discovery/ita-surface-weather-observations.yml index f91d1bfad..f8113adb5 100644 --- a/tests/data/metadata/discovery/ita-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/ita-surface-weather-observations.yml @@ -24,11 +24,11 @@ identification: - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [6.74995, 36.61998, 18.48024, 47.11539] @@ -39,6 +39,7 @@ identification: resolution: P1H rights: WMO Unified Policy for the International Exchange of Earth System Data url: https://example.org/malawi-surface-weather-observations + wmo_data_policy: core contact: pointOfContact: &contact_poc @@ -46,13 +47,13 @@ contact: url: http://www.meteoam.it/ individualname: Firstname Lastname positionname: Position Name - phone: NA - fax: NA - address: NA - city: NA - administrativearea: NA - postalcode: NA - country: NA + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null + country: null email: you@example.org hoursofservice: 0700h - 1500h UTC contactinstructions: email diff --git a/tests/data/metadata/discovery/mwi-surface-weather-observations.yml b/tests/data/metadata/discovery/mwi-surface-weather-observations.yml index 2ce570d83..c669548e5 100644 --- a/tests/data/metadata/discovery/mwi-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/mwi-surface-weather-observations.yml @@ -24,11 +24,11 @@ identification: - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [32.6881653175,-16.8012997372,35.7719047381,-9.23059905359] @@ -46,8 +46,8 @@ contact: url: https://www.metmalawi.gov.mw individualname: Firstname Lastname positionname: Position Name - phone: +265-1-822-014 - fax: +265-1-822-215 + phone: "+2651822014" + fax: "+2651822215" address: P.O. Box 1808 city: Blantyre administrativearea: Blantyre District diff --git a/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml b/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml index be556c5db..a79a30c8a 100644 --- a/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml +++ b/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml @@ -24,11 +24,11 @@ identification: - observations wmo: keywords: - - weatherObservations + - weather keywords_type: theme vocabulary: - name: WMO Category Code - url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://github.com/wmo-im/wis2-topic-hierarchy/blob/main/topic-hierarchy/earth-system-discipline.csv extents: spatial: - bbox: [20.2201924985,43.6884447292,29.62654341,48.2208812526] @@ -46,8 +46,8 @@ contact: url: https://www.meteoromania.ro individualname: Firstname Lastname positionname: Position Name - phone: +40 21 318 32 40 - fax: +40 21 316 31 43 + phone: "+40213183240" + fax: "+40213163143" address: Building A, Șoseaua București-Ploiești 9 city: Bucharest administrativearea: Municipality of Bucharest diff --git a/wis2box-management/Dockerfile b/wis2box-management/Dockerfile index df0c04391..bb3030cdf 100644 --- a/wis2box-management/Dockerfile +++ b/wis2box-management/Dockerfile @@ -42,9 +42,9 @@ RUN apt-get update -y && apt-get install -y ${DEBIAN_PACKAGES} \ https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.1.zip \ https://github.com/wmo-im/bufr2geojson/archive/refs/tags/v0.5.0.zip \ https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip \ - https://github.com/wmo-cop/pyoscar/archive/refs/tags/0.6.3.zip \ + https://github.com/wmo-cop/pyoscar/archive/refs/tags/0.6.4.zip \ https://github.com/wmo-im/synop2bufr/archive/refs/tags/v0.6.1.zip \ - https://github.com/geopython/pygeometa/archive/refs/tags/0.14.0.zip \ + https://github.com/geopython/pygeometa/archive/refs/tags/0.15.3.zip \ https://github.com/wmo-im/pywcmp/archive/refs/tags/0.4.0.zip \ # install shapely && pip3 install --no-cache-dir cython pygeos==0.13 \ diff --git a/wis2box-management/requirements.txt b/wis2box-management/requirements.txt index 411d762ea..1a757d324 100644 --- a/wis2box-management/requirements.txt +++ b/wis2box-management/requirements.txt @@ -4,7 +4,7 @@ isodate minio OWSLib paho-mqtt -pygeometa<0.15 +pygeometa pywis-pubsub PyYAML requests diff --git a/wis2box-management/wis2box/api/backend/elastic.py b/wis2box-management/wis2box/api/backend/elastic.py index 14d5e0515..9c672fd3e 100644 --- a/wis2box-management/wis2box/api/backend/elastic.py +++ b/wis2box-management/wis2box/api/backend/elastic.py @@ -41,6 +41,16 @@ 'geometry': { 'type': 'geo_shape' }, + 'time': { + 'properties': { + 'interval': { + 'type': 'date', + 'null_value': '1850', + 'format': 'year||year_month||year_month_day||date_time||t_time||t_time_no_millis', # noqa + 'ignore_malformed': True + } + } + }, 'reportId': { 'type': 'text', 'fields': { diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py index 899927452..d194a5d06 100644 --- a/wis2box-management/wis2box/metadata/discovery.py +++ b/wis2box-management/wis2box/metadata/discovery.py @@ -102,6 +102,17 @@ def generate(self, mcf: dict) -> str: LOGGER.debug('Generating OARec discovery metadata') record = WMOWCMP2OutputSchema().write(md, stringify=False) record['properties']['wmo:topicHierarchy'] = mqtt_topic + record['properties']['contacts'][0]['organization'] = record['properties']['contacts'][0].pop('name') # noqa + + try: + phone = record['properties']['contacts'][0]['phones'][0]['value'] + if isinstance(phone, int): + record['properties']['contacts'][0]['phones'][0]['value'] = f'+{phone}' # noqa + elif not phone.startswith('+'): + LOGGER.debug('Casting phone to string') + record['properties']['contacts'][0]['phones'][0]['value'] = f'+{phone}' # noqa + except KeyError: + LOGGER.debug('No phone number defined') return record