merge country and centre-id (#562)

wmo-im · Nov 22, 2023 · 1ac5c1f · 1ac5c1f
1 parent 6978f68
commit 1ac5c1f
Show file tree

Hide file tree

Showing 19 changed files with 74 additions and 85 deletions.
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
@@ -49,9 +49,9 @@ jobs:
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: add Malawi data 🇲🇼
       env:
-        TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mwi-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/malawi
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -61,9 +61,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Italy data 🇮🇹
       env:
-        TOPIC_HIERARCHY: ita.roma_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ita-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:ita:roma_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/italy
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -73,9 +73,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Algeria data 🇩🇿
       env:
-        TOPIC_HIERARCHY: dza.alger_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dza-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:dza:dza_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/algeria
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -85,9 +85,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Romania data 🇷🇴
       env:
-        TOPIC_HIERARCHY: rou.rnimh.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/rou-synoptic-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations
+        TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
         TEST_DATA: /data/wis2box/observations/romania
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA
@@ -97,9 +97,9 @@ jobs:
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
     - name: add Congo data 🇨🇩
       env:
-        TOPIC_HIERARCHY: cog.brazza_met_centre.data.core.weather.surface-based-observations.synop
-        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cog-surface-weather-observations.yml
-        DISCOVERY_METADATA_ID: urn:x-wmo:md:cog:brazza_met_centre:surface-weather-observations
+        TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml
+        DISCOVERY_METADATA_ID: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/congo
       run: |
         python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA

diff --git a/config-templates/data-mappings.yml.tmpl b/config-templates/data-mappings.yml.tmpl
@@ -1,5 +1,5 @@
 data:
-    $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.synop:
+    $CENTRE_ID.data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR
@@ -37,7 +37,7 @@ data:
                   buckets:
                     - $${WIS2BOX_STORAGE_PUBLIC}
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$$'
-    $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.temp:
+    $CENTRE_ID.data.core.weather.surface-based-observations.temp:
         plugins:
             b:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/config-templates/metadata-synop.yml.tmpl b/config-templates/metadata-synop.yml.tmpl
@@ -1,14 +1,13 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.synop
-    country: $COUNTRY_CODE
+    topic_hierarchy: $CENTRE_ID.data.core.weather.surface-based-observations.synop
     centre_id: $CENTRE_ID
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:$COUNTRY_CODE:$CENTRE_ID:surface-based-observations.synop
+    identifier: urn:x-wmo:md:$CENTRE_ID:surface-based-observations.synop
     language: en
     charset: utf8
     hierarchylevel: dataset

diff --git a/config-templates/metadata-temp.yml.tmpl b/config-templates/metadata-temp.yml.tmpl
@@ -1,14 +1,13 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.temp
-    country: $COUNTRY_CODE
+    topic_hierarchy: $CENTRE_ID.data.core.weather.surface-based-observations.temp
     centre_id: $CENTRE_ID
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:$COUNTRY_CODE:$CENTRE_ID:surface-based-observations.temp
+    identifier: urn:x-wmo:md:$CENTRE_ID:surface-based-observations.temp
     language: en
     charset: utf8
     hierarchylevel: dataset

diff --git a/docs/source/user/data-ingest.rst b/docs/source/user/data-ingest.rst
@@ -61,8 +61,8 @@ Select 'browse' on the ``wis2box-incoming`` bucket and select 'Choose or create
 
     For example:
 
-    * data to be published on:  ``origin/a/wis2/cog/brazza_met_centre/data/core/weather/surface-based-observations/synop``
-    * upload data in the path: ``cog/brazza_met_centre/data/core/weather/surface-based-observations/synop``. 
+    * data to be published on: ``origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop``
+    * upload data in the path: ``cd-brazza_met_centre/data/core/weather/surface-based-observations/synop``
 
     The error message ``Topic Hierarchy validation error: No plugins for minio:9000/wis2box-incoming/... in data mappings`` indicates you stored a file in a folder for which no matching dataset was defined in ``data-mappings.yml``.
 
@@ -160,7 +160,7 @@ Then start the ``wis2box-ftp`` service with the following command:
 
 When using the wis2box-ftp service to ingest data, please note that the topic is determined by the directory structure in which the data arrives.
 
-For example to correctly ingest data on the topic ``ita.roma_met_centre.data.core.weather.surface-based-observations.synop`` you need to copy the data into the directory ``/ita/roma_met_centre/data/core/weather/surface-based-observations/synop`` on the FTP server:
+For example, to correctly ingest data on the topic ``it-roma_met_centre.data.core.weather.surface-based-observations.synop`` you need to copy the data into the directory ``/ita/roma_met_centre/data/core/weather/surface-based-observations/synop`` on the FTP server:
 
 .. image:: ../_static/winscp_wis2box-ftp_example.png
     :width: 600
@@ -188,7 +188,7 @@ To start the ``wis2box-data-subscriber``, add the following additional variables
 
 These variables determine the destination path in the ``wis2box-incoming`` bucket:
 
-``{COUNTRY_ID}/{CENTRE_ID}/data/core/weather/surface-based-observations/synop/``
+``{CENTRE_ID}/data/core/weather/surface-based-observations/synop/``
 
 You then you can activate the optional 'wis2box-data-subscriber' service as follows:
 

diff --git a/examples/config/surface-weather-observations.yml b/examples/config/surface-weather-observations.yml
@@ -1,23 +1,23 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: country.centre_id.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: centre_id.data.core.weather.surface-based-observations.synop
     country: country
     centre_id: centre_id
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:country:centre_id:surface-weather-observations
+    identifier: urn:x-wmo:md:centre_id:surface-weather-observations
     language: en
     charset: utf8
     hierarchylevel: dataset
 
 identification:
     language: en
     charset: utf8
-    title: Surface weather observations from country.centre_id
-    abstract: Surface weather observations from country.centre_id
+    title: Surface weather observations from centre_id
+    abstract: Surface weather observations from centre_id
     dates:
         creation: 2021-11-29
         publication: 2021-11-29

diff --git a/examples/config/synop-bin-mappings.yml b/examples/config/synop-bin-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             bin:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/examples/config/synop-bufr-mappings.yml b/examples/config/synop-bufr-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             bufr:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/examples/config/synop-csv-mappings.yml b/examples/config/synop-csv-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR

diff --git a/examples/config/synop-synop-mappings.yml b/examples/config/synop-synop-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    [country].[centre_id].data.core.weather.surface-based-observations.synop:
+    [centre_id].data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR

diff --git a/tests/data/data-mappings.yml b/tests/data/data-mappings.yml
@@ -1,5 +1,5 @@
 data:
-    cog.brazza_met_centre.data.core.weather.surface-based-observations.synop:
+    cd-brazza_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR
@@ -10,7 +10,7 @@ data:
                   buckets:
                     - ${WIS2BOX_STORAGE_PUBLIC}
                   file-pattern: '^.*\.bufr4$' 
-    rou.rnimh.data.core.weather.surface-based-observations.synop:
+    ro-rnimh.data.core.weather.surface-based-observations.synop:
         plugins:
             txt:
                 - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR
@@ -24,7 +24,7 @@ data:
             bufr4:
                 - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
                   file-pattern: '^A_SMR.*EDZW_(\d{4})(\d{2}).*.bufr4$'
-    mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop:
+    mw-mw_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR
@@ -34,7 +34,7 @@ data:
             bufr4:
                 - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$'
-    ita.roma_met_centre.data.core.weather.surface-based-observations.synop:
+    it-roma_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             bin:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR
@@ -43,7 +43,7 @@ data:
             bufr4:
                 - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$'
-    dza.alger_met_centre.data.core.weather.surface-based-observations.synop:
+    dz-alger_met_centre.data.core.weather.surface-based-observations.synop:
         plugins:
             bufr4:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

diff --git a/...very/cog-surface-weather-observations.yml → ...overy/cd-surface-weather-observations.yml b/...very/cog-surface-weather-observations.yml → ...overy/cd-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P180D
-    topic_hierarchy: cog.brazza_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
     country: cog
-    centre_id: brazza_met_centre
+    centre_id: cd-brazza_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:cog:brazza_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:cd-brazza_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...very/dza-surface-weather-observations.yml → ...overy/dz-surface-weather-observations.yml b/...very/dza-surface-weather-observations.yml → ...overy/dz-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: dza.alger_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
     country: dza
-    centre_id: alger_met_centre
+    centre_id: dz-alger_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:dza:dza_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:dz-alger_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...very/ita-surface-weather-observations.yml → ...overy/it-surface-weather-observations.yml b/...very/ita-surface-weather-observations.yml → ...overy/it-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: ita.roma_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: it-roma_met_centre.data.core.weather.surface-based-observations.synop
     country: ita
-    centre_id: roma_met_centre
+    centre_id: it-roma_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:ita:roma_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:it-roma_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...very/mwi-surface-weather-observations.yml → ...overy/mw-surface-weather-observations.yml b/...very/mwi-surface-weather-observations.yml → ...overy/mw-surface-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
     country: mwi
-    centre_id: mwi_met_centre
+    centre_id: mw-mw_met_centre
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:mwi:mwi_met_centre:surface-weather-observations
+    identifier: urn:x-wmo:md:mw-mw_met_centre:surface-weather-observations
     hierarchylevel: dataset
 
 identification:

diff --git a/...ery/rou-synoptic-weather-observations.yml → ...very/ro-synoptic-weather-observations.yml b/...ery/rou-synoptic-weather-observations.yml → ...very/ro-synoptic-weather-observations.yml
@@ -1,14 +1,14 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: rou.rnimh.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: ro-rnimh.data.core.weather.surface-based-observations.synop
     country: rou
-    centre_id: rnimh
+    centre_id: ro-rnimh
 
 mcf:
     version: 1.0
 
 metadata:
-    identifier: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations
+    identifier: urn:x-wmo:md:ro-rnimh:synoptic-weather-observations
     hierarchylevel: dataset
 
 identification: