improve metadata detection and publication (#642)

* detect MCF or WCMP2 for metadata publishing

* move distribution link generation to a function

* test for empty links

* fix

* add generated by

* fix for data from dict

* metadata publish fixes

* add arg to detect how distribution links are generated (mcf or wcmp2)

* protection for missing data-mappings

* improved publish checks, update version

* add force and flush data capabilities

* fix ref

* attempt to test mqtt-connect, move refresh data-mappings

* fix/debug GH test

* wait longer

* reduce sleep, update test-count

* sleep longer

* test messages in order of CI

* reduce test cases temporarily

* revert

* fix unpublish

* cleanup and update testing

* fix unpublish workflow, broker publish URL

* prune romania

* reduce sleep, reduce romania count

* improved testing, set NotHandledError on STORAGE_PUBLIC

* fix channels, fix stations count

* sleep longer

* sleep even longer

* prune romania data some more

* update minio: log errors + doc-strings

* remove pub-conn-check, add refresh_data_mappings

* remove time from mqtt

* simplify wis2box-create-config, update docs

* updated docs, improve local SSL certificates support

* fix WCMP2 MQTT outbound link

---------

Co-authored-by: Tom Kralidis <[email protected]>

.github/workflows/tests-docker.yml

@@ -42,102 +42,126 @@ jobs:
         docker logs wis2box-management
     - name: setup wis2box-management ⚙️
       run: |
-        sleep 30
+        sleep 20
         python3 wis2box-ctl.py execute wis2box environment show
     - name: populate stations from CSV 📡
       run: |
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼
       env:
         TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
+        CHANNEL: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop
+        TERRITORY: MWI
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/malawi
         TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY  $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
         python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA_UPDATE
     - name: add Italy synop data (bufr2bufr) 🇮🇹
       env:
         TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop
+        CHANNEL: origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop
+        TERRITORY: ITA
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:it-roma_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/italy
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add Algeria synop data (bufr2bufr) 🇩🇿
       env:
         TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
+        CHANNEL: origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop
+        TERRITORY: DZA
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:dz-alger_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/algeria
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add Romania synop data  (synop2bufr and csv2bufr aws-template) 🇷🇴
       env:
         TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop
+        CHANNEL: origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop
+        TERRITORY: ROU
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations
         TEST_DATA: /data/wis2box/observations/romania
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add Congo synop data (synop2bufr) 🇨🇩
       env:
         TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
+        CHANNEL: origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop
+        TERRITORY: COD
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations
         TEST_DATA: /data/wis2box/observations/congo
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
-
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add example ship data (bufr2bufr) WMO
       env:
         TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.ship
+        CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship
         TEST_DATA: /data/wis2box/observations/wmo/ship
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-3WL23XC $CHANNEL
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-4EEWQMW $CHANNEL
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-4SGNZUJ $CHANNEL
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-EUCDE09 $CHANNEL
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-EUCDE34 $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
-        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID        
+        check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add example buoy data (bufr2bufr) WMO
       env:
         TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.buoy
+        CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-buoy.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:buoy
         TEST_DATA: /data/wis2box/observations/wmo/buoy
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-1400011 $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add example wind profiler data (bufr2bufr) WMO
       env:
         TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.wind_profiler
+        CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind_profiler.yml
         DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind_profiler
         TEST_DATA: /data/wis2box/observations/wmo/wind_profiler
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+        python wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-702-0-48698 $CHANNEL
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
     - name: add China GRIB2 data (universal pipeline) 🇨🇳
       env:
         TOPIC_HIERARCHY: cn-cma.data.core.weather.prediction.forecast.medium-range.probabilistic.global
@@ -146,14 +170,10 @@ jobs:
         TEST_DATA: /data/wis2box/observations/china
       run: |
         python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
-        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
         curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID
         check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID
-    - name: sync stations 🔄
-      run: |
-        sleep 30
-        python3 wis2box-ctl.py execute wis2box metadata station publish-collection
-    - name: run integration tests ⚙️
+        python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA
+    - name: sleep 30 seconds then run integration tests ⚙️
       run: |
         sleep 30
         pytest -s tests/integration