add WCMP2 migration script (#663) #3548
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test docker deployment ⚙️ | |
| on: [ push, pull_request ] | |
| jobs: | |
| main: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| name: setup Python | |
| with: | |
| python-version: 3.8 | |
| - name: install testing requirements 📦 | |
| working-directory: tests | |
| run: | | |
| pip3 install -r requirements.txt | |
| pip3 install check-jsonschema | |
| - name: cache schemas 📦 | |
| run: | | |
| pywis-pubsub schema sync | |
| curl -s https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json | |
| - name: display Docker and Python versions 📦 | |
| run: | | |
| docker version | |
| docker compose version | |
| python3 -V | |
| - name: setup wis2box configuration | |
| run: | | |
| cp tests/test.env wis2box.env | |
| cat wis2box.env | |
| python3 wis2box-ctl.py config | |
| - name: build wis2box | |
| run: | | |
| python3 wis2box-ctl.py build | |
| python3 wis2box-ctl.py update | |
| - name: start containers ⚙️ | |
| run: | | |
| python3 wis2box-ctl.py start | |
| python3 wis2box-ctl.py status -a | |
| docker logs wis2box-management | |
| - name: setup wis2box-management ⚙️ | |
| run: | | |
| sleep 20 | |
| python3 wis2box-ctl.py execute wis2box environment show | |
| - name: populate stations from CSV 📡 | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box metadata station publish-collection | |
| - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼 | |
| env: | |
| TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: MWI | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/malawi | |
| TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA_UPDATE | |
| - name: add Italy synop data (bufr2bufr) 🇮🇹 | |
| env: | |
| TOPIC_HIERARCHY: it-roma_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: ITA | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:it-roma_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/italy | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add Algeria synop data (bufr2bufr) 🇩🇿 | |
| env: | |
| TOPIC_HIERARCHY: dz-alger_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: DZA | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:dz-alger_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/algeria | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 | |
| env: | |
| TOPIC_HIERARCHY: ro-rnimh.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop | |
| TERRITORY: ROU | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations | |
| TEST_DATA: /data/wis2box/observations/romania | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add Congo synop data (synop2bufr) 🇨🇩 | |
| env: | |
| TOPIC_HIERARCHY: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop | |
| CHANNEL: origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop | |
| TERRITORY: COD | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations | |
| TEST_DATA: /data/wis2box/observations/congo | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add example ship data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.ship | |
| CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship | |
| TEST_DATA: /data/wis2box/observations/wmo/ship | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-3WL23XC $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-4EEWQMW $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-4SGNZUJ $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-EUCDE09 $CHANNEL | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-EUCDE34 $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add example buoy data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.buoy | |
| CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-buoy.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:buoy | |
| TEST_DATA: /data/wis2box/observations/wmo/buoy | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python3 wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-22000-0-1400011 $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add example wind profiler data (bufr2bufr) WMO | |
| env: | |
| TOPIC_HIERARCHY: int-wmo-test.data.core.weather.surface-based-observations.wind_profiler | |
| CHANNEL: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind_profiler.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind_profiler | |
| TEST_DATA: /data/wis2box/observations/wmo/wind_profiler | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| python wis2box-ctl.py execute wis2box metadata station add-topic --wsi 0-702-0-48698 $CHANNEL | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: add China GRIB2 data (universal pipeline) 🇨🇳 | |
| env: | |
| TOPIC_HIERARCHY: cn-cma.data.core.weather.prediction.forecast.medium-range.probabilistic.global | |
| DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml | |
| DISCOVERY_METADATA_ID: urn:wmo:md:cn-cma:grapes-geps-global | |
| TEST_DATA: /data/wis2box/observations/china | |
| run: | | |
| python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA | |
| curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID | |
| check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID | |
| python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA | |
| - name: sleep 30 seconds then run integration tests ⚙️ | |
| run: | | |
| sleep 30 | |
| pytest -s tests/integration | |
| - name: run flake8 ⚙️ | |
| run: | | |
| find . -type f -name "*.py" | xargs flake8 | |
| - name: failed tests 🚩 | |
| if: ${{ failure() }} | |
| run: | | |
| docker logs -n1000 wis2box-management | |
| docker logs -n1000 wis2box-api |