diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 57817061f..b578a3850 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -30,7 +30,6 @@ jobs: - wis2box-broker - wis2box-management - wis2box-mqtt-metrics-collector - - wis2downloader # - wis2box-ui diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 8b0e1d494..e01cb4d51 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -16,16 +16,26 @@ jobs: working-directory: tests run: | pip3 install -r requirements.txt - pip3 install check-jsonschema - name: cache schemas 📦 run: | pywis-pubsub schema sync - curl -s https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json --output /tmp/wcmp2-bundled.json + pywcmp bundle sync - name: display Docker and Python versions 📦 run: | docker version docker compose version python3 -V + - name: setup wis2downloader + run: | + # make sure current user in docker group + sudo usermod -aG docker $(whoami) + # Add env parameters to tests/test.env + echo "" >> tests/test.env #no new line at end of test.env file + echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> tests/test.env + # make sure data dir is in group docker and has 775 permissions + mkdir -p tests/data/downloads + chown -R $(whoami):docker tests/data/downloads + chmod -R 775 tests/data/downloads - name: setup wis2box configuration, replace localhost with IP on host 📦 run: | export IP=$(hostname -I | awk '{print $1}') @@ -55,115 +65,115 @@ jobs: curl -X POST http://localhost/wis2downloader/subscriptions -H "Content-Type: application/json" -H "Authorization: Bearer github123" -d @test.json - name: add Malawi synop data (csv2bufr synop_bufr template) 🇲🇼 env: - TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop TERRITORY: MWI DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations STATION_METADATA: /data/wis2box/metadata/station/malawi.csv TEST_DATA: /data/wis2box/observations/malawi TEST_DATA_UPDATE: /data/wis2box/observations/malawi_update run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA_UPDATE - name: add Italy synop data (bufr2bufr) 🇮🇹 env: - TOPIC_HIERARCHY: origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop TERRITORY: ITA DISCOVERY_METADATA: /data/wis2box/metadata/discovery/it-surface-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:it-roma_met_centre:surface-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:it-meteoam:surface-weather-observations STATION_METADATA: /data/wis2box/metadata/station/italy.csv TEST_DATA: /data/wis2box/observations/italy run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add Algeria synop data (bufr2bufr) 🇩🇿 env: - TOPIC_HIERARCHY: origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop TERRITORY: DZA DISCOVERY_METADATA: /data/wis2box/metadata/discovery/dz-surface-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:dz-alger_met_centre:surface-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:dz-meteoalgerie:surface-weather-observations STATION_METADATA: /data/wis2box/metadata/station/algeria.csv TEST_DATA: /data/wis2box/observations/algeria run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 + - name: add Romania synop data (synop2bufr and csv2bufr aws-template) 🇷🇴 env: - TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop TERRITORY: ROU DISCOVERY_METADATA: /data/wis2box/metadata/discovery/ro-synoptic-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh:synoptic-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations STATION_METADATA: /data/wis2box/metadata/station/romania.csv TEST_DATA: /data/wis2box/observations/romania run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add Congo synop data (synop2bufr) 🇨🇩 env: - TOPIC_HIERARCHY: origin/a/wis2/cd-brazza_met_centre/data/recommended/weather/surface-based-observations/synop + TOPIC_HIERARCHY: origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop TERRITORY: COD DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cd-surface-weather-observations.yml - DISCOVERY_METADATA_ID: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations + DISCOVERY_METADATA_ID: urn:wmo:md:cg-met:surface-weather-observations STATION_METADATA: /data/wis2box/metadata/station/congo.csv TEST_DATA: /data/wis2box/observations/congo run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + #pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID # uncomment once wis2box improves support for recommended data python3 wis2box-ctl.py execute wis2box auth add-token --metadata-id $DISCOVERY_METADATA_ID -p token123 -y python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add example ship data (bufr2bufr) WMO + - name: add example hourly ship data (bufr2bufr) WMO env: - TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship.yml - DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship - STATION_METADATA: /data/wis2box/metadata/station/wmo-test-ship.csv - TEST_DATA: /data/wis2box/observations/wmo/ship + TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-ship-hourly.yml + DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly + STATION_METADATA: /data/wis2box/metadata/station/wmo-test-ship-hourly.csv + TEST_DATA: /data/wis2box/observations/wmo/ship-hourly run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add example buoy data (bufr2bufr) WMO + - name: add example drifting buoy data (bufr2bufr) WMO env: - TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-buoy.yml - DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:buoy - STATION_METADATA: /data/wis2box/metadata/station/wmo-test-buoy.csv - TEST_DATA: /data/wis2box/observations/wmo/buoy + TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-drifting-buoys.yml + DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:drifting-buoys + STATION_METADATA: /data/wis2box/metadata/station/wmo-test-drifting-buoys.csv + TEST_DATA: /data/wis2box/observations/wmo/drifting-buoys run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add example wind profiler data (bufr2bufr) WMO + - name: add example wind profile data (bufr2bufr) WMO env: - TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind_profiler.yml - DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind_profiler - STATION_METADATA: /data/wis2box/metadata/station/wmo-test-wind_profiler.csv - TEST_DATA: /data/wis2box/observations/wmo/wind_profiler + TOPIC_HIERARCHY: origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-wmo-test-wind-profile.yml + DISCOVERY_METADATA_ID: urn:wmo:md:int-wmo-test:surface-weather-observations:wind-profile + STATION_METADATA: /data/wis2box/metadata/station/wmo-test-wind-profile.csv + TEST_DATA: /data/wis2box/observations/wmo/wind-profile run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add China GRIB2 data (universal pipeline) 🇨🇳 env: @@ -172,8 +182,8 @@ jobs: TEST_DATA: /data/wis2box/observations/china run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA - curl -s http://localhost/oapi/collections/discovery-metadata/items/$DISCOVERY_METADATA_ID --output /tmp/$DISCOVERY_METADATA_ID - check-jsonschema --schemafile /tmp/wcmp2-bundled.json /tmp/$DISCOVERY_METADATA_ID + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: sleep 30 seconds then run integration tests ⚙️ run: | diff --git a/docker-compose.yml b/docker-compose.yml index e24586ac1..cb13799ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,12 +146,14 @@ services: wis2downloader: container_name: wis2downloader + image: ghcr.io/wmo-im/wis2downloader:latest restart: always - build: ./wis2downloader env_file: - wis2box.env + user: 12135:${DOCKER_GID} volumes: - - ${WIS2BOX_HOST_DATADIR}/downloads:/app/downloads:rw + - ${WIS2BOX_HOST_DATADIR}/downloads:/home/wis2downloader/app/data/downloads + volumes: es-data: diff --git a/docs/source/_static/aws-example.csv b/docs/source/_static/aws-example.csv new file mode 100644 index 000000000..d2100ea43 --- /dev/null +++ b/docs/source/_static/aws-example.csv @@ -0,0 +1,4 @@ +wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,station_type,year,month,day,hour,minute,latitude,longitude,station_height_above_msl,barometer_height_above_msl,station_pressure,msl_pressure,geopotential_height,thermometer_height,air_temperature,dewpoint_temperature,relative_humidity,method_of_ground_state_measurement,ground_state,method_of_snow_depth_measurement,snow_depth,precipitation_intensity,anemometer_height,time_period_of_wind,wind_direction,wind_speed,maximum_wind_gust_direction_10_minutes,maximum_wind_gust_speed_10_minutes,maximum_wind_gust_direction_1_hour,maximum_wind_gust_speed_1_hour,maximum_wind_gust_direction_3_hours,maximum_wind_gust_speed_3_hours,rain_sensor_height,total_precipitation_1_hour,total_precipitation_3_hours,total_precipitation_6_hours,total_precipitation_12_hours,total_precipitation_24_hours +0,20000,0,60351,15,15,1,2022,3,31,1,0,47.77706163,23.94046026,503,,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60355,15,15,1,2022,3,31,1,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 +0,20000,0,60360,15,15,1,2022,3,31,1,0,47.77706163,23.94046026,503,504.43,100940,101040,1448,5,298.15,294.55,80,3,1,1,0,0.004,10,-10,30,3,30,5,40,9,20,11,2,4.7,5.3,7.9,9.5,11.4 \ No newline at end of file diff --git a/docs/source/reference/auth.rst b/docs/source/reference/auth.rst index 8d9d920cf..c870fc23a 100644 --- a/docs/source/reference/auth.rst +++ b/docs/source/reference/auth.rst @@ -44,7 +44,7 @@ first time a token is generated for a dataset .. code-block:: bash - wis2box auth add-token --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations mytoken + wis2box auth add-token --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations mytoken If no token is provided, a random string will be generated. Be sure to the record token now, there is no @@ -58,8 +58,8 @@ Token credentials can be validated using the wis2box command line utility. .. code-block:: bash wis2box auth show - wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations mytoken - wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre:surface-weather-observations notmytoken + wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations mytoken + wis2box auth has-access-topic --metadata-id urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations notmytoken Once a token has been generated, access to any data of that topic in the WAF or API requires token authentication. @@ -68,8 +68,8 @@ easily added to requests using `cURL`_. .. code-block:: bash - curl -H "Authorization: Bearer mytoken" "http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop" - curl -H "Authorization: Bearer notmytoken" "http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop" + curl -H "Authorization: Bearer mytoken" "http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations" + curl -H "Authorization: Bearer notmytoken" "http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations" Removing Access Control @@ -80,7 +80,7 @@ deleting individual tokens, or all tokens for a given topic hierarchy. .. code-block:: bash - wis2box auth remove-tokens --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop + wis2box auth remove-tokens --metadata-id "urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations" wis2box auth show diff --git a/docs/source/reference/data-access/python-api-owslib.ipynb b/docs/source/reference/data-access/python-api-owslib.ipynb index 02c3cf5f7..cdb78d7c5 100644 --- a/docs/source/reference/data-access/python-api-owslib.ipynb +++ b/docs/source/reference/data-access/python-api-owslib.ipynb @@ -69,11 +69,11 @@ "for dataset in collections['collections']:\n", " print(dataset['title'])\n", "\n", - "malawi_obs = oafeat.collection_items('mw-mw_met_centre.data.core.weather.surface-based-observations.synop')\n", + "malawi_obs = oafeat.collection_items('urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations')\n", "malawi_obs_df = pd.DataFrame(malawi_obs['features'])\n", "\n", "# then filter by station\n", - "obs = oafeat.collection_items('mw-mw_met_centre.data.core.weather.surface-based-observations.synop', wigos_station_identifier='0-454-2-AWSCHIDOOLE', name='air_temperature', limit=10000)\n", + "obs = oafeat.collection_items('urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations', wigos_station_identifier='0-454-2-AWSCHIDOOLE', name='air_temperature', limit=10000)\n", "\n", "datestamp = [obs['properties']['resultTime'] for obs in obs['features']]\n", "air_temperature = [obs['properties']['value'] for obs in obs['features']]\n", diff --git a/docs/source/reference/data-access/python-api-requests.ipynb b/docs/source/reference/data-access/python-api-requests.ipynb index 625372ad5..71765c0e4 100644 --- a/docs/source/reference/data-access/python-api-requests.ipynb +++ b/docs/source/reference/data-access/python-api-requests.ipynb @@ -119,7 +119,7 @@ "Datasets:\n", "\n", "id: data.core.test-passthrough, title: Surface weather observations (passthrough)\n", - "id: mw-mw_met_centre.data.core.weather.surface-based-observations.synop, title: Surface weather observations (hourly)\n" + "id: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations, title: Surface weather observations (hourly)\n" ] } ], @@ -151,18 +151,18 @@ "text": [ "Data access links:\n", "\n", - "{'rel': 'self', 'type': 'application/geo+json', 'title': 'This document as GeoJSON', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json (application/geo+json) self\n", - "{'rel': 'alternate', 'type': 'application/ld+json', 'title': 'This document as RDF (JSON-LD)', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld (application/ld+json) alternate\n", - "{'rel': 'alternate', 'type': 'text/html', 'title': 'This document as HTML', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html'} http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html (text/html) alternate\n", + "{'rel': 'self', 'type': 'application/geo+json', 'title': 'This document as GeoJSON', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=json'} http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=json (application/geo+json) self\n", + "{'rel': 'alternate', 'type': 'application/ld+json', 'title': 'This document as RDF (JSON-LD)', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=jsonld'} http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=jsonld (application/ld+json) alternate\n", + "{'rel': 'alternate', 'type': 'text/html', 'title': 'This document as HTML', 'href': 'http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=html'} http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=html (text/html) alternate\n", "{'rel': 'collection', 'type': 'application/json', 'title': 'Discovery metadata', 'href': 'http://localhost/oapi/collections/discovery-metadata'} http://localhost/oapi/collections/discovery-metadata (application/json) collection\n" ] }, { "data": { "text/plain": [ - "['http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=json',\n", - " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=jsonld',\n", - " 'http://localhost/oapi/collections/discovery-metadata/items/mw-mw_met_centre.data.core.weather.surface-based-observations.synop?f=html',\n", + "['http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=json',\n", + " 'http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=jsonld',\n", + " 'http://localhost/oapi/collections/discovery-metadata/items/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations?f=html',\n", " 'http://localhost/oapi/collections/discovery-metadata']" ] }, @@ -172,7 +172,7 @@ } ], "source": [ - "dataset_id = 'mw-mw_met_centre.data.core.weather.surface-based-observations.synop'\n", + "dataset_id = 'urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations'\n", "\n", "url = f\"{api}/collections/discovery-metadata/items/{dataset_id}\"\n", "\n", @@ -201,7 +201,7 @@ { "data": { "text/plain": [ - "'http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop'" + "'http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations'" ] }, "execution_count": 5, @@ -210,7 +210,7 @@ } ], "source": [ - "dataset_api_link = 'http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop'\n", + "dataset_api_link = 'http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations'\n", "\n", "dataset_api_link" ] diff --git a/docs/source/reference/data-access/r-api.ipynb b/docs/source/reference/data-access/r-api.ipynb index d82c219da..bcb3e205d 100644 --- a/docs/source/reference/data-access/r-api.ipynb +++ b/docs/source/reference/data-access/r-api.ipynb @@ -203,7 +203,7 @@ } ], "source": [ - "malawi_obs <- read_sf(paste0(oapi,\"/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items\"))\n", + "malawi_obs <- read_sf(paste0(oapi,\"/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/items\"))\n", "print(malawi_obs)" ] }, diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index 996b0532b..f3f5fe5b3 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -54,35 +54,36 @@ Publish test datasets: wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/dz-surface-weather-observations.yml wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/ro-synoptic-weather-observations.yml wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/cd-surface-weather-observations.yml - wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-ship.yml - wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-buoy.yml - wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-wind_profiler.yml + wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-ship-hourly.yml + wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-drifting-buoys.yml + wis2box dataset publish $WIS2BOX_DATADIR/metadata/discovery/int-wmo-test-wind-profile.yml Load initial stations: .. code-block:: bash - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/malawi.csv --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/italy.csv --topic-hierarchy it-roma_met_centre.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/algeria.csv --topic-hierarchy dz-alger_met_centre.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/romania.csv --topic-hierarchy ro-rnimh.data.core.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/congo.csv --topic-hierarchy cd-brazza_met_centre.data.recommended.weather.surface-based-observations.synop - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-ship.csv --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.ship - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-buoy.csv --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.buoy - wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-wind-profiler.csv --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.wind_profiler + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/malawi.csv --topic-hierarchy origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/italy.csv --topic-hierarchy origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/algeria.csv --topic-hierarchy origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/romania.csv --topic-hierarchy origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/congo.csv --topic-hierarchy origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-ship-hourly.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-drifting-buoys.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys + wis2box metadata station publish-collection --path /data/wis2box/metadata/station/wmo-test-wind-profile.csv --topic-hierarchy origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile Ingest data using the data ingest command to push data to the ``wis2box-incoming`` bucket: .. code-block:: bash - wis2box data ingest --topic-hierarchy mw-mw_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/malawi - wis2box data ingest --topic-hierarchy it-roma_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/italy - wis2box data ingest --topic-hierarchy dz-alger_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/algeria - wis2box data ingest --topic-hierarchy ro-rnimh.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/romania - wis2box data ingest --topic-hierarchy cd-brazza_met_centre.data.recommended.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/congo - wis2box data ingest --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.ship --path $WIS2BOX_DATADIR/observations/wmo/ship - wis2box data ingest --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.buoy --path $WIS2BOX_DATADIR/observations/wmo/buoy - wis2box data ingest --topic-hierarchy int-wmo-test.data.core.weather.surface-based-observations.wind_profiler --path $WIS2BOX_DATADIR/observations/wmo/wind_profiler + wis2box data ingest --metadata-id "urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/malawi + wis2box data ingest --metadata-id "urn:wmo:md:it-meteoam:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/italy + wis2box data ingest --metadata-id "urn:wmo:md:dz-meteoalgerie:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/algeria + wis2box data ingest --metadata-id "urn:wmo:md:ro-rnimh-test:synoptic-weather-observations" --path $WIS2BOX_DATADIR/observations/romania + wis2box data ingest --metadata-id "urn:wmo:md:cg-met:surface-weather-observations" --path $WIS2BOX_DATADIR/observations/congo + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly" --path $WIS2BOX_DATADIR/observations/wmo/ship-hourly + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:drifting-buoys" --path $WIS2BOX_DATADIR/observations/wmo/drifting-buoys + wis2box data ingest --metadata-id "urn:wmo:md:int-wmo-test:surface-weather-observations:wind-profile" --path $WIS2BOX_DATADIR/observations/wmo/wind-profile + Logout of wis2box-management container: diff --git a/docs/source/reference/services.rst b/docs/source/reference/services.rst index 96bd99dfd..ea01099c1 100644 --- a/docs/source/reference/services.rst +++ b/docs/source/reference/services.rst @@ -45,21 +45,21 @@ Below are some examples of working with the discovery catalogue. .. note:: - - the examples below use the ``mw-mw_met_centre.data.core.weather.surface-based-observations.synop`` collection as described + - the examples below use the ``urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations`` collection as described in the :ref:`quickstart`. For other dataset collections, use the same query patterns below, substituting the collection id accordingly - list of dataset collections: http://localhost/oapi/collections -- collection description: http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop -- collection queryables: http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/queryables -- collection items (browse): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items +- collection description: http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations +- collection queryables: http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/queryables +- collection items (browse): http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/items - collection queries - - set limit/offset (paging): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?limit=1&startindex=2 - - query by spatial (bounding box): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?bbox=32,-17,36,-8 - - query by temporal extent (since): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?datetime=2021/.. - - query by temporal extent (before): http://localhost/oapi/collections/mw-mw_met_centre.data.core.weather.surface-based-observations.synop/items?datetime=../2022 + - set limit/offset (paging): http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/items?limit=1&startindex=2 + - query by spatial (bounding box): http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/items?bbox=32,-17,36,-8 + - query by temporal extent (since): http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/items?datetime=2021/.. + - query by temporal extent (before): http://localhost/oapi/collections/urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations/items?datetime=../2022 .. note:: diff --git a/docs/source/reference/storage.rst b/docs/source/reference/storage.rst index 7de3be963..32af6cb17 100644 --- a/docs/source/reference/storage.rst +++ b/docs/source/reference/storage.rst @@ -67,7 +67,7 @@ The below example copies a local file (``myfile.csv``) to the ``wis2box-incoming To allow uploading files into MinIO remotely, the ``wis2box-incoming`` bucket is proxied via Nginx. For example, to upload the local file (``WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv with topic``) to the folder -``mw-mw_met_centre/data/core/weather/surface-based-observations/synop``: +``mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop``: .. code-block:: python @@ -83,7 +83,7 @@ For example, to upload the local file (``WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0 s3client = session.client('s3', endpoint_url=endpoint_url) - folder = 'mw-mw_met_centre/data/core/weather/surface-based-observations/synop' + folder = 'mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop' with open(filename, 'rb') as fh: s3client.upload_fileobj(fh, 'wis2box-incoming', f'{folder}/{filename}') @@ -106,7 +106,7 @@ The below example copies a local file (``mydata.bin``) to the ``wis2box-incoming from minio import Minio filepath = '/home/wis2box-user/local-data/mydata.bin' - minio_path = '/it-roma_met_centre/data/core/weather/surface-based-observations/synop/' + minio_path = '/it-meteoam/data/core/weather/surface-based-observations/synop/' endpoint = 'http://localhost:9000' WIS2BOX_STORAGE_USERNAME = 'wis2box' diff --git a/docs/source/user/data-ingest.rst b/docs/source/user/data-ingest.rst index 5fb44e34a..40ae95445 100644 --- a/docs/source/user/data-ingest.rst +++ b/docs/source/user/data-ingest.rst @@ -15,8 +15,8 @@ wis2box-webapp The wis2box-webapp is a web application that includes the following forms for data validation and ingestion: -* user interface to ingest SYNOP data -* user interface to ingest CSV data +* user interface to ingest `FM-12 SYNOP data `_ +* user interface to ingest CSV data using the :ref:`AWS template` The wis2box-webapp is available on your host at `http:///wis2box-webapp`. @@ -39,18 +39,20 @@ The plugins you have configured for your dataset mappings will determine the act The wis2box provides 3 types of built-in plugins to publish data in BUFR format: * `bufr2bufr` : the input is received in BUFR format and split by subset, where each subset is published as a separate bufr message -* `synop2bufr` : the input is received in FM-12 SYNOP format and converted to BUFR format. The year and month are extracted from the file pattern +* `synop2bufr` : the input is received in `FM-12 SYNOP format `_ and converted to BUFR format. The year and month are extracted from the file pattern * `csv2bufr` : the input is received in csv format and converted to BUFR format To publish data for other data formats you can use the 'Universal' plugin, which will pass through the data without any conversion. Please note that you will need to ensure that the date timestamp can be extracted from the file pattern when using this plugin. +.. _aws-template: + The AWS template in csv2bufr plugin ----------------------------------- When using the csv2bufr plugin, the columns are mapped to BUFR encoded values using a template as defined in the repository `csv2bufr-templates`_. -An example of a CSV file that can be ingested using the 'AWS' mappings template can be downloaded here :download:`AWS-example <../_static/aws-minimal.csv>` +An example of a CSV file that can be ingested using the 'AWS' mappings template can be downloaded here :download:`AWS-example <../_static/aws-example.csv>` The CSV columns description of the AWS template can be downloaded here :download:`AWS-reference <../_static/aws-minimal.csv>` @@ -85,13 +87,13 @@ Select 'browse' on the ``wis2box-incoming`` bucket and select 'Choose or create For example, using a filepath matching the metadata identifier: - * Metadata identifier: ``urn:wmo:md:it-roma_met_centre:surface-weather-observations.synop`` - * upload data in path containing: ``it-roma_met_centre:surface-weather-observations.synop`` + * Metadata identifier: ``urn:wmo:md:it-meteoam:surface-weather-observations.synop`` + * upload data in path containing: ``it-meteoam:surface-weather-observations.synop`` For example using a filepath matching the topic hierarchy: - * Topic Hierarchy: ``origin/a/wis2/cd-brazza_met_centre/data/recommended/weather/surface-based-observations/synop`` - * upload data in the path containing: ``cd-brazza_met_centre/data/recommended/weather/surface-based-observations/synop`` + * Topic Hierarchy: ``origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop`` + * upload data in the path containing: ``cg-met/data/recommended/weather/surface-based-observations/synop`` The error message ``Path validation error: Could not match http://minio:9000/wis2box-incoming/... to dataset, ...`` indicates that a file was stored in a directory that could not be matched to a dataset. @@ -138,7 +140,7 @@ See below a Python example to upload data using the MinIO package: filepath = '/home/wis2box-user/local-data/mydata.bin' # path should match the metadata or the topic in the data mappings - minio_path = 'urn:wmo:md:it-roma_met_centre:surface-weather-observations' + minio_path = 'urn:wmo:md:it-meteoam:surface-weather-observations' endpoint = 'http://localhost:9000' WIS2BOX_STORAGE_USERNAME = 'wis2box' @@ -200,7 +202,7 @@ Then start the ``wis2box-ftp`` service with the following command: When using the wis2box-ftp service to ingest data, please note that the topic is determined by the directory structure in which the data arrives. -For example, to correctly ingest data on the topic ``it-roma_met_centre.data.core.weather.surface-based-observations.synop`` you need to copy the data into the directory ``/it-roma_met_centre/data/core/weather/surface-based-observations/synop`` on the FTP server: +For example, to correctly ingest data on the topic ``it-meteoam/data/core/weather/surface-based-observations/synop`` you need to copy the data into the directory ``/it-meteoam/data/core/weather/surface-based-observations/synop`` on the FTP server: .. image:: ../_static/winscp_wis2box-ftp_example.png :width: 600 diff --git a/docs/source/user/setup.rst b/docs/source/user/setup.rst index f77bdcc16..57c7b6b0e 100644 --- a/docs/source/user/setup.rst +++ b/docs/source/user/setup.rst @@ -268,7 +268,7 @@ You can also bulk insert a set of stations from a CSV file, by defining the stat .. code-block:: bash python3 wis2box-ctl.py login - wis2box metadata station publish-collection --path /data/wis2box/mystations.csv --topic-hierarchy origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop + wis2box metadata station publish-collection --path /data/wis2box/mystations.csv --topic-hierarchy origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop .. note:: diff --git a/tests/data/metadata/discovery/cd-surface-weather-observations.yml b/tests/data/metadata/discovery/cd-surface-weather-observations.yml index 87980e5a6..2acda5756 100644 --- a/tests/data/metadata/discovery/cd-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/cd-surface-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P180D - topic_hierarchy: cd-brazza_met_centre/data/recommended/weather/surface-based-observations/synop + topic_hierarchy: cg-met/data/recommended/weather/surface-based-observations/synop country: cog - centre_id: cd-brazza_met_centre + centre_id: cg-met data_mappings: plugins: txt: @@ -19,7 +19,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations + identifier: urn:wmo:md:cg-met:surface-weather-observations hierarchylevel: dataset identification: diff --git a/tests/data/metadata/discovery/dz-surface-weather-observations.yml b/tests/data/metadata/discovery/dz-surface-weather-observations.yml index a850c49c7..a76bc5abd 100644 --- a/tests/data/metadata/discovery/dz-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/dz-surface-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: dz-alger_met_centre/data/core/weather/surface-based-observations/synop + topic_hierarchy: dz-meteoalgerie/data/core/weather/surface-based-observations/synop country: dza - centre_id: dz-alger_met_centre + centre_id: dz-meteoalgerie data_mappings: plugins: bufr4: @@ -20,7 +20,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:dz-alger_met_centre:surface-weather-observations + identifier: urn:wmo:md:dz-meteoalgerie:surface-weather-observations hierarchylevel: dataset identification: diff --git a/tests/data/metadata/discovery/int-wmo-test-buoy.yml b/tests/data/metadata/discovery/int-wmo-test-drifting-buoys.yml similarity index 84% rename from tests/data/metadata/discovery/int-wmo-test-buoy.yml rename to tests/data/metadata/discovery/int-wmo-test-drifting-buoys.yml index e439731de..1f356f7d9 100644 --- a/tests/data/metadata/discovery/int-wmo-test-buoy.yml +++ b/tests/data/metadata/discovery/int-wmo-test-drifting-buoys.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/buoy + topic_hierarchy: int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys country: int centre_id: int-wmo-test data_mappings: @@ -16,12 +16,12 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:buoy + identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:drifting-buoys hierarchylevel: dataset identification: - title: Example surface observations from GOOS buoy networks - abstract: Surface observations from GOOS buoy networks downloaded from the WIS. + title: Example surface observations from GOOS drifting buoy networks + abstract: Surface observations from GOOS drifting buoy networks downloaded from the WIS. dates: creation: 2024-02-16 keywords: @@ -30,7 +30,7 @@ identification: - surface weather - temperature - observations - - buoy + - drifting buoy wmo: keywords: - weather @@ -50,7 +50,7 @@ identification: url: https://example.org/fixme contact: - pointOfContact: &contact_poc + host: &contact_poc organization: World Meteorological Organisation url: https://www.wmo.int/ individualname: Firstname Lastname @@ -65,5 +65,3 @@ contact: email: null hoursofservice: 0800h - 1600h UTC contactinstructions: email - - distributor: *contact_poc diff --git a/tests/data/metadata/discovery/int-wmo-test-ship.yml b/tests/data/metadata/discovery/int-wmo-test-ship-hourly.yml similarity index 95% rename from tests/data/metadata/discovery/int-wmo-test-ship.yml rename to tests/data/metadata/discovery/int-wmo-test-ship-hourly.yml index 56a37695e..3d5bd9f80 100644 --- a/tests/data/metadata/discovery/int-wmo-test-ship.yml +++ b/tests/data/metadata/discovery/int-wmo-test-ship-hourly.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/ship + topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/ship-hourly country: int centre_id: int-wmo-test data_mappings: @@ -16,7 +16,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:ship + identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:ship-hourly hierarchylevel: dataset identification: @@ -50,7 +50,7 @@ identification: url: https://example.org/fixme contact: - pointOfContact: &contact_poc + host: &contact_poc organization: World Meteorological Organisation url: https://www.wmo.int/ individualname: Firstname Lastname @@ -65,5 +65,3 @@ contact: email: you@example.org hoursofservice: 0800h - 1600h UTC contactinstructions: email - - distributor: *contact_poc diff --git a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml b/tests/data/metadata/discovery/int-wmo-test-wind-profile.yml similarity index 94% rename from tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml rename to tests/data/metadata/discovery/int-wmo-test-wind-profile.yml index e6eeacd36..1e12f3395 100644 --- a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml +++ b/tests/data/metadata/discovery/int-wmo-test-wind-profile.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/wind_profiler + topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/wind-profile country: int centre_id: int-wmo-test data_mappings: @@ -16,7 +16,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:wind_profiler + identifier: urn:wmo:md:int-wmo-test:surface-weather-observations:wind-profile hierarchylevel: dataset identification: @@ -47,7 +47,7 @@ identification: url: https://example.org/fixme contact: - pointOfContact: &contact_poc + host: &contact_poc organization: World Meteorological Organisation url: https://www.wmo.int/ individualname: Firstname Lastname @@ -62,5 +62,3 @@ contact: email: you@example.org hoursofservice: 0800h - 1600h UTC contactinstructions: email - - distributor: *contact_poc diff --git a/tests/data/metadata/discovery/it-surface-weather-observations.yml b/tests/data/metadata/discovery/it-surface-weather-observations.yml index 14e2c1a14..ced07454b 100644 --- a/tests/data/metadata/discovery/it-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/it-surface-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: it-roma_met_centre/data/core/weather/surface-based-observations/synop + topic_hierarchy: it-meteoam/data/core/weather/surface-based-observations/synop country: ita - centre_id: it-roma_met_centre + centre_id: it-meteoam data_mappings: plugins: bin: @@ -17,7 +17,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:it-roma_met_centre:surface-weather-observations + identifier: urn:wmo:md:it-meteoam:surface-weather-observations hierarchylevel: dataset identification: diff --git a/tests/data/metadata/discovery/mw-surface-weather-observations.yml b/tests/data/metadata/discovery/mw-surface-weather-observations.yml index bfd80d255..b7c3da6cd 100644 --- a/tests/data/metadata/discovery/mw-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/mw-surface-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: mw-mw_met_centre/data/core/weather/surface-based-observations/synop + topic_hierarchy: mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop country: mwi - centre_id: mw-mw_met_centre + centre_id: mw-mw_met_centre-test data_mappings: plugins: csv: @@ -18,7 +18,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:mw-mw_met_centre:surface-weather-observations + identifier: urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations hierarchylevel: dataset identification: diff --git a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml index e55d791bf..bd1a59734 100644 --- a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml +++ b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: ro-rnimh/data/core/weather/surface-based-observations/synop + topic_hierarchy: ro-rnimh-test/data/core/weather/surface-based-observations/synop country: rou - centre_id: ro-rnimh + centre_id: ro-rnimh-test data_mappings: plugins: txt: @@ -19,7 +19,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:ro-rnimh:synoptic-weather-observations + identifier: urn:wmo:md:ro-rnimh-test:synoptic-weather-observations hierarchylevel: dataset identification: diff --git a/tests/data/metadata/station/wmo-test-buoy.csv b/tests/data/metadata/station/wmo-test-drifting-buoys.csv similarity index 100% rename from tests/data/metadata/station/wmo-test-buoy.csv rename to tests/data/metadata/station/wmo-test-drifting-buoys.csv diff --git a/tests/data/metadata/station/wmo-test-ship.csv b/tests/data/metadata/station/wmo-test-ship-hourly.csv similarity index 100% rename from tests/data/metadata/station/wmo-test-ship.csv rename to tests/data/metadata/station/wmo-test-ship-hourly.csv diff --git a/tests/data/metadata/station/wmo-test-wind_profiler.csv b/tests/data/metadata/station/wmo-test-wind-profile.csv similarity index 100% rename from tests/data/metadata/station/wmo-test-wind_profiler.csv rename to tests/data/metadata/station/wmo-test-wind-profile.csv diff --git a/tests/data/observations/wmo/buoy/A_IOBH03LFPW161000_C_EDZW_20240216112202_27622614.bin b/tests/data/observations/wmo/drifting-buoys/A_IOBH03LFPW161000_C_EDZW_20240216112202_27622614.bin similarity index 100% rename from tests/data/observations/wmo/buoy/A_IOBH03LFPW161000_C_EDZW_20240216112202_27622614.bin rename to tests/data/observations/wmo/drifting-buoys/A_IOBH03LFPW161000_C_EDZW_20240216112202_27622614.bin diff --git a/tests/data/observations/wmo/ship/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin b/tests/data/observations/wmo/ship-hourly/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin similarity index 100% rename from tests/data/observations/wmo/ship/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin rename to tests/data/observations/wmo/ship-hourly/A_ISSA05LFPW111700RRA_C_EDZW_20240211170705_50982396.bin diff --git a/tests/data/observations/wmo/wind_profiler/WIGOS_0-20000-0-48698_20240301T140000-578.bin b/tests/data/observations/wmo/wind-profile/WIGOS_0-20000-0-48698_20240301T140000-578.bin similarity index 100% rename from tests/data/observations/wmo/wind_profiler/WIGOS_0-20000-0-48698_20240301T140000-578.bin rename to tests/data/observations/wmo/wind-profile/WIGOS_0-20000-0-48698_20240301T140000-578.bin diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 2aa29766c..513536730 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -35,7 +35,7 @@ URL = 'http://localhost' API_URL = f'{URL}/oapi' -ID = 'urn:wmo:md:mw-mw_met_centre:surface-weather-observations' +ID = 'urn:wmo:md:mw-mw_met_centre-test:surface-weather-observations' SESSION = Session() SESSION.hooks = { 'response': lambda r, *args, **kwargs: r.raise_for_status() @@ -49,15 +49,15 @@ def test_wis2downloader(): DOWNLOAD_DIR = DATADIR / 'downloads' topic_nfiles_dict = { - 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop': 23, # noqa - 'origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop': 28, # noqa + 'origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop': 23, # noqa + 'origin/a/wis2/dz-meteoalgerie/data/core/weather/surface-based-observations/synop': 28, # noqa 'origin/a/wis2/cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global': 10, # noqa - 'origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop': 49, # noqa - 'origin/a/wis2/cd-brazza_met_centre/data/recommended/weather/surface-based-observations/synop': 0, # noqa - 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy': 2, # noqa - 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler': 1, # noqa - 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa - 'origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop': 31 # noqa + 'origin/a/wis2/ro-rnimh-test/data/core/weather/surface-based-observations/synop': 49, # noqa + 'origin/a/wis2/cg-met/data/recommended/weather/surface-based-observations/synop': 0, # noqa + 'origin/a/wis2/int-wmo-test/data/core/ocean/surface-based-observations/drifting-buoys': 2, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile': 1, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship-hourly': 5, # noqa + 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31 # noqa } topic_nfiles_dict_found = {} @@ -150,7 +150,7 @@ def test_metadata_discovery_publish(): mqtt_link = [d for d in r['links'] if d['href'].startswith('mqtt')][0] assert 'everyone:everyone' in mqtt_link['href'] - assert mqtt_link['channel'] == 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop' # noqa + assert mqtt_link['channel'] == 'origin/a/wis2/mw-mw_met_centre-test/data/core/weather/surface-based-observations/synop' # noqa params = { 'q': 'temperature' @@ -164,11 +164,11 @@ def test_metadata_discovery_publish(): # test access of discovery metadata from notification message centre_ids = [ - 'mw-mw_met_centre', - 'it-roma_met_centre', - 'dz-alger_met_centre', - 'ro-rnimh', - 'cd-brazza_met_centre', + 'mw-mw_met_centre-test', + 'it-meteoam', + 'dz-meteoalgerie', + 'ro-rnimh-test', + 'cg-met', 'int-wmo-test' ] @@ -196,7 +196,7 @@ def test_metadata_discovery_publish(): r = r.json() assert r['conformsTo'][0] == 'http://wis.wmo.int/spec/wcmp/2/conf/core' - id_ = 'urn:wmo:md:cd-brazza_met_centre:surface-weather-observations' + id_ = 'urn:wmo:md:cg-met:surface-weather-observations' r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items/{id_}').json() # noqa assert 'has_auth' in r['wis2box'] @@ -298,11 +298,11 @@ def test_message_api(): # test messages per test dataset counts = { 'mw-mw_met_centre': 25, - 'it-roma_met_centre': 33, - 'dz-alger_met_centre': 29, + 'it-meteoam': 33, + 'dz-meteoalgerie': 29, 'ro-rnimh': 50, - 'cd-brazza_met_centre': 15, - 'int-wmo-test': 11, + 'cg-met': 15, + 'int-wmo': 11, 'cn-cma': 11 } for key, value in counts.items(): @@ -317,7 +317,7 @@ def test_message_api(): assert r['numberMatched'] == sum(counts.values()) # we want to find a particular message with data ID for core data - target_data_id = 'mw-mw_met_centre:surface-weather-observations/WIGOS_0-454-2-AWSLOBI_20211111T125500' # noqa + target_data_id = 'mw-mw_met_centre-test:surface-weather-observations/WIGOS_0-454-2-AWSLOBI_20211111T125500' # noqa msg = None for feature in r['features']: @@ -336,7 +336,7 @@ def test_message_api(): assert props['datetime'] == '2021-11-11T12:55:00Z' assert props['wigos_station_identifier'] == '0-454-2-AWSLOBI' assert props['integrity']['method'] == 'sha512' - assert not props['data_id'].startswith(('wis2', 'origin/a/wis2')) + assert not props['data_id'].startswith(('origin/a/wis2', 'wis2')) assert props['data_id'].startswith('mw') assert props['content']['size'] == 247 assert props['content']['encoding'] == 'base64' @@ -355,10 +355,10 @@ def test_message_api(): assert b'BUFR' in r.content # we want to find a particular message with data ID for recommended data - url = f'{API_URL}/collections/messages/items?sortby=-datetime&q=cd-brazza_met_centre' # noqa + url = f'{API_URL}/collections/messages/items?sortby=-datetime&q=cg-met' # noqa r = SESSION.get(url).json() - target_data_id = "cd-brazza_met_centre:surface-weather-observations/WIGOS_0-20000-0-64406_20230803T090000" # noqa + target_data_id = "cg-met:surface-weather-observations/WIGOS_0-20000-0-64406_20230803T090000" # noqa msg = None for feature in r['features']: @@ -377,9 +377,8 @@ def test_message_api(): assert props['datetime'] == '2023-08-03T09:00:00Z' assert props['wigos_station_identifier'] == '0-20000-0-64406' assert props['integrity']['method'] == 'sha512' - assert not props['data_id'].startswith('wis2') - assert not props['data_id'].startswith('origin/a/wis2') - assert props['data_id'].startswith('cd') + assert not props['data_id'].startswith(('origin/a/wis2', 'wis2')) + assert props['data_id'].startswith('cg') assert 'content' not in props assert 'gts' in props assert props['gts']['ttaaii'] == 'SICG20' diff --git a/tests/requirements.txt b/tests/requirements.txt index d910638e2..4a353aca0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ flake8 paho-mqtt<2 pytest +pywcmp pywis-pubsub requests \ No newline at end of file diff --git a/tests/test.env b/tests/test.env index 82ae0dff9..d1807fa73 100644 --- a/tests/test.env +++ b/tests/test.env @@ -68,4 +68,4 @@ DOWNLOAD_BROKER_PORT=1883 DOWNLOAD_BROKER_USERNAME=everyone DOWNLOAD_BROKER_PASSWORD=everyone DOWNLOAD_BROKER_TRANSPORT=tcp -DOWNLOAD_MIN_FREE_SPACE_GB=0 \ No newline at end of file +DOWNLOAD_MIN_FREE_SPACE_GB=0 diff --git a/wis2box-create-config.py b/wis2box-create-config.py index 11a5ff6d0..cd3107b79 100644 --- a/wis2box-create-config.py +++ b/wis2box-create-config.py @@ -18,16 +18,26 @@ # under the License. # ############################################################################### - import datetime import json import os +import platform from pathlib import Path import random import string from string import Template from typing import Tuple +# Identify platform type +WINDOWS = False +if platform.system() == 'Windows': + WINDOWS = True + +if not WINDOWS: + import grp + import shutil + DOCKER_GROUP = grp.getgrnam('docker') + OTHER_TLDS = ['org', 'int'] @@ -274,11 +284,11 @@ def get_wis2box_url() -> str: return wis2box_url -def create_wis2box_env(config_dir: str) -> None: +def create_wis2box_env(host_datadir: str) -> None: """ - creates the wis2box.env file in the config_dir + creates the wis2box.env file in the host_datadir - :param config_dir: `str` of path to the config directory + :param host_datadir: `str` of path to the config directory :returns: None """ @@ -287,7 +297,7 @@ def create_wis2box_env(config_dir: str) -> None: with wis2box_env.open('w') as fh: fh.write('# directory on the host with wis2box-configuration\n') # noqa - fh.write(f'WIS2BOX_HOST_DATADIR={config_dir}\n') + fh.write(f'WIS2BOX_HOST_DATADIR={host_datadir}\n') fh.write(f'# directory in the wis2box container with wis2box-configuration\n') # noqa fh.write('WIS2BOX_DATADIR=/data/wis2box\n') fh.write('\n') @@ -368,6 +378,8 @@ def create_wis2box_env(config_dir: str) -> None: fh.write('DOWNLOAD_BROKER_TRANSPORT=websockets\n') fh.write('# maximum MB in download directory\n') fh.write('DOWNLOAD_MIN_FREE_SPACE_GB=1\n') + if not WINDOWS: + fh.write(f'DOCKER_GID={DOCKER_GROUP.gr_gid}\n') fh.write('\n') print('*' * 80) @@ -375,88 +387,93 @@ def create_wis2box_env(config_dir: str) -> None: print('*' * 80) -def create_config_dir() -> str: +def create_host_datadir() -> str: """ - Creates the directory config_dir + Creates the directory wis2box_host_datadir If the directory already exists, asks the user if they want to overwrite the existing files - :returns: `str` of path to directory where configuration files - are to be stored + :returns: `str` of path to directory for wis2box_host_datadir """ - config_dir = "" + host_datadir = "" answer = "n" while answer != "y": if answer == "exit": exit() - print("Please enter the directory on the host where wis2box-configuration-files are to be stored:") # noqa - config_dir = input() + print("Please enter the directory to be used for WIS2BOX_HOST_DATADIR:") # noqa + host_datadir = input() - if config_dir == "": + if host_datadir == "": print("The directory cannot be empty.") continue - print("Configuration-files will be stored in the following directory:") # noqa - print(f" {config_dir}") + print("The directory to be used for WIS2BOX_HOST_DATADIR will be set to:") # noqa + print(f" {host_datadir}") print("Is this correct? (y/n/exit)") answer = input() # check if the directory exists try: - config_dir = Path(config_dir) - if config_dir.is_dir(): + host_datadir = Path(host_datadir) + if host_datadir.is_dir(): # if it exists warn the user # tell them that the directory needs to be remove to continue print("WARNING:") - print(f"The directory {config_dir} already exists.") + print(f"The directory {host_datadir} already exists.") print("Please remove the directory to restart the configuration process.") # noqa exit() else: # if it does not exist, create it - config_dir.mkdir(parents=True) + host_datadir.mkdir(parents=True) # check if the directory was created - if not config_dir.is_dir(): + if not host_datadir.is_dir(): print("ERROR:") - print(f"The directory {config_dir} could not be created.") + print(f"The directory {host_datadir} could not be created.") print("Please check the path and your permissions.") exit() - print(f"The directory {config_dir} has been created.") + print(f"The directory {host_datadir} has been created.") + + download_dir = host_datadir / 'downloads' + download_dir.mkdir(mode=0o775) + if not WINDOWS: + shutil.chown(download_dir, group='docker') + except Exception: print("ERROR:") - print(f"The directory {config_dir} could not be created.") + print(f"The directory {host_datadir} could not be created.") print("Please provide an absolute path to the directory.") print("and check your permissions.") exit() - return config_dir + return host_datadir -def create_metadata_file(config_dir: str, country_name: str, +def create_metadata_file(host_datadir: str, country_name: str, centre_id: str, centre_name: str, wis2box_email: str, bounding_box: str, template: str) -> str: """ - creates the metadata file in the directory config_dir + creates the metadata file in the directory host_datadir - :param config_dir: `str` of the path to the directory where the configuration files are to be stored # noqa + :param host_datadir: `str` of the path to the wis2box-host-datadir :param country_name: `str` of the country name of the wis2box :param centre_id: `str` of the centre id of the wis2box :param centre_name: `str` of centre name of the wis2box :param wis2box_email: `str` of centre email :param bounding_box: `str` of CSV of bounding box :param template: `str` of synop or temp - + :returns: `str` of the path to the metadata file """ # get current date as a string current_date = datetime.datetime.now().strftime("%Y-%m-%d") - config_dir = Path(config_dir) - discovery_metadata_dir = config_dir / 'metadata' / 'discovery' + host_datadir = Path(host_datadir) + discovery_metadata_dir = host_datadir / 'metadata' / 'discovery' # create directory for discovery metadata if it does not exist if not discovery_metadata_dir.exists(): @@ -486,12 +503,12 @@ def create_metadata_file(config_dir: str, country_name: str, return new_config_file.name -def create_metadata_files(config_dir: str, country_code: str, +def create_metadata_files(host_datadir: str, country_code: str, centre_id: str) -> None: """ - creates the metadata files in the directory config_dir + creates the metadata files in the directory host_datadir - :config_dir: `str` of path to directory where configuration files + :host_datadir: `str` of path to directory where configuration files are to be stored # noqa :country_code: `str` of country code of the country hosting the wis2box :centre_id: `str` of centre id of the organization hosting the wis2box @@ -531,7 +548,7 @@ def create_metadata_files(config_dir: str, country_code: str, country_name, bounding_box = get_bounding_box(country_code) create_metadata_file( - config_dir, + host_datadir, country_name, centre_id, centre_name, @@ -540,7 +557,7 @@ def create_metadata_files(config_dir: str, country_code: str, template="synop" ) create_metadata_file( - config_dir, + host_datadir, country_name, centre_id, centre_name, @@ -550,22 +567,22 @@ def create_metadata_files(config_dir: str, country_code: str, ) print("*" * 80) - print(f"Metadata files for {centre_id} created in directory {config_dir}.") # noqa + print(f"Metadata files for {centre_id} created in directory {host_datadir}.") # noqa print("Please review the files and edit where necessary.") print("*" * 80) -def create_station_list(config_dir: str) -> None: +def create_station_list(host_datadir: str) -> None: """ - creates the station list file in the directory config_dir + creates the station list file in the directory host_datadir - :param config_dir: `str` of path to directory where configuration files + :param host_datadir: `str` of path to directory where configuration files are to be stored :returns: None """ - station_metadata_dir = Path(config_dir) / 'metadata' / 'station' + station_metadata_dir = Path(host_datadir) / 'metadata' / 'station' # create directory for station metadata if it does not exist if not station_metadata_dir.exists(): station_metadata_dir.mkdir(parents=True) @@ -576,41 +593,41 @@ def create_station_list(config_dir: str) -> None: fh2.write(headers) -def get_config_dir() -> str: +def get_host_datadir() -> str: """ reads the value of WIS2BOX_HOST_DATADIR from wis2box.env - returns: `str` of path to directory where configuration files - are to be stored + returns: `str` of path for WIS2BOX_HOST_DATADIR """ - config_dir = None + host_datadir = None with Path("wis2box.env").open() as fh: lines = fh.readlines() for line in lines: if "WIS2BOX_HOST_DATADIR" in line: - config_dir = line.split("=")[1].strip() + host_datadir = line.split("=")[1].strip() - if not config_dir: + if not host_datadir: print("WARNING:") print("The file wis2box.env does not contain the variable WIS2BOX_HOST_DATADIR.") # noqa print("Please edit the file and add the variable WIS2BOX_HOST_DATADIR.") # noqa print("Or remove wis2box.env and run 'python3 wis2box-create-config.py' again.") # noqa exit() - return config_dir + return host_datadir def main(): """ mainline function - creates the configuration files for the wis2box + creates the wis2box.env file + and sets up the directory for WIS2BOX_HOST_DATADIR """ - config_dir = None + host_datadir = None dev_env = Path("wis2box.env") # check if wis2box.env exists @@ -626,18 +643,18 @@ def main(): elif answer == "exit": exit() else: - config_dir = get_config_dir() + host_datadir = get_host_datadir() - # if config_dir is not defined - if not config_dir: - config_dir = create_config_dir() + # if host_datadir is not defined + if not host_datadir: + host_datadir = create_host_datadir() - create_station_list(config_dir) + create_station_list(host_datadir) # if wis2box.env does not exist - # create it and write config_dir as the value for WIS2BOX_HOST_DATADIR to wis2box.env # noqa + # create it and write host_datadir as the value for WIS2BOX_HOST_DATADIR to wis2box.env # noqa if not dev_env.is_file(): - create_wis2box_env(config_dir) + create_wis2box_env(host_datadir) print("The configuration is complete.") exit() diff --git a/wis2box-ctl.py b/wis2box-ctl.py index 1e6477260..070a8d191 100755 --- a/wis2box-ctl.py +++ b/wis2box-ctl.py @@ -123,20 +123,13 @@ def find_files(path: str, extension: str) -> list: return file_list -def run(args, cmd, asciiPipe=False) -> str: +def run(cmd, silence_stderr=False) -> None: - if args.simulate: - if asciiPipe: - print(f"simulation: {' '.join(cmd)} >/tmp/temp_buffer$$.txt") - else: - print(f"simulation: {' '.join(cmd)}") - return '`cat /tmp/temp_buffer$$.txt`' + if not silence_stderr: + subprocess.run(cmd) else: - if asciiPipe: - return subprocess.run( - cmd, stdout=subprocess.PIPE).stdout.decode('ascii') - else: - subprocess.run(cmd) + subprocess.run(cmd, stderr=subprocess.DEVNULL) + return None @@ -174,66 +167,64 @@ def make(args) -> None: container = "wis2box-management" if not args.args else ' '.join(args.args) if args.command == "config": - run(args, split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} config')) + run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} config')) elif args.command == "build": - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} build {containers}')) elif args.command in ["up", "start", "start-dev"]: - run(args, split( - 'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions')) - run(args, split( - 'docker plugin enable loki')) + run(split( + 'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions'), + silence_stderr=True) + run(split('docker plugin enable loki'), silence_stderr=True) if containers: - run(args, split(f"{DOCKER_COMPOSE_COMMAND} {docker_compose_args} start {containers}")) + run(split(f"{DOCKER_COMPOSE_COMMAND} {docker_compose_args} start {containers}")) else: if args.command == 'start-dev': - run(args, split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} --file docker-compose.dev.yml up -d')) + run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} --file docker-compose.dev.yml up -d')) else: - run(args, split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d')) + run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d')) elif args.command == "execute": - run(args, ['docker', 'exec', '-i', 'wis2box-management', 'sh', '-c', containers]) + run(['docker', 'exec', '-i', 'wis2box-management', 'sh', '-c', containers]) elif args.command == "login": - run(args, split(f'docker exec -it {container} /bin/bash')) + run(split(f'docker exec -it {container} /bin/bash')) elif args.command == "login-root": - run(args, split(f'docker exec -u -0 -it {container} /bin/bash')) + run(split(f'docker exec -u -0 -it {container} /bin/bash')) elif args.command == "logs": - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} logs --follow {containers}')) elif args.command in ["stop", "down"]: if containers: - run(args, split(f"{DOCKER_COMPOSE_COMMAND} {docker_compose_args} {containers}")) + run(split(f"{DOCKER_COMPOSE_COMMAND} {docker_compose_args} {containers}")) else: - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans {containers}')) elif args.command == "update": - run(args, split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} pull')) + run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} pull')) elif args.command == "prune": - run(args, split('docker builder prune -f')) - run(args, split('docker container prune -f')) - run(args, split('docker volume prune -f')) - _ = run(args, - split('docker images --filter dangling=true -q --no-trunc'), - asciiPipe=True) - run(args, split(f'docker rmi {_}')) - _ = run(args, split('docker ps -a -q'), asciiPipe=True) - run(args, split(f'docker rm {_}')) + run(split('docker builder prune -f')) + run(split('docker container prune -f')) + run( split('docker volume prune -f')) + _ = run(split('docker images --filter dangling=true -q --no-trunc')) + run(split(f'docker rmi {_}')) + _ = run(split('docker ps -a -q')) + run(split(f'docker rm {_}')) elif args.command == "restart": if containers: - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} stop {containers}')) - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} start {containers}')) else: - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans')) - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d')) elif args.command == "status": - run(args, split( + run(split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} ps {containers}')) elif args.command == "lint": files = find_files(".", '.py') - run(args, ('python3', '-m', 'flake8', *files)) + run(('python3', '-m', 'flake8', *files)) if __name__ == "__main__": diff --git a/wis2box-management/wis2box/pubsub/subscribe.py b/wis2box-management/wis2box/pubsub/subscribe.py index 19e8aaed5..a4e095575 100644 --- a/wis2box-management/wis2box/pubsub/subscribe.py +++ b/wis2box-management/wis2box/pubsub/subscribe.py @@ -106,10 +106,12 @@ def handle_publish(self, message, publisher='wis2box'): topic_hierarchy = message['channel'] metadata_id = message.get('metadata_id') + # if metadata_id not provided, log error and return if metadata_id is None: - msg = 'metadata_id not found in message, cannot publish data' - LOGGER.error(msg) - return False + LOGGER.error('metadata_id not provided in message received on topic wis2box/data/publication') # noqa + # ensure topic_hierarchy starts with 'origin/a/wis2/' + if not topic_hierarchy.startswith('origin/a/wis2/'): + topic_hierarchy = f'origin/a/wis2/{topic_hierarchy}' defs = { 'topic_hierarchy': topic_hierarchy, diff --git a/wis2downloader/Dockerfile b/wis2downloader/Dockerfile deleted file mode 100644 index c52784ec0..000000000 --- a/wis2downloader/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Use an official Python runtime as a parent image -FROM python:3.9-slim - -EXPOSE 5000 - -# define config variables -ENV DOWNLOAD_DIR /app/downloads -ENV DOWNLOAD_RETENTION_PERIOD_HOURS 24 -ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr" -ENV DOWNLOAD_BROKER_PORT 443 -ENV DOWNLOAD_BROKER_USERNAME "everyone" -ENV DOWNLOAD_BROKER_PASSWORD "everyone" -ENV DOWNLOAD_BROKER_TRANSPORT "websockets" -ENV DOWNLOAD_WORKERS 8 -ENV DOWNLOAD_MIN_FREE_SPACE_GB 1 -ENV DOWNLOAD_VALIDATE_TOPICS "false" -ENV LOG_PATH /app/logs -ENV WIS2DOWNLOADER_CONFIG "/app/config.json" - -# update pyopenssl and pin requests and urllib3 to avoid SSL error -RUN pip install pyopenssl --upgrade && pip install requests==2.26.0 urllib3==1.26.0 -# install cron and envsubst and guincorn -RUN apt-get update && apt-get install -y cron gettext-base gunicorn - -# copy all the code to the Docker image -COPY . /app - -# Set the working directory to /app -WORKDIR /app - -# install wis2downloader from pypi -RUN pip install wis2downloader - -# add wis2box.cron to crontab -COPY ./clean.cron /etc/cron.d/clean.cron - -# set permissions for the cron job and install it -RUN chmod 0644 /etc/cron.d/clean.cron && crontab /etc/cron.d/clean.cron - -# set permissions for the entrypoint script -RUN chmod +x /app/entrypoint.sh - -ENTRYPOINT [ "/app/entrypoint.sh" ] - -# Run wis2downloader when the container launches -CMD ["gunicorn","--bind","0.0.0.0:5000","--pythonpath","/usr/local/lib/python3.9/site-packages/","--workers", "1", "wis2downloader.app:app"] diff --git a/wis2downloader/clean.cron b/wis2downloader/clean.cron deleted file mode 100644 index 1ae6e9c16..000000000 --- a/wis2downloader/clean.cron +++ /dev/null @@ -1 +0,0 @@ -*/10 * * * * /usr/local/bin/python /app/clean.py > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/wis2downloader/clean.py b/wis2downloader/clean.py deleted file mode 100644 index 2fd2dc7ea..000000000 --- a/wis2downloader/clean.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import time - -# get download dir from environment variable -download_dir = os.environ.get('DOWNLOAD_DIR', '/app/downloads') -# get retention period from environment variable -retention_period_hours = int(os.environ.get('RETENTION_PERIOD_HOURS', 24)) - - -def clean_directory(directory): - # get the current time - current_time = time.time() - - files_removed = 0 - directories_removed = 0 - # loop through the files in the directory, including subdirectories - for file in os.listdir(directory): - # get the full path of the file - file_path = os.path.join(directory, file) - # check if the path is a file or a directory - if os.path.isfile(file_path): - # get the time the file was last modified - file_time = os.path.getmtime(file_path) - # check if the file is older than the retention period - if current_time - file_time > retention_period_hours * 3600: - os.remove(file_path) - files_removed += 1 - elif os.path.isdir(file_path): - # recursively clean the directory - clean_directory(file_path) - # if the directory is empty, remove it - if not os.listdir(file_path): - os.rmdir(file_path) - directories_removed += 1 - print(f'CLEANER: removed {files_removed} old files and {directories_removed} empty directories') # noqa - - -# start cleaning from the download directory -clean_directory(download_dir) diff --git a/wis2downloader/config.template b/wis2downloader/config.template deleted file mode 100644 index 7f11111ec..000000000 --- a/wis2downloader/config.template +++ /dev/null @@ -1,18 +0,0 @@ -{ - "broker_hostname": "${DOWNLOAD_BROKER_HOST}", - "broker_port": ${DOWNLOAD_BROKER_PORT}, - "broker_username": "${DOWNLOAD_BROKER_USERNAME}", - "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", - "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", - "download_workers": ${DOWNLOAD_WORKERS}, - "min_free_space": ${DOWNLOAD_MIN_FREE_SPACE_GB}, - "validate_topics": ${DOWNLOAD_VALIDATE_TOPICS}, - "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json", - "download_dir": "${DOWNLOAD_DIR}", - "log_level": "INFO", - "base_url": "http://localhost:5000", - "flask_host": "0.0.0.0", - "flask_port": 5000, - "save_logs": false, - "log_path": "logs" -} diff --git a/wis2downloader/entrypoint.sh b/wis2downloader/entrypoint.sh deleted file mode 100644 index 61386e132..000000000 --- a/wis2downloader/entrypoint.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -echo "START /entrypoint.sh" - -set -e - -# ensure cron is running -service cron start -service cron status - -echo "END /entrypoint.sh" - -# print the download_dir -echo "Download directory in container: $DOWNLOAD_DIR" -# print the retention period hours -echo "Retention period in hours: $RETENTION_PERIOD_HOURS" - -# ensure DOWNLOAD_DIR exists -if [ ! -d $DOWNLOAD_DIR ]; then - echo "Creating download directory: $DOWNLOAD_DIR" - mkdir -p $DOWNLOAD_DIR -fi -envsubst < config.template > config.json - -# if session-info.json does not exists in $DOWNLOAD_DIR, create it -if [ ! -f $DOWNLOAD_DIR/.session-info.json ]; then - echo "Creating .session-info.json" - echo "{" > $DOWNLOAD_DIR/.session-info.json - echo ' "topics": {},' >> $DOWNLOAD_DIR/.session-info.json - # generate a random string for client_id - echo "Generating random client_id" - echo ' "client_id": "wis2box-wis2downloader-'$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)'"' >> $DOWNLOAD_DIR/.session-info.json - echo "}" >> $DOWNLOAD_DIR/.session-info.json -fi - -# print the config -echo "Config:" -cat /app/config.json - -exec "$@"