diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 373ff270..b895d3d0 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -28,8 +28,8 @@ jobs: python3 -V - name: setup wis2box configuration run: | - cp tests/test.env dev.env - cat dev.env + cp tests/test.env wis2box.env + cat wis2box.env python3 wis2box-ctl.py config - name: build wis2box run: | @@ -44,6 +44,9 @@ jobs: run: | sleep 30 python3 wis2box-ctl.py execute wis2box environment show + - name: populate stations from CSV 📡 + run: | + python3 wis2box-ctl.py execute wis2box metadata station publish-collection - name: add Malawi data 🇲🇼 env: TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop @@ -110,7 +113,7 @@ jobs: python3 wis2box-ctl.py execute wis2box metadata station publish-collection - name: run integration tests ⚙️ run: | - sleep 10 + sleep 15 pytest -s tests/integration - name: run flake8 ⚙️ run: | @@ -118,5 +121,6 @@ jobs: - name: failed tests 🚩 if: ${{ failure() }} run: | - docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project ps - docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project logs + docker logs wis2box-management + docker logs wis2box-api + docker logs wis2box-minio diff --git a/.gitignore b/.gitignore index c4ff2aca..886d2bb3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ nosetests.xml wis2box-data .pytest_cache dev.env +wis2box.env docker/.env .ipynb_checkpoints diff --git a/config-templates/csv2bufr_mappings.json b/config-templates/csv2bufr_mappings.json deleted file mode 100644 index 46ecec2c..00000000 --- a/config-templates/csv2bufr_mappings.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [0,0], - "inputExtendedDelayedDescriptorReplicationFactor": [], - "wigos_station_identifier": "data:WIGOS_ID", - "number_header_rows": 1, - "column_names_row": 1, - "delimiter": ",", - "quoting": "QUOTE_NONE", - "header":[ - {"eccodes_key": "edition", "value": "const:4"}, - {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:0"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"}, - {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, - {"eccodes_key": "dataCategory", "value": "const:0"}, - {"eccodes_key": "internationalDataSubCategory", "value": "const:6"}, - {"eccodes_key": "masterTablesVersionNumber", "value": "const:30"}, - {"eccodes_key": "numberOfSubsets", "value": "const:1"}, - {"eccodes_key": "observedData", "value": "const:1"}, - {"eccodes_key": "compressedData", "value": "const:0"}, - {"eccodes_key": "typicalYear", "value":"data:YEAR"}, - {"eccodes_key": "typicalMonth", "value":"data:MONTH"}, - {"eccodes_key": "typicalDay", "value":"data:DAY OF MONTH"}, - {"eccodes_key": "typicalHour", "value":"data:HOUR"}, - {"eccodes_key": "typicalMinute", "value":"data:MINUTE"}, - {"eccodes_key": "unexpandedDescriptors", "value": "array:301150,307080"} - ], - "data": [ - {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:_wsi_series"}, - {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:_wsi_issuer"}, - {"eccodes_key": "#1#wigosIssueNumber", "value":"data:_wsi_issue_number"}, - {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:_wsi_local"}, - {"eccodes_key": "#1#year", "value":"data:YEAR", "valid_min": "const:2000", "valid_max": "const:2100"}, - {"eccodes_key": "#1#month", "value":"data:MONTH", "valid_min": "const:1", "valid_max": "const:12"}, - {"eccodes_key": "#1#day", "value":"data:DAY OF MONTH", "valid_min": "const:1", "valid_max": "const:31"}, - {"eccodes_key": "#1#hour", "value":"data:HOUR", "valid_min": "const:0", "valid_max": "const:23"}, - {"eccodes_key": "#1#minute", "value":"data:MINUTE", "valid_min": "const:0", "valid_max": "const:59"}, - {"eccodes_key": "#1#latitude", "value":"data:LATITUDE", "valid_min": "const:-90", "valid_max": "const:90"}, - {"eccodes_key": "#1#longitude", "value":"data:LONGITUDE", "valid_min": "const:-180", "valid_max": "const:180"}, - {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:STATION HEIGHT (m)"}, - {"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:BP HEIGHT (m)"}, - {"eccodes_key": "#1#nonCoordinatePressure", "value":"data:BARO PRESSURE(hPa)"}, - {"eccodes_key": "#1#airTemperature", "value":"data:AIR TEMP(C)", "offset": "const:273.15", "scale": "const:0"}, - {"eccodes_key": "#1#dewpointTemperature", "value":"data:DEW POINT(C)", "offset": "const:273.15", "scale": "const:0"}, - {"eccodes_key": "#1#relativeHumidity", "value":"data:REL HUMID(%)"}, - {"eccodes_key": "#1#windSpeed", "value":"data:WIND SPD(m/s)"}, - {"eccodes_key": "#1#windDirection", "value":"data:WIND DIR(deg)"}, - {"eccodes_key": "#2#timePeriod", "value":"const:-1"}, - {"eccodes_key": "#1#totalSunshine", "value":"data:HOURS SUN(hrs)"}, - {"eccodes_key": "#1#totalPrecipitationOrTotalWaterEquivalent", "value":"data:PRECIP(mm)"}, - {"eccodes_key": "#1#maximumWindGustSpeed", "value":"data:GUST(m/s)"}, - {"eccodes_key": "#1#maximumWindGustDirection", "value": "data:GUSTDIR(deg)"}, - {"eccodes_key": "#14#timePeriod", "value":"const:-1"}, - {"eccodes_key": "#1#globalSolarRadiationIntegratedOverPeriodSpecified", "value":"data:SOLAR RAD(WH/m2)", "offset": "const:0", "scale": "const:3.56"} - ] -} \ No newline at end of file diff --git a/config-templates/data-mappings.yml.tmpl b/config-templates/data-mappings.yml.tmpl index d5439529..0eabe4aa 100644 --- a/config-templates/data-mappings.yml.tmpl +++ b/config-templates/data-mappings.yml.tmpl @@ -9,7 +9,7 @@ data: file-pattern: '^.*-(\d{4})(\d{2}).*\.txt$$' csv: - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: /data/wis2box/csv2bufr_mappings.json + template: aws-template notify: true buckets: - $${WIS2BOX_STORAGE_INCOMING} @@ -37,7 +37,7 @@ data: buckets: - $${WIS2BOX_STORAGE_PUBLIC} file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$$' - COUNTRY_CODE.CENTRE_ID.data.core.weather.surface-based-observations.temp: + $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.temp: plugins: b: - plugin: wis2box.data.bufr4.ObservationDataBUFR diff --git a/config-templates/station_list_example.csv b/config-templates/station_list_example.csv index d0adcebd..c73ea327 100644 --- a/config-templates/station_list_example.csv +++ b/config-templates/station_list_example.csv @@ -1,4 +1 @@ station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region -EXAMPLE1-SURFACE,0-454-0-EXAMPLE,,Land (fixed),-14.98333,34.96666,618,,Malawi,I -EXAMPLE2-SURFACE,0-20000-0-12345,12345,Land (fixed),41.56361,14.65500,793,807.0,Italy,VI -EXAMPLE3-UPPERAIR,0-52-0-5432,54321,Air (fixed),13.08585,-59.48701,56.6,,Barbados,IV diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e6caf503..9bc3fcf0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,7 +4,7 @@ services: - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw - ./wis2box-management/wis2box/wis2box.cron:/etc/cron.d/wis2box:ro - ./wis2box-management/wis2box:/app/wis2box - command: ["wis2box", "pubsub" , "subscribe", "--broker", "http://wis2box-minio:9000", "--topic", "wis2box-storage/#", "--verbosity", "INFO"] + command: ["wis2box", "pubsub" , "subscribe"] wis2box-api: volumes: diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 9053bbb7..2bbfe6f1 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -37,8 +37,7 @@ services: container_name: mqtt_metrics_collector restart: unless-stopped env_file: - - default.env - - dev.env + - wis2box.env #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:1.0.beta1 build: context: ./wis2box-mqtt-metrics-collector @@ -59,25 +58,25 @@ services: command: - '--config.file=/etc/prometheus/prometheus.yml' - '--storage.tsdb.retention.time=10d' - # cadvisor to monitor our containers! - cadvisor: - image: gcr.io/cadvisor/cadvisor:v0.45.0 - container_name: cadvisor - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk:/dev/disk/:ro - depends_on: - - wis2box-management + # uncomment cadvisor to monitor containers + # cadvisor: + # image: gcr.io/cadvisor/cadvisor:v0.45.0 + # container_name: cadvisor + # volumes: + # - /:/rootfs:ro + # - /var/run:/var/run:ro + # - /var/run/docker.sock:/var/run/docker.sock:ro + # - /sys:/sys:ro + # - /var/lib/docker/:/var/lib/docker:ro + # - /dev/disk:/dev/disk/:ro + # depends_on: + # - wis2box-management # Grafana, graphical monitoring dashboards for wis2box using data from loki and prometheus grafana: <<: *logging container_name: grafana env_file: - - dev.env + - wis2box.env image: grafana/grafana-oss:9.0.3 volumes: - ./grafana/dashboards:/etc/grafana/provisioning/dashboards diff --git a/docker-compose.yml b/docker-compose.yml index 73817343..ddde1289 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,7 @@ services: image: nginx:alpine restart: always env_file: - - default.env - - dev.env + - wis2box.env depends_on: - wis2box-ui volumes: @@ -16,8 +15,7 @@ services: image: ghcr.io/wmo-im/wis2box-ui:0.5.3 restart: always env_file: - - default.env - - dev.env + - wis2box.env depends_on: - wis2box-api @@ -26,18 +24,23 @@ services: # image: ghcr.io/wmo-im/wis2box-ui-admin:latest # restart: always # env_file: -# - default.env -# - dev.env +# - wis2box.env # depends_on: # - wis2box-api + wis2box-webapp: + container_name: wis2box-webapp + image: ghcr.io/wmo-im/wis2box-webapp:latest + env_file: + - wis2box.env + wis2box-api: container_name: wis2box-api - image: ghcr.io/wmo-im/wis2box-api:latest + # image: ghcr.io/wmo-im/wis2box-api:latest + image: wmoim/wis2box-api:wis2box-1.0b5 restart: always env_file: - - default.env - - dev.env + - wis2box.env depends_on: elasticsearch: condition: service_healthy @@ -55,8 +58,7 @@ services: memswap_limit: 512m restart: always env_file: - - default.env - - dev.env + - wis2box.env command: server --console-address ":9001" /data # in a production-setup minio needs to be volumes: @@ -105,8 +107,7 @@ services: build: context: ./wis2box-broker env_file: - - default.env - - dev.env + - wis2box.env wis2box-management: container_name: wis2box-management @@ -118,8 +119,7 @@ services: context: ./wis2box-management #user: wis2box:wis2box env_file: - - default.env - - dev.env + - wis2box.env volumes: - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw depends_on: @@ -129,15 +129,14 @@ services: condition: service_started wis2box-api: condition: service_healthy - command: ["wis2box", "pubsub" , "subscribe", "--broker", "http://wis2box-minio:9000", "--topic", "wis2box-storage/#"] + command: ["wis2box", "pubsub" , "subscribe"] wis2box-auth: container_name: wis2box-auth image: ghcr.io/wmo-im/wis2box-auth:latest restart: always env_file: - - default.env - - dev.env + - wis2box.env volumes: - auth-data:/data/wis2box:rw depends_on: diff --git a/docs/source/_static/minio-login-screen2.png b/docs/source/_static/minio-login-screen2.png index 6b9e1d1d..6da7d613 100644 Binary files a/docs/source/_static/minio-login-screen2.png and b/docs/source/_static/minio-login-screen2.png differ diff --git a/docs/source/_static/wis2box-api-added-collection.png b/docs/source/_static/wis2box-api-added-collection.png index d2f75058..6ba4140d 100644 Binary files a/docs/source/_static/wis2box-api-added-collection.png and b/docs/source/_static/wis2box-api-added-collection.png differ diff --git a/docs/source/_static/wis2box-api-initial.png b/docs/source/_static/wis2box-api-initial.png index e1f23433..46e10f54 100644 Binary files a/docs/source/_static/wis2box-api-initial.png and b/docs/source/_static/wis2box-api-initial.png differ diff --git a/docs/source/_static/wis2box-webapp-stations.png b/docs/source/_static/wis2box-webapp-stations.png new file mode 100644 index 00000000..27b9da41 Binary files /dev/null and b/docs/source/_static/wis2box-webapp-stations.png differ diff --git a/docs/source/_static/wis2box-webapp-synop-form.png b/docs/source/_static/wis2box-webapp-synop-form.png new file mode 100644 index 00000000..474f97d8 Binary files /dev/null and b/docs/source/_static/wis2box-webapp-synop-form.png differ diff --git a/docs/source/community/troubleshooting.rst b/docs/source/community/troubleshooting.rst index 8ffa4c36..bca46641 100644 --- a/docs/source/community/troubleshooting.rst +++ b/docs/source/community/troubleshooting.rst @@ -15,7 +15,7 @@ The wis2box logging displays the error: OSError: Missing data mappings: [Errno 2] No such file or directory: '/data/wis2box/data-mappings.yml' -Check your dev.env and check value that was set for WIS2BOX_HOST_DATADIR +Check ``wis2box.env`` and check value that was set for WIS2BOX_HOST_DATADIR .. code-block:: bash @@ -39,7 +39,7 @@ After you have ensured the data-mappings.yml is in the directory defined by WIS2 Topic Hierarchy validation error: Unknown file type --------------------------------------------------- -Check your ``data-mappings.yml`` file to adjust the file extension expected by the plugins processing your dataset. +Check ``data-mappings.yml`` file to adjust the file extension expected by the plugins processing your dataset. If you are ingesting files with extension .bin: @@ -70,13 +70,12 @@ The Access Key Id you provided does not exist in our records ------------------------------------------------------------ If you see this error when uploading data to the wis2box-incoming storage, you have provided the wrong username and/or password to access MinIO. -Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` you have provided in your ``dev.env`` file. -The default username/password for MinIO is ``minio/minio123``. +Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` set in the ``wis2box.env`` file. Topic Hierarchy validation error: No plugins for ... in data mappings --------------------------------------------------------------------- -A file arrived a folder for which no matching dataset was defined in your ``data-mappings.yml``. +A file arrived a folder for which no matching dataset was defined in ``data-mappings.yml``. For dataset ``foo.bar``, store your file in the path ``/foo/bar/``. @@ -86,12 +85,18 @@ ERROR - Failed to publish, wsi: ..., tsi: XXXXX ----------------------------------------------- Data arrived for a station that is not present in the station metadata cache. -To add missing stations, update the file ``metadata/station/station_list.csv`` in the wis2box data directory (see :ref:`setup`). + +To add missing stations, use the station-editor in wis2box-webapp (from wis2box-1.0b5) or update the file ``metadata/station/station_list.csv`` in the wis2box data directory and run the command: + +.. code-block:: bash + + python3 wis2box-ctl.py login + wis2box metadata stations publish-collections Error: no such container: wis2box-management -------------------------------------------- -If the wis2box-management container is not running the 'login' command will fail. +If the wis2box-management container is not running, the ``login`` command will fail. The wis2box-management container depends on other services being available before it can successfully started. Please check all services are Running using the following command: @@ -102,26 +107,27 @@ Please check all services are Running using the following command: Possible issues are: -- The directory defined by WIS2BOX_HOST_DATADIR does not contain the file 'data-mappings.yml' or the file is invalid -- The directory defined by WIS2BOX_HOST_DATADIR does not contain the file 'metastation/station/station_list.csv' or the file is invalid -- WIS2BOX_STORAGE_PASSWORD is too short, minio will fail to start if you specify a WIS2BOX_STORAGE_PASSWORD of less than 8 characters +- The host ran out of diskspace, check the output of ``df -h`` and ensure there is sufficient space available +- The directory defined by ``WIS2BOX_HOST_DATADIR`` does not contain the file ``data-mappings.yml`` or the file is invalid +- The directory defined by ``WIS2BOX_HOST_DATADIR`` does not contain the file ``metastation/station/station_list.csv`` or the file is invalid +- ``WIS2BOX_STORAGE_PASSWORD`` is too short, MinIO will fail to start if you specify a ``WIS2BOX_STORAGE_PASSWORD`` of less than 8 characters wisbox-UI is empty ------------------ -If when you access the wis2box-UI you see the interface but no datasets are visible, check the WIS2BOX_URL and WIS2BOX_API_URL are set correctly. +If when you access the wis2box UI you see the interface but no datasets are visible; check the ``WIS2BOX_URL`` and ``WIS2BOX_API_URL`` are set correctly. -Please note that after changing the WIS2BOX_URL and WIS2BOX_API_URL, you will have to restart your wis2box: +Please note that after changing the ``WIS2BOX_URL`` and ``WIS2BOX_API_URL``, you will have to restart wis2box: .. code-block:: bash python3 wis2box-ctl.py stop python3 wis2box-ctl.py start -And repeat the commands for adding your dataset and publishing your metadata, to ensure the URLs are updated in the records: +..and then repeat the commands for adding your dataset and publishing your metadata, to ensure the URLs are updated in the records accordingly: .. code-block:: bash python3 wis2box-ctl.py login - wis2box data add-collection ${WIS2BOX_DATADIR}/surface-weather-observations.yml - wis2box metadata discovery publish ${WIS2BOX_DATADIR}/surface-weather-observations.yml + wis2box data add-collection /data/wis2box/metadata/discovery/metadata-synop.yml + wis2box metadata discovery publish /data/wis2box/metadata/discovery/metadata-synop.yml diff --git a/docs/source/reference/administration.rst b/docs/source/reference/administration.rst index 5b4363f8..ab63a2e8 100644 --- a/docs/source/reference/administration.rst +++ b/docs/source/reference/administration.rst @@ -75,7 +75,7 @@ Changing default ports ^^^^^^^^^^^^^^^^^^^^^^ The ``docker-compose.override.yml`` file provides definitions on utilized ports. To change default -ports, edit ``default.env`` before stopping and starting wis2box for changes to take effect. +ports, edit ``wis2box.env`` before stopping and starting wis2box for changes to take effect. MQTT Quality of Service (QoS) diff --git a/docs/source/reference/auth.rst b/docs/source/reference/auth.rst index 2f43cc4e..336031a7 100644 --- a/docs/source/reference/auth.rst +++ b/docs/source/reference/auth.rst @@ -16,11 +16,21 @@ To add a token to the execution of a wis2box process, use the following command: .. code-block:: bash - wis2box auth add-token --path processes/wis2box myexecutiontoken + wis2box auth add-token --path processes/wis2box + +This will generate a random token that can be used to execute ``wis2box-synop2bufr`` and ``wis2box-csv2bufr``. + +.. note:: + + Be sure to record the token now, there is no way to retrieve it once it is lost. To add a token to PUT/POST/DELETE requests to the stations collection, use the following command: - wis2box auth add-token --path collections/stations mystationupdatetoken +.. code-block:: bash + + wis2box auth add-token --path collections/stations + +This will generate a random token that can be use to update the stations collection. Adding Access Control on topics ------------------------------- diff --git a/docs/source/reference/configuration.rst b/docs/source/reference/configuration.rst index 00ddb2a1..9e801932 100644 --- a/docs/source/reference/configuration.rst +++ b/docs/source/reference/configuration.rst @@ -9,19 +9,24 @@ a simple configuration that can be adjusted depending the user's needs and deplo Environment variables --------------------- -wis2box configuration is driven primarily by a small set of environment variables. The runtime -configuration is defined in the `Env`_ format in a plain text file named ``dev.env`` and ``default.env``. +wis2box configuration is driven primarily by a set of environment variables. The runtime +configuration is defined in the `Env`_ format in a plain text file named ``wis2box.env``. +An example is provided in ``wis2box.env.example``. -Any values set in ``dev.env`` override the default environment variables in ``default.env``. For further / specialized -configuration, see the sections below. +You can either copy the example-file to ``wis2box.env`` and adjust the values to your needs or run the following command +to create a new ``wis2box.env`` file by answering a few questions on the command line: + +.. code-block:: bash + + python3 wis2box-create-config.py + +For further / specialized configuration, see the sections below. ``WIS2BOX_HOST_DATADIR`` ^^^^^^^^^^^^^^^^^^^^^^^^ -The minimum required setting in ``dev.env`` is the ``WIS2BOX_HOST_DATADIR`` environment variable. Setting this -value is **required** to map the wis2box data directory from the host system to the containers. +The value of ``WIS2BOX_HOST_DATADIR`` maps the wis2box data directory from the host system to the containers. -It is recommended to set this value to an absolute path on your system. Sections -------- @@ -29,10 +34,7 @@ Sections .. note:: A reference configuration can always be found in the wis2box `GitHub`_ repository. The :ref:`quickstart` - uses a variant of ``wis2box.env`` with mappings to the test data, as an example. For complex installations, - it is recommended to start configuring wis2box by copying the example ``wis2box.env`` file and modifying - accordingly. - + uses a variant of ``wis2box.env.example`` with mappings to the test data, as an example. wis2box environment variables can be categorized via the following core sections: @@ -64,7 +66,7 @@ The following environment variables can be used to configure `WIS2BOX_STORAGE`. WIS2BOX_STORAGE_TYPE=S3 WIS2BOX_STORAGE_SOURCE=http://minio:9000 - WIS2BOX_STORAGE_USERNAME=minio # username for the storage-layer + WIS2BOX_STORAGE_USERNAME=wis2box # username for the storage-layer WIS2BOX_STORAGE_PASSWORD=minio123 # password for the storage-layer WIS2BOX_STORAGE_INCOMING=wis2box-incoming # name of the storage-bucket/folder for incoming files WIS2BOX_STORAGE_PUBLIC=wis2box-public # name of the storage-bucket/folder for public files @@ -178,7 +180,7 @@ Web application configuration provides the ability to customize web components. Other ^^^^^ -Additional directives provide various configurationscontrol of configuration options for the deployment of wis2box. +Additional directives provide various configuration options for the deployment of wis2box. .. code-block:: bash @@ -193,16 +195,13 @@ Additional directives provide various configurationscontrol of configuration opt A full configuration example can be found below: -.. literalinclude:: ../../../examples/config/wis2box.env - :language: bash - -.. literalinclude:: ../../../default.env +.. literalinclude:: ../../../wis2box.env.example :language: bash Docker Compose -------------- -The Docker Compose setup is driven from the resulting ``dev.env`` file created. For advanced cases and/or power users, +The Docker Compose setup is driven from the resulting ``wis2box.env`` file created. For advanced cases and/or power users, updates can also be made to ``docker-compose.yml`` or ``docker-compose.override.yml`` (for changes to ports). .. _`Env`: https://en.wikipedia.org/wiki/Env diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index 6b8a7ea4..b610f149 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -17,11 +17,11 @@ To download wis2box from source: The test enviroment file is provided in ``tests/test.env``. -To run with the 'quickstart' configuration, copy this file to ``dev.env`` in your working directory: +To run with the 'quickstart' configuration, copy this file to ``wis2box.env`` in your working directory: .. code-block:: bash - cp tests/test.env dev.env + cp tests/test.env wis2box.env Build and update wis2box: diff --git a/docs/source/reference/running/environment.rst b/docs/source/reference/running/environment.rst index b26ca39c..dd686511 100644 --- a/docs/source/reference/running/environment.rst +++ b/docs/source/reference/running/environment.rst @@ -15,5 +15,5 @@ directory for all data managed in wis2box. The default enviroment variables are below. -.. literalinclude:: ../../../../default.env +.. literalinclude:: ../../../../wis2box.env.example :language: bash diff --git a/docs/source/user/data-ingest.rst b/docs/source/user/data-ingest.rst index 997ab194..604d4016 100644 --- a/docs/source/user/data-ingest.rst +++ b/docs/source/user/data-ingest.rst @@ -10,10 +10,31 @@ The wis2box storage is provided using a `MinIO`_ container that provides S3-comp Any file received in the ``wis2box-incoming`` storage bucket will trigger an action to process the file. What action to take is determined by the ``data-mappings.yml`` you've setup in the previous section. +wis2box-webapp +-------------- + +The wis2box-webapp is a web application that includes the following forms for data validation and ingestion: + +* user interface to ingest SYNOP data +* user interface to ingest CSV data + +The wis2box-webapp is available on your host at `http:///wis2box-webapp`. + +Interactive data ingestion requires an execution token, which can be generated using the ``wis2box auth add-token`` command inside the wis2box-management container: + +.. code-block:: bash + + python3 wis2box-ctl.py login + wis2box auth add-token --path processes/wis2box + +.. note:: + + Be sure to record the token value, as it will not be shown again. If you lose the token, you can generate a new one. + MinIO user interface -------------------- -To access the MinIO user interface, visit ``http://localhost:9001`` in your web browser. +To access the MinIO user interface, visit ``http://:9001`` in your web browser. You can login with your ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD``: @@ -21,6 +42,10 @@ You can login with your ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSW :width: 400 :alt: MinIO login screen +.. note:: + + The ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` are defined in the ``wis2box.env`` file. + To test the data ingest, add a sample file for your observations in the ``wis2box-incoming`` storage bucket. Select 'browse' on the ``wis2box-incoming`` bucket and select 'Choose or create a new path' to define a new folder path: @@ -30,11 +55,16 @@ Select 'browse' on the ``wis2box-incoming`` bucket and select 'Choose or create :alt: MinIO new folder path .. note:: - The folder in which the file is placed defines the dataset for the data you are sharing. For example, for dataset ``foo.bar``, store your file in the path ``/foo/bar/``. + The folder in which the file is placed defines the topic you want to share the data on and should match the datasets defined in ``data-mappings.yml`` - The path is also used to define the topic hierarchy for your data (see `WIS2 topic hierarchy`_). The first 3 levels of the WIS2 topic hierarchy ``origin/a/wis2`` are automatically included by wis2box when publishing data notification messages. + The first 3 levels of the WIS2 topic hierarchy ``origin/a/wis2`` are automatically included by wis2box when publishing data notification messages. - * The error message ``Topic Hierarchy validation error: No plugins for minio:9000/wis2box-incoming/... in data mappings`` indicates you stored a file in a folder for which no matching dataset was defined in your ``data-mappings.yml``. + For example: + + * data to be published on: ``origin/a/wis2/cog/brazza_met_centre/data/core/weather/surface-based-observations/synop`` + * upload data in the path: ``cog/brazza_met_centre/data/core/weather/surface-based-observations/synop``. + + The error message ``Topic Hierarchy validation error: No plugins for minio:9000/wis2box-incoming/... in data mappings`` indicates you stored a file in a folder for which no matching dataset was defined in ``data-mappings.yml``. After uploading a file to ``wis2box-incoming`` storage bucket, you can browse the content in the ``wis2box-public`` bucket. If the data ingest was successful, new data will appear as follows: @@ -48,7 +78,7 @@ If no data appears in the ``wis2box-public`` storage bucket, you can inspect the python3 wis2box-ctl.py logs wis2box -Or by visiting the local Grafana instance running at ``http://localhost:3000`` +Or by visiting the local Grafana instance running at ``http://:3000`` wis2box workflow monitoring --------------------------- @@ -81,7 +111,7 @@ See below a Python example to upload data using the MinIO package: minio_path = '/ita/italy_wmo_demo/data/core/weather/surface-based-observations/synop/' endpoint = 'http://localhost:9000' - WIS2BOX_STORAGE_USERNAME = '' + WIS2BOX_STORAGE_USERNAME = 'wis2box' WIS2BOX_STORAGE_PASSWORD = '' client = Minio( @@ -93,6 +123,12 @@ See below a Python example to upload data using the MinIO package: filename = filepath.split('/')[-1] client.fput_object('wis2box-incoming', minio_path+filename, filepath) +.. note:: + + In the example the file ``mydata.bin`` in ingested from the directory ``/home/wis2box-user/local-data/`` on the host running wis2box. + If you are running the script on the same host as wis2box, you can use the endpoint ``http://localhost:9000`` as in the example. + Otherwise, replace localhost with the IP address of the host running wis2box. + wis2box-ftp ----------- @@ -109,12 +145,12 @@ To use the ``docker-compose.wis2box-ftp.yml`` template included in wis2box, crea FTP_HOST=${MYHOSTNAME} WIS2BOX_STORAGE_ENDPOINT=http://${MYHOSTNAME}:9000 - WIS2BOX_STORAGE_USER=minio - WIS2BOX_STORAGE_PASSWORD=minio123 + WIS2BOX_STORAGE_USER=wis2box + WIS2BOX_STORAGE_PASSWORD=XXXXXXXX LOGGING_LEVEL=INFO -and ensure ``MYHOSTNAME`` is set to **your** hostname (fully qualified domain name). +ensure ``MYHOSTNAME`` is set to **your** hostname (fully qualified domain name) and ``WIS2BOX_STORAGE_PASSWORD`` is set to **your** MinIO password. Then start the ``wis2box-ftp`` service with the following command: @@ -135,11 +171,15 @@ See the GitHub repository `wis2box-ftp`_ for more information on this service. wis2box-data-subscriber ----------------------- -You can add an additional service on the host running your wis2box instance to allow data to be ingested by publishing an MQTT message to the wis2box broker. +.. note:: + + This service currently only works with Campbell scientific data loggers version CR1000X. + +You can add an additional service on the host running your wis2box instance to allow data to be received over MQTT. This service subscribes to the topic ``data-incoming/#`` on the wis2box broker and parses the content of received messages and publishes the result in the ``wis2box-incoming`` bucket. -To start the ``wis2box-data-subscriber``, add the following additional variables to ``dev.env``: +To start the ``wis2box-data-subscriber``, add the following additional variables to ``wis2box.env``: .. code-block:: bash @@ -154,7 +194,7 @@ You then you can activate the optional 'wis2box-data-subscriber' service as foll .. code-block:: bash - docker compose -f docker-compose.data-subscriber.yml --env-file dev.env up -d + docker compose -f docker-compose.data-subscriber.yml --env-file wis2box.env up -d See the GitHub `wis2box-data-subscriber`_ repository for more information on this service. diff --git a/docs/source/user/getting-started.rst b/docs/source/user/getting-started.rst index 941b996c..e6f3d1e1 100644 --- a/docs/source/user/getting-started.rst +++ b/docs/source/user/getting-started.rst @@ -10,6 +10,24 @@ The recommended OS is Ubuntu 20.04 LTS. wis2box may work on other operating systems (for example AlmaLinux), but the officially supported OS is Ubuntu. +Network requirements +-------------------- + +The wis2box-software requires the following network ports to be available on the host system: + +* 80/tcp (HTTP) +* 1883/tcp (MQTT) +* 3000/tcp (Grafana) +* 9000/tcp (MinIO) +* 9001/tcp (MinIO Console) + +In order for the wis2box to be accessible from the Internet, the following ports on the host should be routed to a public IP address: + +* 80/tcp (HTTP) +* 1883/tcp (MQTT) + +It is recommended to use a reverse proxy (for example `NGINX`_) to provide HTTPS access to the wis2box. + System requirements ------------------- @@ -39,7 +57,11 @@ wis2box requires the following prior to installation: Docker Engine, 20.10.14 or higher Docker Compose, 2.0 or higher -The following commands be used to setup the required software on Ubuntu 20.04 LTS: +The following commands be used to setup the required software on Ubuntu (20.04 LTS, 22.04 LTS) systems: + +.. note:: + + Execute the below commands one by one, and do not copy / paste the entire block .. code-block:: bash @@ -61,7 +83,40 @@ The following commands can be used to inspect the available versions of Python, docker compose version python3 -V +The wis2box software should be run by system user that is part of the ``docker`` group. +The following command can be used to add the current user to the ``docker`` group: + +.. code-block:: bash + + sudo usermod -aG docker $USER + +Switch to this user and check that you can run docker hello-world: + +.. code-block:: bash + + sudo su - $USER + docker run hello-world + +You should see the following output: + +.. code-block:: bash + + Hello from Docker! + This message shows that your installation appears to be working correctly. + + To generate this message, Docker took the following steps: + 1. The Docker client contacted the Docker daemon. + 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. + (amd64) + 3. The Docker daemon created a new container from that image which runs the + executable that produces the output you are currently reading. + 4. The Docker daemon streamed that output to the Docker client, which sent it + to your terminal. + + (...) + Once you have verified these requirements, go to :ref:`setup` for a step-by-step guide to install and configure your wis2box. .. _`Docker`: https://docs.docker.com/get-started/overview .. _`Docker Compose`: https://github.com/docker/compose/releases +.. _`NGINX`: https://nginx.org diff --git a/docs/source/user/introduction.rst b/docs/source/user/introduction.rst index c4f64c91..33d02e91 100644 --- a/docs/source/user/introduction.rst +++ b/docs/source/user/introduction.rst @@ -13,16 +13,18 @@ wis2box implements the core WIS2 requirements of a WIS2 Node: * MQTT broker * HTTP endpoint to enable data download -Additional services included in wis2box include: +Additional services included in wis2box: -* Customizable plugins to transform input data -* API exposing data in GeoJSON using `pygeoapi`_ +* Customizable plugins to transform input data (default plugins for synop2bufr, csv2bufr and bufr2geojson) +* API using `pygeoapi`_ interaction with wis2box data and metadata via `OGC API`_. +* Web-based administration/configuration of station metadata and SYNOP / CSV data publication * Monitoring functions using `Prometheus`_ and `Grafana`_ -* Data visualization through the wis2box user interface +* Data visualization through wis2box user interface Next: :ref:`getting-started`. .. _`WIS2`: https://community.wmo.int/activity-areas/wis/wis2-implementation .. _`pygeoapi`: https://pygeoapi.io +.. _`OGC API`: https://ogcapi.ogc.org .. _`Prometheus`: https://prometheus.io/docs/introduction/overview .. _`Grafana`: https://grafana.com/docs/grafana/latest/introduction diff --git a/docs/source/user/public-services-setup.rst b/docs/source/user/public-services-setup.rst index d4f6eb90..a31d7224 100644 --- a/docs/source/user/public-services-setup.rst +++ b/docs/source/user/public-services-setup.rst @@ -5,24 +5,13 @@ Public services setup To share your data with the WIS2 network, you need to expose some of your wis2box services to the Global Services: -* The Global Cache needs to be able to access to your HTTP endpoint to download data published by your wis2box instance -* The Global Broker needs to be able to subscribe to your MQTT endpoint to receive WIS2 notifications published by your wis2box instance +* The Global Cache needs to be able to access to your HTTP endpoint at port 80 to download data published by your wis2box instance +* The Global Broker needs to be able to subscribe to your MQTT endpoint at port 1883 to receive WIS2 notifications published by your wis2box instance SSL ^^^ -To enable HTTPS and MQTTS on your wis2box you can run wis2box with the option `--ssl`: - -.. code-block:: bash - - python3 wis2box-ctl.py --ssl start - -When running wis2box with SSL, you have to set additional environment variables in your dev.env defining the location of your SSL certificate and private key: - -.. code-block:: bash - - WIS2BOX_SSL_CERT=/etc/letsencrypt/live/example.wis2box.io/fullchain.pem - WIS2BOX_SSL_KEY=/etc/letsencrypt/live/example.wis2box.io/privkey.pem +It is recommended to use a reverse proxy to route HTTP and MQTT traffic from/to your wis2box, and to enable TLS (HTTPS/MQTTS) on your wis2box. Please remember to update the ``WIS2BOX_URL`` and ``WIS2BOX_API_URL``environment variable after enabling SSL, ensuring your URL starts with ``https://``. @@ -31,15 +20,15 @@ Please note that after changing the ``WIS2BOX_URL`` and ``WIS2BOX_API_URL`` envi .. code-block:: bash python3 wis2box-ctl.py stop - python3 wis2box-ctl.py --ssl start + python3 wis2box-ctl.py start After restarting wis2box, repeat the commands for adding your dataset and publishing your metadata, to ensure the URLs are updated accordingly: .. code-block:: bash python3 wis2box-ctl.py login - wis2box data add-collection ${WIS2BOX_HOST_DATADIR}/surface-weather-observations.yml - wis2box metadata discovery publish ${WIS2BOX_HOST_DATADIR}/surface-weather-observations.yml + wis2box data add-collection /data/wis2box/metadata/discovery/metadata-synop.yml + wis2box metadata discovery publish /data/wis2box/metadata/discovery/metadata-synop.yml Nginx (HTTP) ^^^^^^^^^^^^ @@ -55,7 +44,6 @@ wis2box runs a local nginx container allowing access to the following HTTP based Storage (incoming data) (minio:wis2box-incoming),`WIS2BOX_URL/wis2box-incoming` Storage (public data) (minio:wis2box-public),`WIS2BOX_URL/data` - You can edit ``nginx/nginx.conf`` to control which services are exposed through the nginx-container include in your stack. You can edit ``docker-compose.override.yml`` to change the port on which the ``web-proxy`` service exposes HTTP on the localhost. @@ -64,7 +52,7 @@ You can edit ``docker-compose.override.yml`` to change the port on which the ``w The WIS2 notifications published by the wis2box includes the path ``/data/``. This path has to be publicly accessible by the client receiving the WIS2 notification over MQTT, or the data referenced cannot be downloaded -To share your data with the WIS2 network, ensure that ``WIS2BOX_URL`` as defined in ``dev.env`` points to the externally accessible URL for your HTTP services. +To share your data with the WIS2 network, ensure that ``WIS2BOX_URL`` as defined in ``wis2box.env`` points to the externally accessible URL for your HTTP services. After updating ``WIS2BOX_URL``, please stop and start your wis2box using ``wis2box-ctl.py`` and republish your data using the command ``wis2box metadata discovery publish``. @@ -116,19 +104,6 @@ To allow the WIS2 Global Broker to subscribe to WIS2 notifications from your wis Internal broker --------------- -The internal MQTT broker uses the default username/password of ``wis2box/wis2box``. Before opening the MQTT port for external access, it is recommended to set a unique password as follows: - -.. code-block:: bash - - WIS2BOX_BROKER_USERNAME=wis2box-utopia - WIS2BOX_BROKER_PASSWORD=myuniquepassword - WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883 - - # update minio settings after updating broker defaults - MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} - MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} - MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT} - The internal MQTT broker is accessible on the host ``mosquitto`` within the Docker network used by wis2box. By default port 1883 of the mosquitto container is mapped to port 1883 of the host running wis2box. @@ -153,14 +128,13 @@ If you do not wish to expose the internal MQTT broker on your wis2box, you can c The ``everyone`` user is defined by default for public readonly access (``origin/#``) as per WIS2 Node requirements. -Sharing data with the WIS2 Global Broker ----------------------------------------- +Registering your WIS2 Node +-------------------------- -The official procedure for a WIS2 Node to share data with the WIS2 network is currently in development. Contact wis at wmo.int for more information on connectivity with the WIS2 network. +Contact wis2-support@wmo.int for the procedure to register your WIS2 Node with the WIS2 network. Next: :ref:`downloading-data` - .. _`Mosquitto`: https://mosquitto.org/ .. _`pygeoapi`: https://pygeoapi.io/ .. _`Elasticsearch`: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html diff --git a/docs/source/user/setup.rst b/docs/source/user/setup.rst index cf876c60..8f0078df 100644 --- a/docs/source/user/setup.rst +++ b/docs/source/user/setup.rst @@ -32,45 +32,24 @@ Run the following command to create the initial configuration files for your wis .. note:: - The ``wis2box-create-config.py`` program will ask for a directory to store the configuration files. + The ``wis2box-create-config.py`` script will ask for a directory to store the configuration files. + Please provide the **absolute** path to the directory where you want to store the configuration files, for example ``/home/wis2box-user/wis2box-data``. This directory will be mapped to ``/data/wis2box`` **inside** the wis2box-management container. + The script will also ask for the URL of your wis2box. Please provide the public URL of your wis2box, for example ``http://mywis2box.example.com``. + For testing purpose you can also provide the internal IP address you use to access the host, for example ``http://192.168.0.3`` and you change the URL in configuration files at a later point in time. -Adding your own station data ----------------------------- + The script will propose to automatically create passwords for ``WIS2BOX_STORAGE_PASSWORD`` and ``WIS2BOX_BROKER_PASSWORD``. + These passwords are for internal use only within the wis2box, and it is recommended to accept the randomly generated passwords. -wis2box requires information about the stations for which you will be sharing data. - -An example of the configuration file for the stations is provided in ``station_list.csv``. - -You can copy this file to ``metadata/station/station_list.csv`` in your $WIS2BOX_HOST_DATADIR: - -.. code-block:: bash - - mkdir -p ~/wis2box-data/metadata/station - cp examples/config/station_list.csv ~/wis2box-data/metadata/station - -And edit ``~/wis2box-data/metadata/station/station_list.csv`` to include the data for your stations. - -The 'wis2box-create-config.py' script will create the file ``metadata/station/station_list.csv`` file in the directory - you specified for your configuration files. You will have to edit this file to add your own station data using the examples provided. + The script will ask for 3-letter ISO country code for your wis2box. Please provide the 3-letter ISO country code for your country, for example ``FRA``. + It will also ask for a centre-id. Please provide a string that identifies your organization and does not use spaces or special characters, for example ``meteofrance``. .. note:: - The ``station_list.csv`` requires column names ``station_name`` and the ``wigos_station_identifier`` (WSI) with which the station is registered in `OSCAR`_. Optionally, you can provide a ``traditional_station_identifier (TSI)`` column. - The TSI can be left empty if your data contains a WSI. If your data contains a TSI but no WSI, the ``station_list.csv`` will be used to derive the corresponding WSI for that station. + In alignment with WMO WIS2 Guidance, the above fields are automatically converted to lowercase when saved in wis2box. -To verify station metadata from OSCAR/Surface, run the following command: - -.. code-block:: bash - - wis2box metadata station get - -Then to add to the station list: - -.. code-block:: bash - - wis2box metadata station get >> ~/wis2box-data/metadata/station/station_list.csv + The remaining questions will be used in the creation the discovery metadata files for the ``synop`` and ``temp`` datasets. Discovery metadata ------------------ @@ -198,25 +177,27 @@ The second step is to publish discovery metadata and cache its content in the wi This command publishes an MQTT message with information about your dataset to the WIS2 Global Discovery Catalogue. Repeat this command whenever you have to provide updated metadata about your dataset. -You can review the discovery metadata just cached through the new link at ``/oapi/collections``: - -.. image:: ../_static/wis2box-api-discovery-metadata.png - :width: 800 - :alt: wis2box API collections list with added discovery metadata +You can review the discovery metadata just cached through the new item at ``/oapi/collections/discovery-metadata/items``: Repeat this step for any other discovery metadata you wish to publish, such as the ``temp`` dataset. -The final step is to publish your station information to the wis2box API from the station metadata list you prepared: +Finally it is recommended to prepare authentication tokens for updating your stations and ingesting data using the wis2box-webapp. + +To create a token for ingesting data: .. code-block:: bash - wis2box metadata station publish-collection + wis2box auth add-token --path processes/wis2box -You can review the stations you just cached through the new link at ``/oapi/collections``: +Record the token value displayed in a safe place, you will need for the :ref:`data-ingest`. -.. image:: ../_static/wis2box-api-stations.png - :width: 800 - :alt: wis2box API collections list with added stations +To create a token for updating stations: + +.. code-block:: bash + + wis2box auth add-token --path collections/stations + +Record the token value displayed in the output of the command above. You will use this token to update stations in the next section. You can now logout of wis2box-management container: @@ -224,6 +205,26 @@ You can now logout of wis2box-management container: exit +Adding station metadata +----------------------- + +The next step is to add station metadata to your wis2box. This can be done interactively in the wis2box-webapp UI on the 'stations' page. + +The wis2box-webapp can be accessed by visiting the URL you specified during the configuration step, and adding ``/wis2box-webapp`` to the URL. + +.. image:: ../_static/wis2box-webapp-stations.png + :width: 800 + :alt: wis2box webapp stations page + +Please note only data for stations that have been added to the wis2box will be ingested and result in WIS2 notifications being published. + +You can also bulk insert a set of stations from a CSV file, by defining the stations in ``metadata/stations/station_list.csv`` in your wis2box host directory and running the following command: + +.. code-block:: bash + + python3 wis2box-ctl.py login + wis2box metadata stations publish-collections + The next is the :ref:`data-ingest`. .. _`wis2box Releases`: https://github.com/wmo-im/wis2box/releases diff --git a/examples/config/synop-csv-mappings.yml b/examples/config/synop-csv-mappings.yml index e8cb096c..816d2eb0 100644 --- a/examples/config/synop-csv-mappings.yml +++ b/examples/config/synop-csv-mappings.yml @@ -3,7 +3,7 @@ data: plugins: csv: - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: synop_bufr.json + template: aws-template notify: true file-pattern: '^.*\.csv$' bufr4: diff --git a/examples/config/wis2box.env b/examples/config/wis2box.env deleted file mode 100644 index c9391554..00000000 --- a/examples/config/wis2box.env +++ /dev/null @@ -1,30 +0,0 @@ -# please define a data-directory on your host machine -# this will map to /data/wis2box on the wis2box-container -WIS2BOX_HOST_DATADIR=/home/example/wis2box-data - -# replace localhost with the IP or public address for your instance -# replace http with https when using SSL certificates -WIS2BOX_URL=http://localhost -WIS2BOX_API_URL=http://localhost/oapi - -# logging and data retention -WIS2BOX_LOGGING_LOGLEVEL=INFO -WIS2BOX_DATA_RETENTION_DAYS=30 - -# update broker default credentials -WIS2BOX_BROKER_USERNAME=wis2box -WIS2BOX_BROKER_PASSWORD=XXXXXXXXX - -WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883 - -# update storage default credentials -# username should be 3 or more characters -WIS2BOX_STORAGE_USERNAME=wis2box -# password should be 8 or more characters -WIS2BOX_STORAGE_PASSWORD=XXXXXXXXX - -# update minio settings after updating storage and broker defaults -MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME} -MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD} -MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} -MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} diff --git a/examples/config/wis2box.extended.dev b/examples/config/wis2box.extended.dev deleted file mode 100644 index 97c27fad..00000000 --- a/examples/config/wis2box.extended.dev +++ /dev/null @@ -1,33 +0,0 @@ -# please define a data directory on your host machine -# this will map to /data/wis2box on the wis2box container -WIS2BOX_HOST_DATADIR=/path/to/local/data/directory - -# Optional -# Environment variable overrides -WIS2BOX_LOGGING_LOGLEVEL=WARNING -WIS2BOX_DATA_RETENTION_DAYS=30 - -# define the URL used by the wis2box-ui and wis2box-api -WIS2BOX_URL=http://localhost -WIS2BOX_API_URL=${WIS2BOX_URL}/oapi - -# Pub/Sub local broker setup for internal commmunication -# please provide your own unique password -WIS2BOX_BROKER_USERNAME=wis2box -WIS2BOX_BROKER_PASSWORD=wis2box - -# use local broker also for public message publishing -# when using external broker, please replace this with the connection string for your external broker -WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883 - -# setup the wis2box storage username password -# please provide your own unique password -WIS2BOX_STORAGE_USERNAME=minio -WIS2BOX_STORAGE_PASSWORD=minio123 -WIS2BOX_STORAGE_DATA_RETENTION_DAYS=30 - -# use minio for storage -MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME} -MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD} -MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} -MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} diff --git a/examples/scripts/copy_to_incoming.py b/examples/scripts/copy_to_incoming.py index 1e5e3e6c..68853e3a 100644 --- a/examples/scripts/copy_to_incoming.py +++ b/examples/scripts/copy_to_incoming.py @@ -36,8 +36,9 @@ print(f'No files found for pattern={local_path}') S3_ENDPOINT = 'http://localhost:9000' -S3_USER = 'minio' -S3_PASSWORD = 'minio123' +S3_USER = 'wis2box' +# replace with the value of WIS2BOX_STORAGE_PASSWORD in your wis2box.env file +S3_PASSWORD = 'XXXXXXXX' S3_BUCKET_INCOMING = 'wis2box-incoming' if S3_ENDPOINT != '': diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf index 2d5b3bc1..aa8b637d 100644 --- a/nginx/nginx-ssl.conf +++ b/nginx/nginx-ssl.conf @@ -60,10 +60,14 @@ proxy_pass http://minio:9000; } location /oapi { + set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; proxy_pass http://wis2box-api:80; } + location /wis2box-webapp/ { + proxy_pass http://wis2box-webapp:4173/wis2box-webapp/; + } location / { proxy_pass http://wis2box-ui:80; } diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 2535d398..525a2513 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -62,8 +62,8 @@ auth_request_set $auth_status $upstream_status; proxy_pass http://wis2box-api:80; } - location / { - proxy_pass http://wis2box-ui:80; + location /wis2box-webapp/ { + proxy_pass http://wis2box-webapp:4173/wis2box-webapp/; } # location /admin/ { # proxy_pass http://wis2box-ui-admin:80/; @@ -77,4 +77,7 @@ proxy_set_header Authorization $http_authorization; proxy_pass_header Authorization; } - } + location / { + proxy_pass http://wis2box-ui:80; + } + } \ No newline at end of file diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index d55da050..4994cf94 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -195,7 +195,7 @@ def test_message_api(): counts = { 'mwi_met_centre': 24, 'roma_met_centre': 33, - 'alger_met_centre': 29, + 'alger_met_centre': 28, 'rnimh': 188, 'brazza_met_centre': 15 } @@ -208,7 +208,7 @@ def test_message_api(): r = SESSION.get(url).json() # should match sum of counts above - assert r['numberMatched'] == 289 + assert r['numberMatched'] == 288 msg = r['features'][5] is_valid, _ = validate_message(msg) diff --git a/tests/test.env b/tests/test.env index 50ede1a8..f7127e93 100644 --- a/tests/test.env +++ b/tests/test.env @@ -2,7 +2,56 @@ # Host machine data directory path WIS2BOX_HOST_DATADIR=${PWD}/tests/data -# Optional -# Environment variable overrides -WIS2BOX_LOGGING_LOGLEVEL=INFO -WIS2BOX_DATA_RETENTION_DAYS=30 +# directory in the wis2box container with wis2box-configuration +WIS2BOX_DATADIR=/data/wis2box + +# wis2box public URL +WIS2BOX_URL=http://localhost + +# api +WIS2BOX_API_TYPE=pygeoapi +WIS2BOX_API_URL=http://localhost/oapi +WIS2BOX_DOCKER_API_URL=http://wis2box-api:80/oapi + +# backend +WIS2BOX_API_BACKEND_TYPE=Elasticsearch +WIS2BOX_API_BACKEND_URL=http://elasticsearch:9200 + +# logging +WIS2BOX_LOGGING_LOGLEVEL=ERROR +WIS2BOX_LOGGING_LOGFILE=stdout + +# map settings for wis2box-ui, wis2box-api and wis2box-webapp +WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png +WIS2BOX_BASEMAP_ATTRIBUTION=OpenStreetMap contributors + +# storage, default is S3 provided by minio +WIS2BOX_STORAGE_TYPE=S3 +WIS2BOX_STORAGE_SOURCE=http://minio:9000 +WIS2BOX_STORAGE_INCOMING=wis2box-incoming +WIS2BOX_STORAGE_ARCHIVE=wis2box-archive +WIS2BOX_STORAGE_PUBLIC=wis2box-public +WIS2BOX_STORAGE_DATA_RETENTION_DAYS=7 +WIS2BOX_STORAGE_USERNAME=wis2box +WIS2BOX_STORAGE_PASSWORD=minio123 + +# broker settings +WIS2BOX_BROKER_PORT=1883 +WIS2BOX_BROKER_HOST=mosquitto +WIS2BOX_BROKER_USERNAME=wis2box +WIS2BOX_BROKER_QUEUE_MAX=1000 +WIS2BOX_BROKER_PASSWORD=wis2box + +# update WIS2BOX_PUBLIC_BROKER settings after updating broker defaults +WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883 + +# minio settings +MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME} +MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD} +MINIO_PROMETHEUS_AUTH_TYPE=public +MINIO_NOTIFY_MQTT_ENABLE_WIS2BOX=on +MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} +MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} +MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT} +MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage +MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 \ No newline at end of file diff --git a/wis2box-broker/acl.conf b/wis2box-broker/acl.conf index f7a82e5a..64f9e36b 100644 --- a/wis2box-broker/acl.conf +++ b/wis2box-broker/acl.conf @@ -4,6 +4,5 @@ topic read origin/# user _WIS2BOX_BROKER_USERNAME topic readwrite origin/# topic readwrite wis2box/# -topic readwrite wis2box-storage/# topic readwrite data-incoming/# topic read $SYS/# \ No newline at end of file diff --git a/wis2box-create-config.py b/wis2box-create-config.py index 5fbaf65e..b4142852 100644 --- a/wis2box-create-config.py +++ b/wis2box-create-config.py @@ -46,7 +46,7 @@ def get_bounding_box(country_code: str) -> Tuple[str, str]: country_name = 'NA' bounding_box = [-180, -90, 180, 90] - print(f"Getting bounding box for '{country_code}'.") + print(f'Getting bounding box for "{country_code}".') # get the path to the data data_path = Path(__file__).parent / 'config-templates' / 'countries.json' @@ -57,38 +57,38 @@ def get_bounding_box(country_code: str) -> Tuple[str, str]: data = json.load(fh) # get the bounding box for the country if country_code in data['countries'] and 'bbox' in data['countries'][country_code]: # noqa - country_name = data['countries'][country_code]["name"] - bbox = data['countries'][country_code]["bbox"] + country_name = data['countries'][country_code]['name'] + bbox = data['countries'][country_code]['bbox'] if not {'minx', 'miny', 'maxx', 'maxy'} <= bbox.keys(): - print(f"Bounding box for '{country_code}' is invalid.") - print("Using global bounding box.") + print(f'Bounding box for "{country_code}" is invalid.') + print('Using global bounding box.') else: minx = bbox['minx'] miny = bbox['miny'] maxx = bbox['maxx'] maxy = bbox['maxy'] # create bounding box as a CSV of four numbers - bounding_box = f"{minx},{miny},{maxx},{maxy}" + bounding_box = f'{minx},{miny},{maxx},{maxy}' else: - print(f"No bounding box found for '{country_code}'.") - print("Using the bounding box for the whole world.") + print(f'No bounding box found for "{country_code}".') + print('Using the bounding box for the whole world.') # ask the user to accept the bounding box or to enter a new one - print(f"bounding box: {bounding_box}.") - print("Do you want to use this bounding box? (y/n/exit)") + print(f'bounding box: {bounding_box}.') + print('Do you want to use this bounding box? (y/n/exit)') answer = input() while answer not in ['y', 'exit']: - print("Please enter the bounding box as a comma-separated list of four numbers:") # noqa - print("The first two numbers are the coordinates of the lower left corner of the bounding box.") # noqa - print("The last two numbers are the coordinates of the upper right corner of the bounding box.") # noqa - print("For example: 5.5,47.2,15.5,55.2") + print('Please enter the bounding box as a comma-separated list of four numbers:') # noqa + print('The first two numbers are the coordinates of the lower left corner of the bounding box.') # noqa + print('The last two numbers are the coordinates of the upper right corner of the bounding box.') # noqa + print('For example: 5.5,47.2,15.5,55.2') bounding_box = input() - print(f"bounding box: {bounding_box}.") - print("Do you want to use this bounding box? (y/n/exit)") + print(f'bounding box: {bounding_box}.') + print('Do you want to use this bounding box? (y/n/exit)') answer = input() - if answer == "exit": + if answer == 'exit': exit() return country_name, bounding_box @@ -102,42 +102,42 @@ def get_country_and_centre_id() -> Tuple[str, str]: :returns: `tuple` of (country_code, centre_id) """ - answer = "" + answer = '' - while answer != "y": - if answer == "exit": + while answer != 'y': + if answer == 'exit': exit() - print("Please enter your 3-letter ISO country code:") + print('Please enter your 3-letter ISO country code:') country_code = input() # check that the input is a 3-letter string # if not repeat the question while len(country_code) != 3: - print("The country code must be a 3-letter string.") - print("Please enter your 3-letter ISO country code:") + print('The country code must be a 3-letter string.') + print('Please enter your 3-letter ISO country code:') country_code = input() # make sure the country code is lowercase country_code = country_code.lower() - print("Please enter the centre-id for your wis2box:") + print('Please enter the centre-id for your wis2box:') centre_id = str(input()).lower() # check that the input is valid # if not repeat the question while any([x in centre_id for x in ['#', '+', ' ']]) or len(centre_id) < 3: # noqa - print("The centre-id cannot contain spaces or the '+' or '#' characters, and must be at least 3 characters long.") # noqa - print("Please enter the string identifying the centre hosting the wis2box:") # noqa + print('The centre-id cannot contain spaces or the "+" or "#" characters, and must be at least 3 characters long.') # noqa + print('Please enter the string identifying the centre hosting the wis2box:') # noqa centre_id = str(input()).lower() # ask the user to confirm their choice and give them the option to change it # noqa # and give them the option to exit the script - print("The country-code will be set to:") - print(f" {country_code}") - print("The centre-id will be set to:") - print(f" {centre_id}") - print("Is this correct? (y/n/exit)") + print('The country-code will be set to:') + print(f' {country_code}') + print('The centre-id will be set to:') + print(f' {centre_id}') + print('Is this correct? (y/n/exit)') answer = input() return (country_code, centre_id) @@ -154,34 +154,34 @@ def get_password(password_name: str) -> str: password = None - answer = "" + answer = '' while answer not in ['y', 'n']: - if answer == "exit": + if answer == 'exit': exit() - print(f"Do you want to use a randomly generated password for {password_name} (y/n/exit)") # noqa + print(f'Do you want to use a randomly generated password for {password_name} (y/n/exit)') # noqa answer = input() - if answer == "y": + if answer == 'y': password = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(8)) # noqa - print(f"{password_name}={password}") + print(f'{password_name}={password}') - while answer != "y": - if answer == "exit": + while answer != 'y': + if answer == 'exit': exit() - print("Please enter the password to be used for the WIS2BOX_STORAGE_PASSWORD:") # noqa + print('Please enter the password to be used for the WIS2BOX_STORAGE_PASSWORD:') # noqa password = input() # check if the password is at least 8 characters long # if not repeat the question while len(password) < 8: - print("The password must be at least 8 characters long.") - print(f"Please enter the password to be used for the {password_name}:") # noqa + print('The password must be at least 8 characters long.') + print(f'Please enter the password to be used for the {password_name}:') # noqa password = input() - print(f"{password_name}={password}") - print("Is this correct? (y/n/exit)") + print(f'{password_name}={password}') + print('Is this correct? (y/n/exit)') answer = input() return f"{password_name}={password}\n" @@ -195,82 +195,117 @@ def get_wis2box_url() -> str: """ wis2box_url = None - answer = "" + answer = '' - while answer != "y": - if answer == "exit": + while answer != 'y': + if answer == 'exit': exit() # ask for the WIS2BOX_URL, use http://localhost as the default - print("Please enter the URL of the wis2box:") - print(" When running the wis2box locally, the default is http://localhost") # noqa - print(" To enable remote access, please enter the public IP address or domain name of the server hosting the wis2box.") # noqa + print('Please enter the URL of the wis2box:') + print(' For local testing the URL is http://localhost') # noqa + print(' To enable remote access, the URL should point to the public IP address or domain name of the server hosting the wis2box.') # noqa # check if the URL starts with http:// or https:// # if not, ask the user to enter the URL again - wis2box_url = "" + wis2box_url = '' wis2box_url = input() - while not wis2box_url.startswith(("http://", "https://")): - print("The URL must start with http:// or https://") - print("Please enter the URL of the wis2box:") + while not wis2box_url.startswith(('http://', 'https://')): + print('The URL must start with http:// or https://') + print('Please enter the URL of the wis2box:') wis2box_url = input() # ask the user to confirm their choice and give them the option to change it # noqa - print("The URL of the wis2box will be set to:") - print(f" {wis2box_url}") - print("Is this correct? (y/n/exit)") + print('The URL of the wis2box will be set to:') + print(f' {wis2box_url}') + print('Is this correct? (y/n/exit)') answer = input() return wis2box_url -def create_dev_env(config_dir: str) -> None: +def create_wis2box_env(config_dir: str) -> None: """ - creates the dev.env file in the config_dir + creates the wis2box.env file in the config_dir :param config_dir: `str` of path to the config directory :returns: None """ - dev_env = Path("dev.env") + wis2box_env = Path('wis2box.env') - with dev_env.open("w") as fh: - fh.write(f"WIS2BOX_HOST_DATADIR={config_dir}\n") - fh.write("\n") + with wis2box_env.open('w') as fh: + fh.write('# directory on the host with wis2box-configuration\n') # noqa + fh.write(f'WIS2BOX_HOST_DATADIR={config_dir}\n') + fh.write(f'# directory in the wis2box container with wis2box-configuration\n') # noqa + fh.write('WIS2BOX_DATADIR=/data/wis2box\n') + fh.write('\n') wis2box_url = get_wis2box_url() - fh.write(f"WIS2BOX_URL={wis2box_url}\n") - fh.write(f"WIS2BOX_API_URL={wis2box_url}/oapi\n") - fh.write("\n") + fh.write('# wis2box public URL\n') + fh.write(f'WIS2BOX_URL={wis2box_url}\n') + fh.write('\n') + fh.write('# api\n') + fh.write('WIS2BOX_API_TYPE=pygeoapi\n') + fh.write(f'WIS2BOX_API_URL={wis2box_url}/oapi\n') + fh.write('WIS2BOX_DOCKER_API_URL=http://wis2box-api:80/oapi\n') + fh.write('\n') + fh.write('# backend\n') + fh.write('WIS2BOX_API_BACKEND_TYPE=Elasticsearch\n') + fh.write('WIS2BOX_API_BACKEND_URL=http://elasticsearch:9200\n') + fh.write('\n') + fh.write('# logging\n') + fh.write('WIS2BOX_LOGGING_LOGLEVEL=ERROR\n') + fh.write('WIS2BOX_LOGGING_LOGFILE=stdout\n') + fh.write('\n') + fh.write('# map settings for wis2box-ui, wis2box-api and wis2box-webapp\n') # noqa + fh.write('WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png\n') # noqa + fh.write('WIS2BOX_BASEMAP_ATTRIBUTION=OpenStreetMap contributors\n') # noqa + fh.write('\n') + fh.write('# storage, default is S3 provided by minio\n') + fh.write('WIS2BOX_STORAGE_TYPE=S3\n') + fh.write('WIS2BOX_STORAGE_SOURCE=http://minio:9000\n') + fh.write('WIS2BOX_STORAGE_INCOMING=wis2box-incoming\n') + fh.write('WIS2BOX_STORAGE_ARCHIVE=wis2box-archive\n') + fh.write('WIS2BOX_STORAGE_PUBLIC=wis2box-public\n') + fh.write('WIS2BOX_STORAGE_DATA_RETENTION_DAYS=30\n') # use the default username wis2box for WIS2BOX_STORAGE_USERNAME - fh.write("WIS2BOX_STORAGE_USERNAME=wis2box\n") - # get password for WIS2BOX_STORAGE_PASSWORD and write it to dev.env - fh.write(get_password("WIS2BOX_STORAGE_PASSWORD")) - fh.write("\n") + fh.write('WIS2BOX_STORAGE_USERNAME=wis2box\n') + # get password for WIS2BOX_STORAGE_PASSWORD and write it to wis2box.env + fh.write(get_password('WIS2BOX_STORAGE_PASSWORD')) + fh.write('\n') # write default port and host for WIS2BOX_BROKER - fh.write("WIS2BOX_BROKER_PORT=1883\n") - fh.write("WIS2BOX_BROKER_HOST=mosquitto\n") + fh.write('# broker settings\n') + fh.write('WIS2BOX_BROKER_PORT=1883\n') + fh.write('WIS2BOX_BROKER_HOST=mosquitto\n') # use the default username wis2box for WIS2BOX_BROKER_USERNAME - fh.write("WIS2BOX_BROKER_USERNAME=wis2box\n") - # get password for WIS2BOX_BROKER_PASSWORD and write it to dev.env - fh.write(get_password("WIS2BOX_BROKER_PASSWORD")) - fh.write("\n") + fh.write('WIS2BOX_BROKER_USERNAME=wis2box\n') + fh.write('WIS2BOX_BROKER_QUEUE_MAX=1000\n') + # get password for WIS2BOX_BROKER_PASSWORD and write it to wis2box.env + fh.write(get_password('WIS2BOX_BROKER_PASSWORD')) + fh.write('\n') # update WIS2BOX_PUBLIC_BROKER settings after updating broker defaults - fh.write("# update WIS2BOX_PUBLIC_BROKER settings after updating broker defaults\n") # noqa - fh.write("WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883\n") # noqa + fh.write('# update WIS2BOX_PUBLIC_BROKER settings after updating broker defaults\n') # noqa + fh.write('WIS2BOX_BROKER_PUBLIC=mqtt://${WIS2BOX_BROKER_USERNAME}:${WIS2BOX_BROKER_PASSWORD}@mosquitto:1883\n') # noqa # update minio settings after updating storage and broker defaults - fh.write("\n") - fh.write("# update minio settings after updating storage and broker defaults\n") # noqa - fh.write("MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME}\n") - fh.write("MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD}\n") - fh.write("MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME}\n") # noqa - fh.write("MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD}\n") # noqa - fh.write("MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT}\n") # noqa - - print("*" * 80) - print("The file dev.env has been created in the current directory.") - print("*" * 80) + fh.write('\n') + fh.write('# minio settings\n') # noqa + # MinIO + fh.write('MINIO_ROOT_USER=${WIS2BOX_STORAGE_USERNAME}\n') + fh.write('MINIO_ROOT_PASSWORD=${WIS2BOX_STORAGE_PASSWORD}\n') + fh.write('MINIO_PROMETHEUS_AUTH_TYPE=public\n') + fh.write('MINIO_NOTIFY_MQTT_ENABLE_WIS2BOX=on\n') + fh.write('MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME}\n') # noqa + fh.write('MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD}\n') # noqa + fh.write('MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT}\n') # noqa + fh.write('MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage\n') + fh.write('MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1\n') + fh.write('\n') + + print('*' * 80) + print('The file wis2box.env has been created in the current directory.') + print('*' * 80) def create_config_dir() -> str: @@ -303,26 +338,32 @@ def create_config_dir() -> str: print("Is this correct? (y/n/exit)") answer = input() - config_dir = Path(config_dir) - # check if the directory exists - if config_dir.is_dir(): - # if it exists warn the user - # tell them that the directory needs to be remove to continue - print("WARNING:") - print(f"The directory {config_dir} already exists.") - print("Please remove the directory to restart the configuration process.") # noqa - exit() - else: - # if it does not exist, create it - config_dir.mkdir() - # check if the directory was created - if not config_dir.is_dir(): - print("ERROR:") - print(f"The directory {config_dir} could not be created.") - print("Please check the path and your permissions.") + try: + config_dir = Path(config_dir) + if config_dir.is_dir(): + # if it exists warn the user + # tell them that the directory needs to be remove to continue + print("WARNING:") + print(f"The directory {config_dir} already exists.") + print("Please remove the directory to restart the configuration process.") # noqa exit() - print(f"The directory {config_dir} has been created.") + else: + # if it does not exist, create it + config_dir.mkdir() + # check if the directory was created + if not config_dir.is_dir(): + print("ERROR:") + print(f"The directory {config_dir} could not be created.") + print("Please check the path and your permissions.") + exit() + print(f"The directory {config_dir} has been created.") + except Exception: + print("ERROR:") + print(f"The directory {config_dir} could not be created.") + print("Please provide an absolute path to the directory.") + print("and check your permissions.") + exit() return config_dir @@ -342,8 +383,6 @@ def create_datamappings_file(config_dir: str, country_code: str, template_file = Path("config-templates/data-mappings.yml.tmpl") new_config_file = Path(config_dir) / "data-mappings.yml" - csv2bufr_mappings_template_file = Path("config-templates/csv2bufr_mappings.json") # noqa - csv2bufr_mappings_file = Path(config_dir) / 'csv2bufr_mappings.json' template_vars = { 'COUNTRY_CODE': country_code, @@ -355,15 +394,9 @@ def create_datamappings_file(config_dir: str, country_code: str, with new_config_file.open("w") as fh2: fh2.write(result) - # also add mappings.json from config-templates to config_dir - with csv2bufr_mappings_template_file.open() as fh: - mappings_file = fh.read() - with csv2bufr_mappings_file.open("w") as fh2: - fh2.write(mappings_file) - print("*" * 80) - print("Initial data_mappings.yml and csv2bufr_mappings.json have been created") # noqa - print("Please review the files and update as needed.") + print("Initial data_mappings.yml have been created") # noqa + print("Please review the file and update as needed.") print("*" * 80) @@ -516,15 +549,15 @@ def create_station_list(config_dir: str) -> None: with new_config_file.open("w") as fh2: fh2.write(config_file) - print("*" * 80) - print(f"Created the file {new_config_file}.") - print("Please add your stations to this file.") - print("*" * 80) + # print("*" * 80) + # print(f"Created the file {new_config_file}.") + # print("Please add your stations to this file.") + # print("*" * 80) def get_config_dir() -> str: """ - reads the value of WIS2BOX_HOST_DATADIR from dev.env + reads the value of WIS2BOX_HOST_DATADIR from wis2box.env returns: `str` of path to directory where configuration files are to be stored @@ -532,7 +565,7 @@ def get_config_dir() -> str: config_dir = None - with Path("dev.env").open() as fh: + with Path("wis2box.env").open() as fh: lines = fh.readlines() for line in lines: @@ -541,9 +574,9 @@ def get_config_dir() -> str: if not config_dir: print("WARNING:") - print("The file dev.env does not contain the variable WIS2BOX_HOST_DATADIR.") # noqa + print("The file wis2box.env does not contain the variable WIS2BOX_HOST_DATADIR.") # noqa print("Please edit the file and add the variable WIS2BOX_HOST_DATADIR.") # noqa - print("Or remove dev.env and run 'python3 wis2box-create-config.py' again.") # noqa + print("Or remove wis2box.env and run 'python3 wis2box-create-config.py' again.") # noqa exit() return config_dir @@ -557,18 +590,18 @@ def main(): """ config_dir = None - dev_env = Path("dev.env") + dev_env = Path("wis2box.env") - # check if dev.env exists + # check if wis2box.env exists # if it does, read the value for WIS2BOX_HOST_DATADIR - # or give the user the option to recreate dev.env + # or give the user the option to recreate wis2box.env if dev_env.is_file(): - print("The file dev.env already exists in the current directory.") - print("Do you want to recreate dev.env? (y/n/exit)") + print("The file wis2box.env already exists in the current directory.") + print("Do you want to recreate wis2box.env? (y/n/exit)") answer = input() if answer == "y": - os.remove("dev.env") + os.remove("wis2box.env") elif answer == "exit": exit() else: @@ -578,10 +611,10 @@ def main(): if not config_dir: config_dir = create_config_dir() - # if dev.env does not exist - # create it and write config_dir as the value for WIS2BOX_HOST_DATADIR to dev.env # noqa + # if wis2box.env does not exist + # create it and write config_dir as the value for WIS2BOX_HOST_DATADIR to wis2box.env # noqa if not dev_env.is_file(): - create_dev_env(config_dir) + create_wis2box_env(config_dir) country_code, centre_id = get_country_and_centre_id() diff --git a/wis2box-ctl.py b/wis2box-ctl.py index 339c9a45..75f9bee5 100755 --- a/wis2box-ctl.py +++ b/wis2box-ctl.py @@ -33,7 +33,7 @@ --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml - --env-file dev.env + --env-file wis2box.env --project-name wis2box_project """ @@ -162,8 +162,8 @@ def make(args) -> None: run(args, split( f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} build {containers}')) elif args.command in ["up", "start", "start-dev"]: - if not os.path.exists('dev.env'): - print("ERROR: dev.env file does not exist. Please create one manually or by running `python3 wis2box-create-config.py`") + if not os.path.exists('wis2box.env'): + print("ERROR: wis2box.env file does not exist. Please create one manually or by running `python3 wis2box-create-config.py`") exit(1) run(args, split( 'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions > /dev/null 2>&1')) diff --git a/wis2box-management/Dockerfile b/wis2box-management/Dockerfile index bb3030cd..eb04fc20 100644 --- a/wis2box-management/Dockerfile +++ b/wis2box-management/Dockerfile @@ -39,11 +39,11 @@ RUN if [ "$WIS2BOX_PIP3_EXTRA_PACKAGES" = "None" ]; \ RUN apt-get update -y && apt-get install -y ${DEBIAN_PACKAGES} \ # install wis2box data pipeline dependencies && pip3 install --no-cache-dir \ - https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.1.zip \ + https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \ https://github.com/wmo-im/bufr2geojson/archive/refs/tags/v0.5.0.zip \ https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip \ https://github.com/wmo-cop/pyoscar/archive/refs/tags/0.6.4.zip \ - https://github.com/wmo-im/synop2bufr/archive/refs/tags/v0.6.1.zip \ + https://github.com/wmo-im/synop2bufr/archive/refs/tags/v0.6.2.zip \ https://github.com/geopython/pygeometa/archive/refs/tags/0.15.3.zip \ https://github.com/wmo-im/pywcmp/archive/refs/tags/0.4.0.zip \ # install shapely diff --git a/wis2box-management/docker/wis2box.cron b/wis2box-management/docker/wis2box.cron index 15b2b939..475cd8cd 100644 --- a/wis2box-management/docker/wis2box.cron +++ b/wis2box-management/docker/wis2box.cron @@ -1,3 +1,3 @@ -0 0 * * * su wis2box -c "wis2box data clean --days=$WIS2BOX_DATA_RETENTION_DAYS" > /proc/1/fd/1 2>/proc/1/fd/2 +0 0 * * * su wis2box -c "wis2box data clean --days=$WIS2BOX_STORAGE_DATA_RETENTION_DAYS" > /proc/1/fd/1 2>/proc/1/fd/2 0 1 * * * su wis2box -c "wis2box data archive" > /proc/1/fd/1 2>/proc/1/fd/2 */10 * * * * su wis2box -c "echo 'wis2box.cron is alive'" > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index fd1c8617..ed4073e4 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -25,7 +25,6 @@ import re from typing import Iterator, Union -from wis2box.api import upsert_collection_item from wis2box.env import (STORAGE_INCOMING, STORAGE_PUBLIC, STORAGE_SOURCE, BROKER_PUBLIC, BROKER_HOST, BROKER_USERNAME, BROKER_PASSWORD, @@ -164,11 +163,6 @@ def notify(self, identifier: str, storage_path: str, broker.pub(topic, wis_message.dumps()) LOGGER.info(f'WISNotificationMessage published for {identifier}') - # message for internal monitoring - notify_msg = { - 'topic': topic, - 'wigos_station_identifier': wigos_station_identifier - } # load plugin for local broker defs_local = { 'codepath': PLUGINS['pubsub']['mqtt']['plugin'], @@ -177,12 +171,8 @@ def notify(self, identifier: str, storage_path: str, } local_broker = load_plugin('pubsub', defs_local) local_broker.pub('wis2box/notifications', - json.dumps(notify_msg), + wis_message.dumps(), qos=0) - - LOGGER.debug('Pushing message to API') - upsert_collection_item('messages', wis_message.message) - return True def publish(self) -> bool: diff --git a/wis2box-management/wis2box/data/synop2bufr.py b/wis2box-management/wis2box/data/synop2bufr.py index aa325847..5bdcf147 100644 --- a/wis2box-management/wis2box/data/synop2bufr.py +++ b/wis2box-management/wis2box/data/synop2bufr.py @@ -26,13 +26,10 @@ from synop2bufr import transform as transform_synop from wis2box.data.base import BaseAbstractData -from wis2box.env import DATADIR -from wis2box.metadata.station import get_valid_wsi +from wis2box.metadata.station import get_valid_wsi, get_stations_csv LOGGER = logging.getLogger(__name__) -STATION_METADATA = DATADIR / 'metadata' / 'station' / 'station_list.csv' - class ObservationDataSYNOP2BUFR(BaseAbstractData): """Synoptic observation data""" @@ -49,8 +46,7 @@ def __init__(self, defs: dict) -> None: self.mappings = {} - with STATION_METADATA.open() as fh: - self.station_metadata = fh.read() + self.station_metadata = get_stations_csv() def transform(self, input_data: Union[Path, bytes], filename: str = '') -> bool: diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index 20d6ceb9..ae959a37 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -27,7 +27,7 @@ LOGGER = logging.getLogger(__name__) -DATADIR = os.environ.get('WIS2BOX_DATADIR') +DATADIR = os.environ.get('WIS2BOX_DATADIR', '/data/wis2box') DATA_MAPPINGS = Path(DATADIR) / 'data-mappings.yml' if not DATA_MAPPINGS.exists(): diff --git a/wis2box-management/wis2box/env.py b/wis2box-management/wis2box/env.py index 9f83fc6f..48cc8998 100644 --- a/wis2box-management/wis2box/env.py +++ b/wis2box-management/wis2box/env.py @@ -39,27 +39,28 @@ LOGGER.error(msg) raise EnvironmentError(msg) -API_TYPE = os.environ.get('WIS2BOX_API_TYPE') -API_URL = os.environ.get('WIS2BOX_API_URL') -API_BACKEND_TYPE = os.environ.get('WIS2BOX_API_BACKEND_TYPE') -API_BACKEND_URL = os.environ.get('WIS2BOX_API_BACKEND_URL').rstrip('/') -DOCKER_API_URL = os.environ.get('WIS2BOX_DOCKER_API_URL') -AUTH_URL = os.environ.get('WIS2BOX_AUTH_URL') -URL = os.environ.get('WIS2BOX_URL') - -BROKER_USERNAME = os.environ.get('WIS2BOX_BROKER_USERNAME') -BROKER_PASSWORD = os.environ.get('WIS2BOX_BROKER_PASSWORD') -BROKER_HOST = os.environ.get('WIS2BOX_BROKER_HOST') -BROKER_PORT = os.environ.get('WIS2BOX_BROKER_PORT') -BROKER_PUBLIC = os.environ.get('WIS2BOX_BROKER_PUBLIC') - -STORAGE_TYPE = os.environ.get('WIS2BOX_STORAGE_TYPE') -STORAGE_SOURCE = os.environ.get('WIS2BOX_STORAGE_SOURCE') -STORAGE_USERNAME = os.environ.get('WIS2BOX_STORAGE_USERNAME') -STORAGE_PASSWORD = os.environ.get('WIS2BOX_STORAGE_PASSWORD') -STORAGE_INCOMING = os.environ.get('WIS2BOX_STORAGE_INCOMING') -STORAGE_ARCHIVE = os.environ.get('WIS2BOX_STORAGE_ARCHIVE') -STORAGE_PUBLIC = os.environ.get('WIS2BOX_STORAGE_PUBLIC') +API_TYPE = os.environ.get('WIS2BOX_API_TYPE', 'pygeoapi') +API_URL = os.environ.get('WIS2BOX_API_URL', 'http://localhost/oapi') +API_BACKEND_TYPE = os.environ.get('WIS2BOX_API_BACKEND_TYPE', 'Elasticsearch') +API_BACKEND_URL = os.environ.get('WIS2BOX_API_BACKEND_URL', 'http://elasticsearch:9200').rstrip('/') # noqa +DOCKER_API_URL = os.environ.get('WIS2BOX_DOCKER_API_URL', 'http://wis2box-api:80') # noqa +AUTH_URL = os.environ.get('WIS2BOX_AUTH_URL', 'http://wis2box-auth') +URL = os.environ.get('WIS2BOX_URL', 'http://localhost') + +BROKER_USERNAME = os.environ.get('WIS2BOX_BROKER_USERNAME', 'wis2box') +BROKER_PASSWORD = os.environ.get('WIS2BOX_BROKER_PASSWORD', 'wis2box') +BROKER_HOST = os.environ.get('WIS2BOX_BROKER_HOST', 'mosquitto') +BROKER_PORT = os.environ.get('WIS2BOX_BROKER_PORT', 1883) +BROKER_PUBLIC = os.environ.get('WIS2BOX_BROKER_PUBLIC', + f'mqtt://{BROKER_USERNAME}:{BROKER_PASSWORD}@{BROKER_HOST}:{BROKER_PORT}') # noqa + +STORAGE_TYPE = os.environ.get('WIS2BOX_STORAGE_TYPE', 'S3') +STORAGE_SOURCE = os.environ.get('WIS2BOX_STORAGE_SOURCE', 'http://minio:9000') +STORAGE_USERNAME = os.environ.get('WIS2BOX_STORAGE_USERNAME', 'wis2box') +STORAGE_PASSWORD = os.environ.get('WIS2BOX_STORAGE_PASSWORD', 'minio123') +STORAGE_INCOMING = os.environ.get('WIS2BOX_STORAGE_INCOMING', 'wis2box-incoming') # noqa +STORAGE_ARCHIVE = os.environ.get('WIS2BOX_STORAGE_ARCHIVE', 'wis2box-archive') +STORAGE_PUBLIC = os.environ.get('WIS2BOX_STORAGE_PUBLIC', 'wis2box-public') try: STORAGE_DATA_RETENTION_DAYS = int(os.environ.get('WIS2BOX_STORAGE_DATA_RETENTION_DAYS')) # noqa diff --git a/wis2box-management/wis2box/metadata/station.py b/wis2box-management/wis2box/metadata/station.py index 9738343a..15bae2bf 100644 --- a/wis2box-management/wis2box/metadata/station.py +++ b/wis2box-management/wis2box/metadata/station.py @@ -23,10 +23,12 @@ from collections import OrderedDict import csv from iso3166 import countries +import io import json import logging from typing import Iterator, Tuple, Union +from elasticsearch import Elasticsearch from owslib.ogcapi.features import Features from pygeometa.schemas.wmo_wigos import WMOWIGOSOutputSchema from pyoscar import OSCARClient @@ -35,13 +37,13 @@ from wis2box.api import ( delete_collection_item, setup_collection, upsert_collection_item ) -from wis2box.env import (DATADIR, DOCKER_API_URL, +from wis2box.env import (DATADIR, API_BACKEND_URL, DOCKER_API_URL, BROKER_HOST, BROKER_USERNAME, BROKER_PASSWORD, BROKER_PORT) from wis2box.metadata.base import BaseMetadata +from wis2box.plugin import load_plugin, PLUGINS from wis2box.util import get_typed_value -from wis2box.plugin import load_plugin, PLUGINS LOGGER = logging.getLogger(__name__) @@ -66,11 +68,6 @@ 'europe': 'VI' } -if not STATIONS.exists(): - msg = f'Please create a station metadata file in {STATION_METADATA}' - LOGGER.error(msg) - raise RuntimeError(msg) - def get_wmo_ra_roman(ra: str) -> str: """ @@ -136,6 +133,86 @@ def station(): pass +def load_stations(wsi: str = '') -> dict: + """Load stations from API + + param wsi: `str` of WIGOS Station Identifier + + :returns: `dict` of stations + """ + + stations = {} + + try: + es = Elasticsearch(API_BACKEND_URL) + nbatch = 500 # TODO: make configurable + res = es.search(index='stations', query={'match_all': {}}, size=nbatch) + if len(res['hits']['hits']) == 0: + LOGGER.debug('No stations found') + return stations + for hit in res['hits']['hits']: + stations[hit['_source']['id']] = hit['_source'] + while len(res['hits']['hits']) > 0: + res = es.search( + index='stations', query={'match_all': {}}, + size=nbatch, from_=len(stations)) # noqa + + for hit in res['hits']['hits']: + stations[hit['_source']['id']] = hit['_source'] + + except Exception as err: + LOGGER.error(f'Failed to load stations from backend: {err}') + + LOGGER.info(f'Loaded {len(stations.keys())} stations from backend') + + return stations + + +def get_stations_csv(wsi: str = '') -> str: + """Load stations into csv-string + + param wsi: `str` of WIGOS Station Identifier + + :returns: `str` of CSV with station data + """ + + LOGGER.info('Loading stations into csv-string') + + csv_output = [] + stations = load_stations(wsi) + + for station in stations.values(): + wsi = station['properties']['wigos_station_identifier'] + if '-' in wsi and len(wsi.split('-')) == 4: + tsi = wsi.split('-')[3] + + barometer_height = station['properties'].get( + 'barometer_height', + station['geometry']['coordinates'][2] + 1.25) + + obj = { + 'station_name': station['properties']['name'], + 'wigos_station_identifier': wsi, + 'traditional_station_identifier': tsi, + 'facility_type': station['properties']['facility_type'], + 'latitude': station['geometry']['coordinates'][1], + 'longitude': station['geometry']['coordinates'][0], + 'elevation': station['geometry']['coordinates'][2], + 'barometer_height': barometer_height, + 'territory_name': station['properties']['territory_name'], + 'wmo_region': station['properties']['wmo_region'] + } + csv_output.append(obj) + + string_buffer = io.StringIO() + csv_writer = csv.DictWriter(string_buffer, fieldnames=csv_output[0].keys()) + csv_writer.writeheader() + csv_writer.writerows(csv_output) + csv_string = string_buffer.getvalue() + string_buffer.close() + return csv_string + + def load_datasets() -> Iterator[Tuple[dict, str]]: """ Load datasets from oapi @@ -192,6 +269,11 @@ def publish_station_collection() -> None: :returns: `None` """ + if not STATIONS.exists(): + msg = f'Please create a station metadata file in {STATION_METADATA}' + LOGGER.error(msg) + raise RuntimeError(msg) + oscar_baseurl = 'https://oscar.wmo.int/surface/#/search/station/stationReportDetails' # noqa LOGGER.debug(f'Publishing station list from {STATIONS}') @@ -231,6 +313,7 @@ def publish_station_collection() -> None: 'properties': { 'name': row['station_name'], 'wigos_station_identifier': wigos_station_identifier, + 'traditional_station_identifier': row['traditional_station_identifier'], # noqa 'barometer_height': barometer_height, 'facility_type': row['facility_type'], 'territory_name': row['territory_name'], @@ -283,12 +366,10 @@ def get_valid_wsi(wsi: str = '', tsi: str = '') -> Union[str, None]: """ LOGGER.info(f'Validating WIGOS Station Identifier: {wsi}') - with STATIONS.open(newline='') as fh: - reader = csv.DictReader(fh) - for row in reader: - if wsi == row['wigos_station_identifier'] or \ - (tsi == row['traditional_station_identifier'] and tsi != ''): - return row['wigos_station_identifier'] + stations = load_stations(wsi) + + if wsi in stations: + return wsi return None @@ -302,27 +383,9 @@ def get_geometry(wsi: str = '') -> Union[dict, None]: :returns: `dict`, of station geometry or `None` """ - LOGGER.info(f'Validating WIGOS Station Identifier: {wsi}') - with STATIONS.open(newline='') as fh: - reader = csv.DictReader(fh) - for row in reader: - if wsi == row['wigos_station_identifier']: - LOGGER.debug('Found matching WSI') - feature = { - 'type': 'Point', - 'coordinates': [ - get_typed_value(row['longitude']), - get_typed_value(row['latitude']) - ] - } - - station_elevation = get_typed_value(row['elevation']) - - if isinstance(station_elevation, (float, int)): - LOGGER.debug('Adding z value to geometry') - feature['coordinates'].append(station_elevation) - - return feature + stations = load_stations(wsi) + if wsi in stations: + return stations[wsi]['geometry'] LOGGER.debug('No matching WSI') return None @@ -345,12 +408,12 @@ def get(ctx, wsi, verbosity): results = OrderedDict({ 'station_name': station['station_name'], 'wigos_station_identifier': station['wigos_station_identifier'], - 'barometer_height': station['barometer_height'], 'traditional_station_identifier': None, 'facility_type': station['facility_type'], 'latitude': station.get('latitude', ''), 'longitude': station.get('longitude', ''), 'elevation': station.get('elevation'), + 'barometer_height': station['barometer_height'], 'territory_name': station.get('territory_name', '') }) diff --git a/wis2box-management/wis2box/pubsub/subscribe.py b/wis2box-management/wis2box/pubsub/subscribe.py index a0b51443..b1588d79 100644 --- a/wis2box-management/wis2box/pubsub/subscribe.py +++ b/wis2box-management/wis2box/pubsub/subscribe.py @@ -27,6 +27,7 @@ import click +from wis2box.api import upsert_collection_item from wis2box import cli_helpers from wis2box.api import setup_collection from wis2box.env import (BROKER_HOST, BROKER_PORT, BROKER_USERNAME, @@ -61,41 +62,42 @@ def handle(filepath): def on_message_handler(client, userdata, msg): LOGGER.debug(f'Raw message: {msg.payload}') + topic = msg.topic message = json.loads(msg.payload) - - if message.get('EventName') == 's3:ObjectCreated:Put': - LOGGER.debug('Incoming data is an s3 data object') - key = str(message['Key']) - filepath = f'{STORAGE_SOURCE}/{key}' - if key.startswith(STORAGE_ARCHIVE): - LOGGER.info(f'Do not process archived-data: {key}') - return - elif 'relPath' in message: - LOGGER.debug('Incoming data is a filesystem path') - filepath = Path(message['relPath']) + LOGGER.info(f'Incoming message on topic {topic}') + if topic == 'wis2box/notifications': + LOGGER.info(f'Notification: {message}') + # store notification in messages collection + upsert_collection_item('messages', message) else: - LOGGER.warning('message payload could not be parsed') - return - - while len(mp.active_children()) == mp.cpu_count(): - sleep(0.1) + if message.get('EventName') == 's3:ObjectCreated:Put': + LOGGER.debug('Incoming data is an s3 data object') + key = str(message['Key']) + filepath = f'{STORAGE_SOURCE}/{key}' + if key.startswith(STORAGE_ARCHIVE): + LOGGER.info(f'Do not process archived-data: {key}') + return + elif 'relPath' in message: + LOGGER.debug('Incoming data is a filesystem path') + filepath = Path(message['relPath']) + else: + LOGGER.debug('ignore message') + return - p = mp.Process(target=handle, args=(filepath,)) - p.start() + while len(mp.active_children()) == mp.cpu_count(): + sleep(0.1) + p = mp.Process(target=handle, args=(filepath,)) + p.start() @click.command() @click.pass_context -@click.option('--broker', '-b', help='URL to broker') -@click.option('--topic', '-t', help='topic to subscribe to') @cli_helpers.OPTION_VERBOSITY -def subscribe(ctx, broker, topic, verbosity): - """Subscribe to a broker/topic""" +def subscribe(ctx, verbosity): + """Subscribe to the internal broker and process incoming messages""" click.echo('Adding messages collection') setup_collection(meta=gcm()) - click.echo(f'Subscribing to broker {broker}, topic {topic}') - defs = { 'codepath': PLUGINS['pubsub']['mqtt']['plugin'], 'url': f'mqtt://{BROKER_USERNAME}:{BROKER_PASSWORD}@{BROKER_HOST}:{BROKER_PORT}', # noqa @@ -106,4 +108,5 @@ def subscribe(ctx, broker, topic, verbosity): broker.bind('on_message', on_message_handler) - broker.sub(topic) + click.echo('Subscribing to internal broker on topic wis2box/#') + broker.sub('wis2box/#') diff --git a/wis2box-mqtt-metrics-collector/mqtt_metrics_collector.py b/wis2box-mqtt-metrics-collector/mqtt_metrics_collector.py index d2cffbfa..7e2aa6c4 100644 --- a/wis2box-mqtt-metrics-collector/mqtt_metrics_collector.py +++ b/wis2box-mqtt-metrics-collector/mqtt_metrics_collector.py @@ -128,7 +128,7 @@ def sub_connect(client, userdata, flags, rc, properties=None): """ logger.info(f"on connection to subscribe: {mqtt.connack_string(rc)}") - for s in ["wis2box/#", "wis2box-storage/#", '$SYS/broker/messages/#']: + for s in ["wis2box/#", '$SYS/broker/messages/#']: print(f'subscribe to: {s}') client.subscribe(s, qos=0) @@ -164,7 +164,7 @@ def sub_mqtt_metrics(client, userdata, msg): update_stations_gauge(m['station_list']) elif str(msg.topic).startswith('wis2box/notifications'): notify_wsi_total.labels( - m['wigos_station_identifier']).inc(1) + m['properties']['wigos_station_identifier']).inc(1) notify_total.inc(1) elif str(msg.topic).startswith('wis2box/failure'): descr = m['description'] @@ -174,7 +174,7 @@ def sub_mqtt_metrics(client, userdata, msg): failure_descr_wsi_total.labels(descr, wsi).inc(1) failure_wsi_total.labels(wsi).inc(1) failure_total.inc(1) - elif str(msg.topic).startswith('wis2box-storage'): + elif str(msg.topic).startswith('wis2box/storage'): if str(m["Key"]).startswith('wis2box-incoming'): storage_incoming_total.inc(1) if str(m["Key"]).startswith('wis2box-public'): diff --git a/default.env b/wis2box.env.example similarity index 94% rename from default.env rename to wis2box.env.example index b424fd49..7194e3cd 100644 --- a/default.env +++ b/wis2box.env.example @@ -56,7 +56,6 @@ MINIO_NOTIFY_MQTT_ENABLE_WIS2BOX=on MINIO_NOTIFY_MQTT_USERNAME_WIS2BOX=${WIS2BOX_BROKER_USERNAME} MINIO_NOTIFY_MQTT_PASSWORD_WIS2BOX=${WIS2BOX_BROKER_PASSWORD} MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_PORT} -MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box-storage/minio -MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box-storage/minio +MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1