Update create config + 1.0b5 updates

.github/workflows/tests-docker.yml

@@ -28,8 +28,8 @@ jobs:
         python3 -V
     - name: setup wis2box configuration
       run: |
-        cp tests/test.env dev.env
-        cat dev.env
+        cp tests/test.env wis2box.env
+        cat wis2box.env
         python3 wis2box-ctl.py config
     - name: build wis2box
       run: |
@@ -44,6 +44,9 @@ jobs:
       run: |
         sleep 30
         python3 wis2box-ctl.py execute wis2box environment show
+    - name: populate stations from CSV 📡
+      run: |
+        python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: add Malawi data 🇲🇼
       env:
         TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
@@ -110,13 +113,14 @@ jobs:
         python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: run integration tests ⚙️
       run: |
-        sleep 10
+        sleep 15
         pytest -s tests/integration
     - name: run flake8 ⚙️
       run: |
         find . -type f -name "*.py" | xargs flake8
     - name: failed tests 🚩
       if: ${{ failure() }}
       run: |
-        docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project ps
-        docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project logs
+        docker logs wis2box-management
+        docker logs wis2box-api
+        docker logs wis2box-minio

.gitignore

@@ -40,6 +40,7 @@ nosetests.xml
 wis2box-data
 .pytest_cache
 dev.env
+wis2box.env
 docker/.env
 
 .ipynb_checkpoints

config-templates/data-mappings.yml.tmpl

@@ -9,7 +9,7 @@ data:
                   file-pattern: '^.*-(\d{4})(\d{2}).*\.txt$$'
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR
-                  template: /data/wis2box/csv2bufr_mappings.json
+                  template: aws-template
                   notify: true
                   buckets:
                     - $${WIS2BOX_STORAGE_INCOMING}
@@ -37,7 +37,7 @@ data:
                   buckets:
                     - $${WIS2BOX_STORAGE_PUBLIC}
                   file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$$'
-    COUNTRY_CODE.CENTRE_ID.data.core.weather.surface-based-observations.temp:
+    $COUNTRY_CODE.$CENTRE_ID.data.core.weather.surface-based-observations.temp:
         plugins:
             b:
                 - plugin: wis2box.data.bufr4.ObservationDataBUFR

config-templates/station_list_example.csv

@@ -1,4 +1 @@
 station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region
-EXAMPLE1-SURFACE,0-454-0-EXAMPLE,,Land (fixed),-14.98333,34.96666,618,,Malawi,I
-EXAMPLE2-SURFACE,0-20000-0-12345,12345,Land (fixed),41.56361,14.65500,793,807.0,Italy,VI
-EXAMPLE3-UPPERAIR,0-52-0-5432,54321,Air (fixed),13.08585,-59.48701,56.6,,Barbados,IV

docker-compose.dev.yml

@@ -4,7 +4,7 @@ services:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
       - ./wis2box-management/wis2box/wis2box.cron:/etc/cron.d/wis2box:ro
       - ./wis2box-management/wis2box:/app/wis2box
-    command: ["wis2box", "pubsub" , "subscribe", "--broker", "http://wis2box-minio:9000", "--topic", "wis2box-storage/#", "--verbosity", "INFO"]
+    command: ["wis2box", "pubsub" , "subscribe"]
 
   wis2box-api:
     volumes:

docker-compose.monitoring.yml

@@ -37,8 +37,7 @@ services:
     container_name: mqtt_metrics_collector
     restart: unless-stopped
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:1.0.beta1
     build:
       context: ./wis2box-mqtt-metrics-collector
@@ -59,25 +58,25 @@ services:
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.retention.time=10d'
-  # cadvisor to monitor our containers!
-  cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.45.0
-    container_name: cadvisor
-    volumes:
-      - /:/rootfs:ro
-      - /var/run:/var/run:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker:ro
-      - /dev/disk:/dev/disk/:ro
-    depends_on:
-      - wis2box-management
+  # uncomment cadvisor to monitor containers
+  # cadvisor:
+  #  image: gcr.io/cadvisor/cadvisor:v0.45.0
+  #  container_name: cadvisor
+  #  volumes:
+  #    - /:/rootfs:ro
+  #    - /var/run:/var/run:ro
+  #    - /var/run/docker.sock:/var/run/docker.sock:ro
+  #    - /sys:/sys:ro
+  #    - /var/lib/docker/:/var/lib/docker:ro
+  #    - /dev/disk:/dev/disk/:ro
+  #  depends_on:
+  #    - wis2box-management
   # Grafana, graphical monitoring dashboards for wis2box using data from loki and prometheus
   grafana:
     <<: *logging
     container_name: grafana
     env_file:
-      - dev.env
+      - wis2box.env
     image: grafana/grafana-oss:9.0.3
     volumes:
       - ./grafana/dashboards:/etc/grafana/provisioning/dashboards

docker-compose.yml

@@ -4,8 +4,7 @@ services:
     image: nginx:alpine
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       - wis2box-ui
     volumes:
@@ -16,8 +15,7 @@ services:
     image: ghcr.io/wmo-im/wis2box-ui:0.5.3
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       - wis2box-api
 
@@ -26,18 +24,23 @@ services:
 #    image: ghcr.io/wmo-im/wis2box-ui-admin:latest
 #    restart: always
 #    env_file:
-#      - default.env
-#      - dev.env
+#      - wis2box.env
 #    depends_on:
 #      - wis2box-api
 
+  wis2box-webapp:
+    container_name: wis2box-webapp
+    image: ghcr.io/wmo-im/wis2box-webapp:latest
+    env_file:
+      - wis2box.env
+
   wis2box-api:
     container_name: wis2box-api
-    image: ghcr.io/wmo-im/wis2box-api:latest
+    # image: ghcr.io/wmo-im/wis2box-api:latest
+    image: wmoim/wis2box-api:wis2box-1.0b5
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -55,8 +58,7 @@ services:
     memswap_limit: 512m
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     command: server --console-address ":9001" /data
     # in a production-setup minio needs to be
     volumes:
@@ -105,8 +107,7 @@ services:
     build:
       context: ./wis2box-broker
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
 
   wis2box-management:
     container_name: wis2box-management
@@ -118,8 +119,7 @@ services:
       context: ./wis2box-management
     #user: wis2box:wis2box
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     volumes:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
     depends_on:
@@ -129,15 +129,14 @@ services:
         condition: service_started
       wis2box-api:
         condition: service_healthy
-    command: ["wis2box", "pubsub" , "subscribe", "--broker", "http://wis2box-minio:9000", "--topic", "wis2box-storage/#"]
+    command: ["wis2box", "pubsub" , "subscribe"]
 
   wis2box-auth:
     container_name: wis2box-auth
     image: ghcr.io/wmo-im/wis2box-auth:latest
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     volumes:
       - auth-data:/data/wis2box:rw
     depends_on:

docs/source/community/troubleshooting.rst

@@ -15,7 +15,7 @@ The wis2box logging displays the error:
 
     OSError: Missing data mappings: [Errno 2] No such file or directory: '/data/wis2box/data-mappings.yml'
 
-Check your dev.env and check value that was set for WIS2BOX_HOST_DATADIR
+Check ``wis2box.env`` and check value that was set for WIS2BOX_HOST_DATADIR
 
 .. code-block:: bash
 
@@ -39,7 +39,7 @@ After you have ensured the data-mappings.yml is in the directory defined by WIS2
 Topic Hierarchy validation error: Unknown file type
 ---------------------------------------------------
 
-Check your ``data-mappings.yml`` file to adjust the file extension expected by the plugins processing your dataset. 
+Check ``data-mappings.yml`` file to adjust the file extension expected by the plugins processing your dataset.
 
 If you are ingesting files with extension .bin:
 
@@ -70,13 +70,12 @@ The Access Key Id you provided does not exist in our records
 ------------------------------------------------------------
 
 If you see this error when uploading data to the wis2box-incoming storage, you have provided the wrong username and/or password to access MinIO.
-Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` you have provided in your ``dev.env`` file. 
-The default username/password for MinIO is ``minio/minio123``.
+Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` set in the ``wis2box.env`` file.
 
 Topic Hierarchy validation error: No plugins for ... in data mappings
 ---------------------------------------------------------------------
 
-A file arrived a folder for which no matching dataset was defined in your ``data-mappings.yml``.
+A file arrived a folder for which no matching dataset was defined in ``data-mappings.yml``.
 
 For dataset ``foo.bar``, store your file in the path ``/foo/bar/``.
 
@@ -86,12 +85,18 @@ ERROR - Failed to publish, wsi: ..., tsi: XXXXX
 -----------------------------------------------
 
 Data arrived for a station that is not present in the station metadata cache. 
-To add missing stations, update the file ``metadata/station/station_list.csv`` in the wis2box data directory (see :ref:`setup`).
+
+To add missing stations, use the station-editor in wis2box-webapp (from wis2box-1.0b5) or update the file ``metadata/station/station_list.csv`` in the wis2box data directory and run the command:
+
+.. code-block:: bash
+
+   python3 wis2box-ctl.py login
+   wis2box metadata stations publish-collections
 
 Error: no such container: wis2box-management
 --------------------------------------------
 
-If the wis2box-management container is not running the 'login' command will fail. 
+If the wis2box-management container is not running, the ``login`` command will fail.
 The wis2box-management container depends on other services being available before it can successfully started.
 
 Please check all services are Running using the following command:
@@ -102,26 +107,27 @@ Please check all services are Running using the following command:
 
 Possible issues are:
 
-- The directory defined by WIS2BOX_HOST_DATADIR does not contain the file 'data-mappings.yml' or the file is invalid
-- The directory defined by WIS2BOX_HOST_DATADIR does not contain the file 'metastation/station/station_list.csv' or the file is invalid
-- WIS2BOX_STORAGE_PASSWORD is too short, minio will fail to start if you specify a WIS2BOX_STORAGE_PASSWORD of less than 8 characters
+- The host ran out of diskspace, check the output of ``df -h`` and ensure there is sufficient space available
+- The directory defined by ``WIS2BOX_HOST_DATADIR`` does not contain the file ``data-mappings.yml`` or the file is invalid
+- The directory defined by ``WIS2BOX_HOST_DATADIR`` does not contain the file ``metastation/station/station_list.csv`` or the file is invalid
+- ``WIS2BOX_STORAGE_PASSWORD`` is too short, MinIO will fail to start if you specify a ``WIS2BOX_STORAGE_PASSWORD`` of less than 8 characters
 
 wisbox-UI is empty
 ------------------
 
-If when you access the wis2box-UI you see the interface but no datasets are visible, check the WIS2BOX_URL and WIS2BOX_API_URL are set correctly.
+If when you access the wis2box UI you see the interface but no datasets are visible; check the ``WIS2BOX_URL`` and ``WIS2BOX_API_URL`` are set correctly.
 
-Please note that after changing the WIS2BOX_URL and WIS2BOX_API_URL, you will have to restart your wis2box:
+Please note that after changing the ``WIS2BOX_URL`` and ``WIS2BOX_API_URL``, you will have to restart wis2box:
 
 .. code-block:: bash
 
   python3 wis2box-ctl.py stop
   python3 wis2box-ctl.py start
 
-And repeat the commands for adding your dataset and publishing your metadata, to ensure the URLs are updated in the records:
+..and then repeat the commands for adding your dataset and publishing your metadata, to ensure the URLs are updated in the records accordingly:
 
 .. code-block:: bash
 
   python3 wis2box-ctl.py login
-  wis2box data add-collection ${WIS2BOX_DATADIR}/surface-weather-observations.yml
-  wis2box metadata discovery publish ${WIS2BOX_DATADIR}/surface-weather-observations.yml
+  wis2box data add-collection /data/wis2box/metadata/discovery/metadata-synop.yml
+  wis2box metadata discovery publish /data/wis2box/metadata/discovery/metadata-synop.yml

docs/source/reference/administration.rst

@@ -75,7 +75,7 @@ Changing default ports
 ^^^^^^^^^^^^^^^^^^^^^^
 
 The ``docker-compose.override.yml`` file provides definitions on utilized ports.  To change default
-ports, edit ``default.env``  before stopping and starting wis2box for changes to take effect.
+ports, edit ``wis2box.env``  before stopping and starting wis2box for changes to take effect.
 
 
 MQTT Quality of Service (QoS)