1.0b5 maaikewip (#524)

.github/workflows/tests-docker.yml

@@ -26,8 +26,8 @@ jobs:
         python3 -V
     - name: setup wis2box configuration
       run: |
-        cp tests/test.env dev.env
-        cat dev.env
+        cp tests/test.env wis2box.env
+        cat wis2box.env
         python3 wis2box-ctl.py config
     - name: build wis2box
       run: |
@@ -42,6 +42,9 @@ jobs:
       run: |
         sleep 30
         python3 wis2box-ctl.py execute wis2box environment show
+    - name: populate stations from csv 📡
+      run: |
+        python3 wis2box-ctl.py execute wis2box metadata station publish-collection
     - name: add Malawi data 🇲🇼
       env:
         TOPIC_HIERARCHY: mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop
@@ -101,5 +104,6 @@ jobs:
     - name: failed tests 🚩
       if: ${{ failure() }}
       run: |
-        docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project ps
-        docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project logs
+        docker logs wis2box-management
+        docker logs wis2box-api
+        docker logs wis2box-minio

.gitignore

@@ -40,6 +40,8 @@ nosetests.xml
 wis2box-data
 .pytest_cache
 dev.env
+wis2box.env
 docker/.env
 
 .ipynb_checkpoints
+wis2box.env

config-templates/data-mappings.yml.tmpl

@@ -9,7 +9,7 @@ data:
                   file-pattern: '^.*-(\d{4})(\d{2}).*\.txt$$'
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR
-                  template: /data/wis2box/csv2bufr_mappings.json
+                  template: aws-template
                   notify: true
                   buckets:
                     - $${WIS2BOX_STORAGE_INCOMING}

config-templates/metadata-synop.yml.tmpl

@@ -24,7 +24,7 @@ identification:
     keywords:
         default:
             keywords:
-                - upper-air weather
+                - surface weather
                 - observations
         wmo:
             keywords:

config-templates/station_list_example.csv

@@ -1,4 +1 @@
 station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region
-EXAMPLE1-SURFACE,0-454-0-EXAMPLE,,Land (fixed),-14.98333,34.96666,618,,Malawi,I
-EXAMPLE2-SURFACE,0-20000-0-12345,12345,Land (fixed),41.56361,14.65500,793,807.0,Italy,VI
-EXAMPLE3-UPPERAIR,0-52-0-5432,54321,Air (fixed),13.08585,-59.48701,56.6,,Barbados,IV

docker-compose.monitoring.yml

@@ -37,8 +37,7 @@ services:
     container_name: mqtt_metrics_collector
     restart: unless-stopped
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:1.0.beta1
     build:
       context: ./wis2box-mqtt-metrics-collector
@@ -59,25 +58,25 @@ services:
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.retention.time=10d'
-  # cadvisor to monitor our containers!
-  cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.45.0
-    container_name: cadvisor
-    volumes:
-      - /:/rootfs:ro
-      - /var/run:/var/run:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker:ro
-      - /dev/disk:/dev/disk/:ro
-    depends_on:
-      - wis2box-management
+  # uncomment cadvisor to monitor our containers
+  # cadvisor:
+  #  image: gcr.io/cadvisor/cadvisor:v0.45.0
+  #  container_name: cadvisor
+  #  volumes:
+  #    - /:/rootfs:ro
+  #    - /var/run:/var/run:ro
+  #    - /var/run/docker.sock:/var/run/docker.sock:ro
+  #    - /sys:/sys:ro
+  #    - /var/lib/docker/:/var/lib/docker:ro
+  #    - /dev/disk:/dev/disk/:ro
+  #  depends_on:
+  #    - wis2box-management
   # Grafana, graphical monitoring dashboards for wis2box using data from loki and prometheus
   grafana:
     <<: *logging
     container_name: grafana
     env_file:
-      - dev.env
+      - wis2box.env
     image: grafana/grafana-oss:9.0.3
     volumes:
       - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
@@ -95,7 +94,6 @@ services:
       - GF_ALERTING_ENABLED=true
       - GF_USERS_DEFAULT_THEME=dark
       - GF_DISABLE_SIGNOUT_MENU=true
-      - GF_SERVER_ROOT_URL=${WIS2BOX_URL:-http://localhost}/monitoring
       - GF_SERVER_SERVE_FROM_SUB_PATH=true
     ports:
       - 3000:3000

docker-compose.yml

@@ -4,8 +4,7 @@ services:
     image: nginx:alpine
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       - wis2box-ui
     volumes:
@@ -16,28 +15,33 @@ services:
     image: ghcr.io/wmo-im/wis2box-ui:0.5.3
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
-      - wis2box-api
+      wis2box-api:
+        condition: service_healthy
+
+  wis2box-webapp:
+    container_name: wis2box-webapp
+    #image: wis2box-webapp:main
+    image: ghcr.io/wmo-im/wis2box-webapp:latest
+    env_file:
+      - wis2box.env
 
 #  wis2box-ui-admin:
 #    container_name: wis2box-ui-admin
 #    image: ghcr.io/wmo-im/wis2box-ui-admin:latest
 #    restart: always
 #    env_file:
-#      - default.env
-#      - dev.env
+#      - wis2box.env
 #    depends_on:
 #      - wis2box-api
 
   wis2box-api:
     container_name: wis2box-api
-    image: ghcr.io/wmo-im/wis2box-api:latest
+    image: wmoim/wis2box-api:wis2box-1.0b5
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -55,8 +59,7 @@ services:
     memswap_limit: 512m
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     command: server --console-address ":9001" /data
     # in a production-setup minio needs to be
     volumes:
@@ -105,8 +108,7 @@ services:
     build:
       context: ./wis2box-broker
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
 
   wis2box-management:
     container_name: wis2box-management
@@ -118,8 +120,7 @@ services:
       context: ./wis2box-management
     #user: wis2box:wis2box
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     volumes:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
     depends_on:
@@ -129,15 +130,18 @@ services:
         condition: service_started
       wis2box-api:
         condition: service_healthy
-    command: ["wis2box", "pubsub" , "subscribe", "--broker", "http://wis2box-minio:9000", "--topic", "wis2box-storage/#"]
+    command: ["wis2box", "pubsub" , "subscribe"]
+    healthcheck:
+      test: ["CMD", "/healthcheck.sh"]
+      interval: 5s
+      retries: 100
 
   wis2box-auth:
     container_name: wis2box-auth
     image: ghcr.io/wmo-im/wis2box-auth:latest
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     volumes:
       - auth-data:/data/wis2box:rw
     depends_on:

docs/source/community/support.rst

@@ -6,3 +6,5 @@ Support
 Please consult the wis2box `Discussions`_ for support with the project.
 
 .. _`Discussions`: https://github.com/wmo-im/wis2box/discussions
+
+Please contact the WMO Secretariat for support with the WIS 2.0 project by writing an email to [email protected].

docs/source/community/troubleshooting.rst

@@ -15,7 +15,7 @@ The wis2box logging displays the error:
 
     OSError: Missing data mappings: [Errno 2] No such file or directory: '/data/wis2box/data-mappings.yml'
 
-Check your dev.env and check value that was set for WIS2BOX_HOST_DATADIR
+Check your wis2box.env and check value that was set for WIS2BOX_HOST_DATADIR
 
 .. code-block:: bash
 
@@ -70,8 +70,7 @@ The Access Key Id you provided does not exist in our records
 ------------------------------------------------------------
 
 If you see this error when uploading data to the wis2box-incoming storage, you have provided the wrong username and/or password to access MinIO.
-Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` you have provided in your ``dev.env`` file. 
-The default username/password for MinIO is ``minio/minio123``.
+Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` set in your ``wis2box.env`` file. 
 
 Topic Hierarchy validation error: No plugins for ... in data mappings
 ---------------------------------------------------------------------

docs/source/reference/administration.rst

@@ -75,7 +75,7 @@ Changing default ports
 ^^^^^^^^^^^^^^^^^^^^^^
 
 The ``docker-compose.override.yml`` file provides definitions on utilized ports.  To change default
-ports, edit ``default.env``  before stopping and starting wis2box for changes to take effect.
+ports, edit ``wis2box.env``  before stopping and starting wis2box for changes to take effect.
 
 
 MQTT Quality of Service (QoS)

docs/source/reference/auth.rst

@@ -16,11 +16,19 @@ To add a token to the execution of a wis2box process, use the following command:
 
 .. code-block:: bash
 
-    wis2box auth add-token --path processes/wis2box myexecutiontoken
+    wis2box auth add-token --path processes/wis2box
+
+This will generate a random token that can be used to execute ``wis2box-synop2bufr`` and ``wis2box-csv2bufr``.
+Be sure to record the token now, there is no way to retrieve it once it is lost.
 
 To add a token to PUT/POST/DELETE requests to the stations collection, use the following command:
 
-    wis2box auth add-token --path collections/stations mystationupdatetoken
+.. code-block:: bash
+
+    wis2box auth add-token --path collections/stations
+
+This will generate a random token that can be use to update the stations collection.
+Be sure to record the token now, there is no way to retrieve it once it is lost. 
 
 Adding Access Control on topics
 -------------------------------