Merge branch 'update-config-creation' into 1.0b5

.github/workflows/tests-docker.yml

@@ -26,8 +26,8 @@ jobs:
         python3 -V
     - name: setup wis2box configuration
       run: |
-        cp tests/test.env dev.env
-        cat dev.env
+        cp tests/test.env wis2box.env
+        cat wis2box.env
         python3 wis2box-ctl.py config
     - name: build wis2box
       run: |
@@ -104,5 +104,6 @@ jobs:
     - name: failed tests 🚩
       if: ${{ failure() }}
       run: |
-        docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project ps
-        docker compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project logs
+        docker logs wis2box-management
+        docker logs wis2box-api
+        docker logs wis2box-minio

.gitignore

@@ -40,6 +40,8 @@ nosetests.xml
 wis2box-data
 .pytest_cache
 dev.env
+wis2box.env
 docker/.env
 
 .ipynb_checkpoints
+wis2box.env

config-templates/data-mappings.yml.tmpl

@@ -9,7 +9,7 @@ data:
                   file-pattern: '^.*-(\d{4})(\d{2}).*\.txt$$'
             csv:
                 - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR
-                  template: /data/wis2box/csv2bufr_mappings.json
+                  template: aws-template
                   notify: true
                   buckets:
                     - $${WIS2BOX_STORAGE_INCOMING}

config-templates/metadata-synop.yml.tmpl

@@ -24,7 +24,7 @@ identification:
     keywords:
         default:
             keywords:
-                - upper-air weather
+                - surface weather
                 - observations
         wmo:
             keywords:

config-templates/station_list_example.csv

@@ -1,4 +1 @@
 station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region
-EXAMPLE1-SURFACE,0-454-0-EXAMPLE,,Land (fixed),-14.98333,34.96666,618,,Malawi,I
-EXAMPLE2-SURFACE,0-20000-0-12345,12345,Land (fixed),41.56361,14.65500,793,807.0,Italy,VI
-EXAMPLE3-UPPERAIR,0-52-0-5432,54321,Air (fixed),13.08585,-59.48701,56.6,,Barbados,IV

docker-compose.monitoring.yml

@@ -37,8 +37,7 @@ services:
     container_name: mqtt_metrics_collector
     restart: unless-stopped
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:1.0.beta1
     build:
       context: ./wis2box-mqtt-metrics-collector
@@ -77,7 +76,7 @@ services:
     <<: *logging
     container_name: grafana
     env_file:
-      - dev.env
+      - wis2box.env
     image: grafana/grafana-oss:9.0.3
     volumes:
       - ./grafana/dashboards:/etc/grafana/provisioning/dashboards

docker-compose.yml

@@ -4,8 +4,7 @@ services:
     image: nginx:alpine
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       - wis2box-ui
     volumes:
@@ -16,8 +15,7 @@ services:
     image: ghcr.io/wmo-im/wis2box-ui:0.5.3
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       wis2box-api:
         condition: service_healthy
@@ -35,8 +33,7 @@ services:
 #    image: ghcr.io/wmo-im/wis2box-ui-admin:latest
 #    restart: always
 #    env_file:
-#      - default.env
-#      - dev.env
+#      - wis2box.env
 #    depends_on:
 #      - wis2box-api
 
@@ -45,8 +42,7 @@ services:
     image: wmoim/wis2box-api:wis2box-1.0b5
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -64,8 +60,7 @@ services:
     memswap_limit: 512m
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     command: server --console-address ":9001" /data
     # in a production-setup minio needs to be
     volumes:
@@ -114,8 +109,7 @@ services:
     build:
       context: ./wis2box-broker
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
 
   wis2box-management:
     container_name: wis2box-management
@@ -127,8 +121,7 @@ services:
       context: ./wis2box-management
     #user: wis2box:wis2box
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     volumes:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
     depends_on:
@@ -149,8 +142,7 @@ services:
     image: ghcr.io/wmo-im/wis2box-auth:latest
     restart: always
     env_file:
-      - default.env
-      - dev.env
+      - wis2box.env
     volumes:
       - auth-data:/data/wis2box:rw
     depends_on:

docs/source/community/troubleshooting.rst

@@ -15,7 +15,7 @@ The wis2box logging displays the error:
     
     OSError: Missing data mappings: [Errno 2] No such file or directory: '/data/wis2box/data-mappings.yml'
 
-Check your dev.env and check value that was set for WIS2BOX_HOST_DATADIR
+Check your wis2box.env and check value that was set for WIS2BOX_HOST_DATADIR
 
 .. code-block:: bash
     
@@ -70,8 +70,7 @@ The Access Key Id you provided does not exist in our records
 ------------------------------------------------------------
 
 If you see this error when uploading data to the wis2box-incoming storage, you have provided the wrong username and/or password to access MinIO.
-Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` you have provided in your ``dev.env`` file. 
-The default username/password for MinIO is ``minio/minio123``.
+Check the values for ``WIS2BOX_STORAGE_USERNAME`` and ``WIS2BOX_STORAGE_PASSWORD`` set in your ``wis2box.env`` file. 
 
 Topic Hierarchy validation error: No plugins for ... in data mappings
 ---------------------------------------------------------------------

docs/source/reference/administration.rst

@@ -75,7 +75,7 @@ Changing default ports
 ^^^^^^^^^^^^^^^^^^^^^^
 
 The ``docker-compose.override.yml`` file provides definitions on utilized ports.  To change default
-ports, edit ``default.env``  before stopping and starting wis2box for changes to take effect.
+ports, edit ``wis2box.env``  before stopping and starting wis2box for changes to take effect.
 
 
 MQTT Quality of Service (QoS)

docs/source/reference/auth.rst

@@ -20,6 +20,8 @@ To add a token to the execution of a wis2box process, use the following command:
 
 To add a token to PUT/POST/DELETE requests to the stations collection, use the following command:
 
+.. code-block:: bash
+
     wis2box auth add-token --path collections/stations mystationupdatetoken
 
 Adding Access Control on topics

docs/source/reference/configuration.rst

@@ -9,30 +9,32 @@ a simple configuration that can be adjusted depending the user's needs and deplo
 Environment variables
 ---------------------
 
-wis2box configuration is driven primarily by a small set of environment variables. The runtime
-configuration is defined in the `Env`_ format in a plain text file named ``dev.env`` and ``default.env``.
+wis2box configuration is driven primarily by a set of environment variables. The runtime
+configuration is defined in the `Env`_ format in a plain text file named ``wis2box.env``. 
+An example is provided in ``wis2box_example.env``.
 
-Any values set in ``dev.env`` override the default environment variables in ``default.env``. For further / specialized
-configuration, see the sections below.
+You can either copy the example-file to ``wis2box.env`` and adjust the values to your needs or run the following command
+to create a new ``wis2box.env`` file by answering a few questions on the command line:
+
+.. code-block:: bash
+
+    python3 wis2box-create-config.py
+
+For further / specialized configuration, see the sections below.
 
 ``WIS2BOX_HOST_DATADIR``
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-The minimum required setting in ``dev.env`` is the ``WIS2BOX_HOST_DATADIR`` environment variable. Setting this
-value is **required** to map the wis2box data directory from the host system to the containers.
+The value of ``WIS2BOX_HOST_DATADIR`` maps the wis2box data directory from the host system to the containers.
 
-It is recommended to set this value to an absolute path on your system.
 
 Sections
 --------
 
 .. note::
 
    A reference configuration can always be found in the wis2box `GitHub`_ repository. The :ref:`quickstart`
-   uses a variant of ``wis2box.env`` with mappings to the test data, as an example. For complex installations,
-   it is recommended to start configuring wis2box by copying the example ``wis2box.env`` file and modifying
-   accordingly.
-
+   uses a variant of ``wis2box_example.env`` with mappings to the test data, as an example. 
 
 wis2box environment variables can be categorized via the following core sections:
 
@@ -64,7 +66,7 @@ The following environment variables can be used to configure `WIS2BOX_STORAGE`.
 
     WIS2BOX_STORAGE_TYPE=S3
     WIS2BOX_STORAGE_SOURCE=http://minio:9000
-    WIS2BOX_STORAGE_USERNAME=minio  # username for the storage-layer
+    WIS2BOX_STORAGE_USERNAME=wis2box  # username for the storage-layer
     WIS2BOX_STORAGE_PASSWORD=minio123  # password for the storage-layer
     WIS2BOX_STORAGE_INCOMING=wis2box-incoming  # name of the storage-bucket/folder for incoming files
     WIS2BOX_STORAGE_PUBLIC=wis2box-public  # name of the storage-bucket/folder for public files
@@ -178,7 +180,7 @@ Web application configuration provides the ability to customize web components.
 Other
 ^^^^^
 
-Additional directives provide various configurationscontrol of configuration options for the deployment of wis2box.
+Additional directives provide various configuration options for the deployment of wis2box.
 
 .. code-block:: bash
 
@@ -193,16 +195,13 @@ Additional directives provide various configurationscontrol of configuration opt
 
 A full configuration example can be found below:
 
-.. literalinclude:: ../../../examples/config/wis2box.env
-   :language: bash
-
-.. literalinclude:: ../../../default.env
+.. literalinclude:: ../../../wis2box_example.env
    :language: bash
 
 Docker Compose
 --------------
 
-The Docker Compose setup is driven from the resulting ``dev.env`` file created. For advanced cases and/or power users,
+The Docker Compose setup is driven from the resulting ``wis2box.env`` file created. For advanced cases and/or power users,
 updates can also be made to ``docker-compose.yml`` or ``docker-compose.override.yml`` (for changes to ports).
 
 .. _`Env`: https://en.wikipedia.org/wiki/Env

docs/source/reference/quickstart.rst

@@ -17,11 +17,11 @@ To download wis2box from source:
 
 The test enviroment file is provided in ``tests/test.env``.
 
-To run with the 'quickstart' configuration, copy this file to ``dev.env`` in your working directory:
+To run with the 'quickstart' configuration, copy this file to ``wis2box.env`` in your working directory:
 
 .. code-block:: bash
 
-    cp tests/test.env dev.env
+    cp tests/test.env wis2box.env
 
 
 Build and update wis2box:

docs/source/reference/running/environment.rst

@@ -15,5 +15,5 @@ directory for all data managed in wis2box.
 
 The default enviroment variables are below.
 
-.. literalinclude:: ../../../../default.env
+.. literalinclude:: ../../../../wis2box_example.env
    :language: bash