Docker2

.github/workflows/test-code-quality.yml

@@ -29,19 +29,21 @@ jobs:
         uses: super-linter/[email protected]
         env:
           VALIDATE_ALL_CODEBASE: false  # only validate changed or updated files
-          #VALIDATE_MARKDOWN: false
+          VALIDATE_BASH_EXEC: false
+          #VALIDATE_CHECKOV: true
+          #VALIDATE_DOCKERFILE_HADOLINT: false
+          #VALIDATE_GITHUB_ACTIONS: false
           VALIDATE_JSCPD: false  # disable, copy paste detection fails when python decorators are used
           VALIDATE_JAVASCRIPT_PRETTIER: false
           VALIDATE_JAVASCRIPT_STANDARD: false
-          # VALIDATE_GITHUB_ACTIONS: false
+          #VALIDATE_MARKDOWN: false
+          #VALIDATE_OPENAPI: false
           VALIDATE_PYTHON_BLACK: false
           VALIDATE_PYTHON_PYLINT: false
           VALIDATE_PYTHON_ISORT: false
           #VALIDATE_PYTHON_FLAKE8: false
           VALIDATE_PYTHON_MYPY: false
           VALIDATE_PYTHON_RUFF: false
           VALIDATE_SHELL_SHFMT: false
-          # VALIDATE_CHECKOV: false
-          # VALIDATE_OPENAPI: false
           # To report GitHub Actions status checks
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-docker.yml

@@ -0,0 +1,81 @@
+name: test-docker-image
+
+on:
+  [ push, pull_request ]
+
+permissions:
+  contents: read
+  packages: write
+  issues: write
+  pull-requests: write
+
+jobs:
+  test-docker-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Build test environment
+        working-directory: docker/tests
+        run: |
+          # create user for wis2downloader
+          sudo usermod -aG docker "$(whoami)"
+          docker compose build  # build containers
+          mkdir ./data  # make sure data directory exists
+          sudo chmod 770 ./data  # update permissions so group read / write
+      - name: Run containers
+        working-directory: docker/tests
+        run: |  
+          DOCKER_GID="$(getent group docker | cut -d: -f3)"
+          export DOCKER_GID
+          docker compose up -d
+      - name: Run CLI tests
+        working-directory: docker/tests
+        run: |
+          docker logs subscriber
+          echo "Testing adding subscription"
+          # test adding a subscription
+          docker exec subscriber  bash -c "source /home/wis2downloader/.venv/bin/activate && wis2downloader add-subscription --topic cache/a/wis2/+/services/#" 
+          # test listing subscriptions
+          echo "Testing listing subscriptions"         
+          docker exec subscriber bash -c "source /home/wis2downloader/.venv/bin/activate && wis2downloader list-subscriptions"
+          # publish a test message
+          echo "Publishing test message"
+          docker exec publisher pywis-pubsub publish --topic cache/a/wis2/my-centre/services/downloader \
+            --config /pywis-pubsub/config/config.yml \
+            -i test -u "http://subscriber:5000/openapi"
+          sleep 1s
+          echo "Verifying data downloaded"
+          # cat file contents (check the published file has been downloaded)
+          cat "./data/$(date +'%Y')/$(date +'%m')/$(date +'%d')/cache/a/wis2/my-centre/services/downloader/openapi.bin"
+          echo "Testing removing subscription"
+          # test deleting subscriptions
+          docker exec subscriber bash -c "source /home/wis2downloader/.venv/bin/activate && wis2downloader remove-subscription --topic cache/a/wis2/+/services/#"
+      - name: Run API tests
+        working-directory: docker/tests
+        run: |          
+          # get metrics
+          curl http://localhost:5000/metrics          
+          # test adding a subscription          
+          curl -X 'POST' \
+            'http://localhost:5000/subscriptions' \
+            -H 'accept: application/json' \
+            -H 'Content-Type: application/json' \
+            -d '{
+                "topic": "cache/a/wis2/+/services/#"
+            }'           
+          # test listing subscriptions
+          curl http://localhost:5000/subscriptions          
+          # publish a test message
+          docker exec publisher pywis-pubsub publish --topic cache/a/wis2/my-centre/services/downloader \
+            --config /pywis-pubsub/config/config.yml \
+            -i test -u "http://subscriber:5000/metrics"
+          sleep 1s
+          # cat file contents (check the published file has been downloaded)
+          cat "./data/$(date +'%Y')/$(date +'%m')/$(date +'%d')/cache/a/wis2/my-centre/services/downloader/metrics.bin"
+          # test deleting subscriptions
+          curl -X DELETE http://localhost:5000/subscriptions/cache/a/wis2/%2B/services/%23
+      - name: Shutdown
+        working-directory: docker/tests
+        run: |
+          docker compose down

.github/workflows/unit-tests.yml

@@ -31,7 +31,7 @@ jobs:
             "broker_protocol": "websockets",
             "broker_username": "everyone",
             "download_workers": 1,
-            "download_dir": "downloads",
+            "download_dir": "downloads",  
             "flask_host": "0.0.0.0",
             "flask_port": 5050,
             "log_level": "DEBUG",
@@ -51,8 +51,13 @@ jobs:
       - name: Install requirements 📦
         run: |
           python3 -m pip install --upgrade pip 
-          pip3 install . 
+          pip3 install --no-cache .
           pip3 install -r requirements-dev.txt
       - name: run tests ⚙️
         run: |
+          python <<!
+          from wis2downloader.app import app
+          from wis2downloader.downloader import DownloadWorker
+          quit()
+          !
           pytest

README.md

@@ -21,14 +21,10 @@ The WIS2 Downloader is a Flask-based Python application that allows you to conne
 
 ### 1. Installation
 
-**NOTE**: The downloader has not yet been uploaded to PyPI and needs to be installed directly from GitHub:
-
 ```bash
-pip install https://github.com/wmo-im/wis2downloader/archive/main.zip
+python -m pip install wis2downloader
 ```
 
-This will install the version from the main development branch.
-
 ### 2. Configuration
 
 Create a file `config.json` in your local directory that conforms with the following schema:

docker/Dockerfile

@@ -0,0 +1,67 @@
+FROM python:3.12-slim-bookworm
+
+SHELL ["/bin/bash", "-c"]
+
+# default ENV / config
+ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr"
+ENV DOWNLOAD_BROKER_PORT 443
+ENV DOWNLOAD_BROKER_USERNAME "everyone"
+ENV DOWNLOAD_BROKER_PASSWORD "everyone"
+ENV DOWNLOAD_BROKER_TRANSPORT "websockets"
+ENV DOWNLOAD_DIR "/home/wis2downloader/app/data/downloads"
+ENV DOWNLOAD_MIN_FREE_SPACE_GB 1
+ENV DOWNLOAD_RETENTION_PERIOD_HOURS 24
+ENV DOWNLOAD_VALIDATE_TOPICS "false"
+ENV DOWNLOAD_WORKERS 8
+ENV LOG_PATH "/home/wis2downloader/app/logs"
+ENV WIS2DOWNLOADER_CONFIG "/home/wis2downloader/app/config/config.json"
+ENV USER_ID 12135
+
+# Update, upgrade packages and install / clean up
+RUN apt-get update && \
+    apt-get upgrade && \
+    apt-get install -y gettext-base=0.21-12 curl=7.88.1-10+deb12u6 cron=3.0pl1-162 git=1:2.39.2-1.1 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Now setup python env and default user
+RUN useradd -l -u "$USER_ID" wis2downloader
+
+USER wis2downloader
+WORKDIR /home/wis2downloader
+
+USER wis2downloader
+
+RUN python3.12 -m venv /home/wis2downloader/.venv && \
+    echo "source /home/wis2downloader/.venv/bin/activate" >> .bashrc && \
+    echo "" >> .bashrc
+
+# install python dependencies
+RUN source /home/wis2downloader/.venv/bin/activate && \
+    python -m pip install --no-cache-dir gunicorn==23.0.0 requests==2.32.3 && \
+    python -m pip install --no-cache-dir pyopenssl==24.2.1 --upgrade && \
+    python -m pip install --no-cache-dir wis2downloader==0.3.0 # git+https://github.com/wmo-im/wis2downloader@docker2
+
+# copy config and entrypoint to the Docker image
+COPY config/. /home/wis2downloader/app/config
+COPY entrypoint.sh /home/wis2downloader/app/entrypoint.sh
+COPY clean_downloads.cron /home/wis2downloader/app/clean_downloads.cron
+COPY clean_downloads.py /home/wis2downloader/app/clean_downloads.py
+
+USER root
+RUN chown -R wis2downloader /home/wis2downloader/app && \
+    chmod +x /home/wis2downloader/app/entrypoint.sh && \
+    chmod 600 /home/wis2downloader/app/clean_downloads.py && \
+    chmod 600 /home/wis2downloader/app/clean_downloads.cron
+
+USER wis2downloader
+# Set the working directory to /app
+WORKDIR /home/wis2downloader
+RUN crontab ./app/clean_downloads.cron
+
+# Add healthcheck
+HEALTHCHECK --interval=1m --timeout=3s \
+  CMD curl -f http://localhost:5000/subscriptions || exit 1
+
+ENTRYPOINT [ "/home/wis2downloader/app/entrypoint.sh" ]
+# Run wis2downloader when the container launches
+CMD ["/bin/bash", "-c", "gunicorn --bind 0.0.0.0:5000 --workers 1 wis2downloader.app:app"]

docker/clean_downloads.cron

@@ -0,0 +1 @@
+*/10 * * * * source /home/wis2downloader/.venv/bin/activate && python /home/wis2downloader/app/clean_downloads.py > /proc/1/fd/1 2>/proc/1/fd/2

docker/clean_downloads.py

@@ -0,0 +1,40 @@
+import os
+import time
+
+# get download dir from environment variable
+download_dir = os.environ.get('DOWNLOAD_DIR', '/home/wis2downloader/app/data/downloads')  # noqa
+# get retention period from environment variable
+retention_period_hours = int(os.environ.get('DOWNLOAD_RETENTION_PERIOD_HOURS', None))  # noqa
+
+
+def clean_directory(directory):
+    # get the current time
+    current_time = time.time()
+
+    files_removed = 0
+    directories_removed = 0
+    # loop through the files in the directory, including subdirectories
+    for file in os.listdir(directory):
+        # get the full path of the file
+        file_path = os.path.join(directory, file)
+        # check if the path is a file or a directory
+        if os.path.isfile(file_path):
+            # get the time the file was last modified
+            file_time = os.path.getmtime(file_path)
+            # check if the file is older than the retention period
+            if current_time - file_time > retention_period_hours * 3600:
+                os.remove(file_path)
+                files_removed += 1
+        elif os.path.isdir(file_path):
+            # recursively clean the directory
+            clean_directory(file_path)
+            # if the directory is empty, remove it
+            if not os.listdir(file_path):
+                os.rmdir(file_path)
+                directories_removed += 1
+    print(f'CLEANER: removed {files_removed} old files and {directories_removed} empty directories')  # noqa
+
+
+# start cleaning from the download directory
+if retention_period_hours is not None:
+    clean_directory(download_dir)

docker/config/config.template

@@ -0,0 +1,18 @@
+{
+    "broker_hostname": "${DOWNLOAD_BROKER_HOST}",
+    "broker_port": ${DOWNLOAD_BROKER_PORT},
+    "broker_username": "${DOWNLOAD_BROKER_USERNAME}",
+    "broker_password":  "${DOWNLOAD_BROKER_PASSWORD}",
+    "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}",
+    "download_workers": ${DOWNLOAD_WORKERS},
+    "min_free_space": ${DOWNLOAD_MIN_FREE_SPACE_GB},
+    "validate_topics": ${DOWNLOAD_VALIDATE_TOPICS},
+    "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json",
+    "download_dir": "${DOWNLOAD_DIR}",
+    "log_level": "INFO",
+    "base_url": "http://localhost:5000",
+    "flask_host": "0.0.0.0",
+    "flask_port": 5000,
+    "save_logs": false,
+    "log_path": "logs"
+}

docker/entrypoint.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+echo "Download directory in container: $DOWNLOAD_DIR"
+# ensure DOWNLOAD_DIR exists
+if [ ! -d "$DOWNLOAD_DIR" ]; then
+    echo "Creating download directory: $DOWNLOAD_DIR"
+    mkdir -p "$DOWNLOAD_DIR"
+fi
+
+envsubst < /home/wis2downloader/app/config/config.template > /home/wis2downloader/app/config/config.json
+
+# if session-info.json does not exists in $DOWNLOAD_DIR, create it
+if [ ! -f "$DOWNLOAD_DIR/.session-info.json" ]; then
+    echo "Creating .session-info.json"
+    echo "{" > "$DOWNLOAD_DIR/.session-info.json"
+    echo '  "topics": {},' >> "$DOWNLOAD_DIR/.session-info.json"
+    # generate a random string for client_id
+    echo "Generating random client_id"
+    client_id=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1)
+    echo "  \"client_id\": \"wis2box-wis2downloader-${client_id}\"" >> "$DOWNLOAD_DIR/.session-info.json"
+    echo "}" >> "$DOWNLOAD_DIR/.session-info.json"
+fi
+
+# print the config
+echo "Config:"
+cat /home/wis2downloader/app/config/config.json
+echo "Initial session info:"
+cat "$DOWNLOAD_DIR/.session-info.json"
+# activate python env
+# shellcheck source=/dev/null
+source /home/wis2downloader/.venv/bin/activate
+exec "$@"

docker/tests/containers/broker/Dockerfile

@@ -0,0 +1,17 @@
+#checkov:skip=CKV_DOCKER_2:No healthcheck, dockerfile only used in testing
+#checkov:skip=CKV_DOCKER_3
+FROM eclipse-mosquitto:2.0.18
+COPY entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+COPY ./config /mosquitto/config
+
+RUN chown mosquitto:mosquitto /mosquitto/config/password.txt && \
+    chmod 700 /mosquitto/config/password.txt
+
+RUN chown mosquitto:mosquitto /mosquitto/config/acl.conf && \
+    chmod 700 /mosquitto/config/acl.conf
+
+RUN chown mosquitto:mosquitto /mosquitto/config/mosquitto.conf && \
+    chmod 700 /mosquitto/config/mosquitto.conf
+

docker/tests/containers/broker/config/acl.conf

@@ -0,0 +1,10 @@
+user everyone
+topic read public/#
+topic read origin/#
+
+user publisher
+topic readwrite origin/#
+topic readwrite public/#
+topic readwrite internal/#
+topic readwrite fs/#
+topic read $SYS/#

docker/tests/containers/broker/config/mosquitto.conf

@@ -0,0 +1,2 @@
+listener 1883
+allow_anonymous true

docker/tests/containers/broker/entrypoint.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+/usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf

docker/tests/containers/publisher/Dockerfile

@@ -0,0 +1,13 @@
+#checkov:skip=CKV_DOCKER_2:No healthcheck, dockerfile only used in testing
+#checkov:skip=CKV_DOCKER_3
+FROM python:3.11-slim-bookworm
+
+# Update / upgrade
+RUN apt-get update && \
+    apt-get upgrade
+
+# install python dependencies
+RUN python -m pip install --no-cache-dir pywis-pubsub==0.7.2
+COPY ./config /pywis-pubsub/config
+
+CMD ["bash"]

docker/tests/containers/publisher/config/config.yml

@@ -0,0 +1,9 @@
+# RFC1738 URL
+broker: mqtt://broker:1883
+# clean mqtt session on connection. Set to false to start from server saved offset
+clean_session: true
+# whether to verify data and message
+verify_data: true
+validate_message: true
+
+