From d90469389a6a58d4458c1190ed785d95aa6d14b3 Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 6 Jun 2024 11:19:41 +0200 Subject: [PATCH 01/21] add GTS-headers using mapping csv --- tests/data/gts_headers_mapping.csv | 8 +++++ ...synop_202308.txt => SICG20FCBB_202308.txt} | 0 tests/integration/test_workflow.py | 3 ++ wis2box-management/wis2box/data/base.py | 10 ++++++- wis2box-management/wis2box/data_mappings.py | 14 ++++++++- wis2box-management/wis2box/handler.py | 6 ++-- wis2box-management/wis2box/pubsub/message.py | 7 ++++- .../wis2box/pubsub/subscribe.py | 29 +++++++++++++++++-- 8 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 tests/data/gts_headers_mapping.csv rename tests/data/observations/congo/{synop_202308.txt => SICG20FCBB_202308.txt} (100%) diff --git a/tests/data/gts_headers_mapping.csv b/tests/data/gts_headers_mapping.csv new file mode 100644 index 00000000..e2a24005 --- /dev/null +++ b/tests/data/gts_headers_mapping.csv @@ -0,0 +1,8 @@ +string_in_filepath,ttaaii,cccc +ISIN21DAMM,ISIN21,DAMM +ISMD01LIIB,ISMD01,LIBB +ISMD02LIIB,ISMD02,LIBB +SMRO01YRBK,SMRO01,YRBK +IOBH01LFPW,IOBH01,LFPW +ISSA05LFPW,ISSA05,LFPW +SICG20FCBB,SICG20,FCBB \ No newline at end of file diff --git a/tests/data/observations/congo/synop_202308.txt b/tests/data/observations/congo/SICG20FCBB_202308.txt similarity index 100% rename from tests/data/observations/congo/synop_202308.txt rename to tests/data/observations/congo/SICG20FCBB_202308.txt diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 9661b538..e224ddd2 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -292,6 +292,9 @@ def test_message_api(): assert props['content']['size'] == 253 assert props['content']['encoding'] == 'base64' assert props['content']['value'] is not None + assert 'gts' in props + assert props['gts']['ttaaii'] == 'SICG20' + assert props['gts']['cccc'] == 'FCBB' link_rel = msg['links'][0] diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index 340cf56b..0bcc05b0 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -60,6 +60,13 @@ def __init__(self, defs: dict) -> None: self.buckets = defs.get('buckets', ()) self.output_data = {} self.discovery_metadata = {} + self.gts = None + if defs.get('gts_ttaaii') and defs.get('gts_cccc'): + self.gts = { + 'ttaaii': defs.get('gts_ttaaii'), + 'cccc': defs.get('gts_cccc') + } + # if discovery_metadata: # self.setup_discovery_metadata(discovery_metadata) @@ -152,7 +159,8 @@ def notify(self, identifier: str, storage_path: str, wis_message = WISNotificationMessage( f"{metadata_id.replace('urn:wmo:md:','')}/{identifier}", metadata_id, storage_path, datetime_, geometry, - wigos_station_identifier, operation) + wigos_station_identifier, self.gts, + operation) # load plugin for public broker defs = { diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index 37300d93..8d3185a3 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -82,6 +82,7 @@ def get_data_mappings() -> dict: def validate_and_load(path: str, data_mappings: dict = None, + gts_mappings: dict = None, file_type: str = None ) -> Tuple[str, Tuple[Any]]: """ @@ -134,8 +135,15 @@ def validate_and_load(path: str, LOGGER.debug(f'Adding plugin definition for {file_type}') + if gts_mappings: + for key in gts_mappings.keys(): + # check if string defined by key is contained in path + if key in path: + gts = gts_mappings[key] + break + def data_defs(plugin): - return { + data_defs = { 'metadata_id': metadata_id, 'íncoming_filepath': path, 'topic_hierarchy': topic_hierarchy, @@ -146,6 +154,10 @@ def data_defs(plugin): 'notify': plugin.get('notify', False), 'format': file_type } + if gts_mappings: + data_defs['gts_ttaaii'] = gts['ttaaii'] + data_defs['gts_cccc'] = gts['cccc'] + return data_defs plugins_ = [load_plugin('data', data_defs(p), data_mappings) for p in plugins[file_type]] diff --git a/wis2box-management/wis2box/handler.py b/wis2box-management/wis2box/handler.py index 1230b3af..b0edbe21 100644 --- a/wis2box-management/wis2box/handler.py +++ b/wis2box-management/wis2box/handler.py @@ -37,8 +37,8 @@ class Handler: def __init__(self, filepath: str, - metadata_id: str = None, - data_mappings: dict = None) -> None: + data_mappings: dict = None, + gts_mappings: dict = None) -> None: self.filepath = filepath self.plugins = () self.input_bytes = None @@ -61,7 +61,7 @@ def __init__(self, filepath: str, raise NotHandledError(msg) try: self.metadata_id, self.plugins = validate_and_load( - self.filepath, data_mappings, self.filetype) + self.filepath, data_mappings, gts_mappings, self.filetype) except Exception as err: msg = f'Path validation error: {err}' # errors in public storage are not handled diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py index d036a630..edba090c 100644 --- a/wis2box-management/wis2box/pubsub/message.py +++ b/wis2box-management/wis2box/pubsub/message.py @@ -126,7 +126,8 @@ def _generate_checksum(self, bytes, algorithm: SecureHashAlgorithms) -> str: # class WISNotificationMessage(PubSubMessage): def __init__(self, identifier: str, metadata_id: str, filepath: str, - datetime_: str, geometry=None, wigos_station_identifier=None, + datetime_: str, geometry=None, + wigos_station_identifier=None, gts: dict = None, operation: str = 'create') -> None: super().__init__('wis2-notification-message', identifier, @@ -186,6 +187,10 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, if metadata_id is not None: self.message['properties']['metadata_id'] = metadata_id + # if gts-dictionary is provided, include it in the message properties + if gts is not None: + self.message['properties']['gts'] = gts + if self.length < 4096: LOGGER.debug('Including data inline via properties.content') content_value = base64.b64encode(self.filebytes) diff --git a/wis2box-management/wis2box/pubsub/subscribe.py b/wis2box-management/wis2box/pubsub/subscribe.py index 8595f5e1..f9ee95dd 100644 --- a/wis2box-management/wis2box/pubsub/subscribe.py +++ b/wis2box-management/wis2box/pubsub/subscribe.py @@ -36,7 +36,8 @@ from wis2box.data_mappings import get_data_mappings from wis2box.data.message import MessageData -from wis2box.env import (DOCKER_BROKER, STORAGE_SOURCE, STORAGE_ARCHIVE) +from wis2box.env import (DATADIR, DOCKER_BROKER, + STORAGE_SOURCE, STORAGE_ARCHIVE) from wis2box.handler import Handler, NotHandledError import wis2box.metadata.discovery as discovery_metadata from wis2box.plugin import load_plugin, PLUGINS @@ -46,10 +47,33 @@ LOGGER = logging.getLogger(__name__) +def get_gts_mappings(): + # read gts mappings from CSV file in DATADIR + gts_mappings = {} + mapping_file = f'{DATADIR}/gts_headers_mappings.csv' + try: + with open(mapping_file, 'r') as f: + for line in f: + key = line.strip().split(',')[0] + if key == 'string_in_filepath': + continue + ttaaii = line.strip().split(',')[1] + cccc = line.strip().split(',')[2] + value = {'ttaaii': ttaaii, 'cccc': cccc} + gts_mappings[key] = value + LOGGER.info(f'GTS mapping: string_in_filepath={key}, {value}') + except FileNotFoundError: + LOGGER.info(f'To add GTS headers, please create {mapping_file}') + except Exception as err: + LOGGER.error(f'Error reading GTS mappings: {err}') + return gts_mappings + + class WIS2BoxSubscriber: def __init__(self, broker): self.data_mappings = get_data_mappings() + self.gts_mappings = get_gts_mappings() self.broker = broker self.broker.bind('on_message', self.on_message_handler) self.broker.sub('wis2box/#') @@ -59,7 +83,8 @@ def handle(self, filepath): LOGGER.info(f'Processing {filepath}') # load handler handler = Handler(filepath=filepath, - data_mappings=self.data_mappings) + data_mappings=self.data_mappings, + gts_mappings=self.gts_mappings) if handler.handle(): LOGGER.debug('Data processed') for plugin in handler.plugins: From d9397d44b214ede2aa37493af8391e1f1bc5e09f Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 6 Jun 2024 14:05:29 +0200 Subject: [PATCH 02/21] fix typo, only check gts-header when notify=true, update version --- wis2box-management/wis2box/__init__.py | 2 +- wis2box-management/wis2box/data_mappings.py | 20 +++++++++---------- .../wis2box/pubsub/subscribe.py | 6 +++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/wis2box-management/wis2box/__init__.py b/wis2box-management/wis2box/__init__.py index fa8a2f53..2b482da1 100644 --- a/wis2box-management/wis2box/__init__.py +++ b/wis2box-management/wis2box/__init__.py @@ -19,7 +19,7 @@ # ############################################################################### -__version__ = '1.0b7' +__version__ = '1.0b8' import click diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index 8d3185a3..e8c91b02 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -135,14 +135,8 @@ def validate_and_load(path: str, LOGGER.debug(f'Adding plugin definition for {file_type}') - if gts_mappings: - for key in gts_mappings.keys(): - # check if string defined by key is contained in path - if key in path: - gts = gts_mappings[key] - break - def data_defs(plugin): + notify = plugin.get('notify', False) data_defs = { 'metadata_id': metadata_id, 'íncoming_filepath': path, @@ -151,12 +145,16 @@ def data_defs(plugin): 'pattern': plugin['file-pattern'], 'template': plugin.get('template'), 'buckets': plugin.get('buckets', ()), - 'notify': plugin.get('notify', False), + 'notify': notify, 'format': file_type } - if gts_mappings: - data_defs['gts_ttaaii'] = gts['ttaaii'] - data_defs['gts_cccc'] = gts['cccc'] + if notify and gts_mappings: + for key in gts_mappings.keys(): + # check if string defined by key is contained in path + if key in path: + data_defs['gts_ttaaii'] = gts_mappings['ttaaii'] + data_defs['gts_cccc'] = gts_mappings['cccc'] + break return data_defs plugins_ = [load_plugin('data', data_defs(p), data_mappings) diff --git a/wis2box-management/wis2box/pubsub/subscribe.py b/wis2box-management/wis2box/pubsub/subscribe.py index f9ee95dd..dd3724ea 100644 --- a/wis2box-management/wis2box/pubsub/subscribe.py +++ b/wis2box-management/wis2box/pubsub/subscribe.py @@ -50,9 +50,9 @@ def get_gts_mappings(): # read gts mappings from CSV file in DATADIR gts_mappings = {} - mapping_file = f'{DATADIR}/gts_headers_mappings.csv' + mapping_file = 'gts_headers_mapping.csv' try: - with open(mapping_file, 'r') as f: + with open(f'{DATADIR}/{mapping_file}', 'r') as f: for line in f: key = line.strip().split(',')[0] if key == 'string_in_filepath': @@ -63,7 +63,7 @@ def get_gts_mappings(): gts_mappings[key] = value LOGGER.info(f'GTS mapping: string_in_filepath={key}, {value}') except FileNotFoundError: - LOGGER.info(f'To add GTS headers, please create {mapping_file}') + LOGGER.warning(f'To add GTS headers, please create {mapping_file} in WIS2BOX_HOST_DATADIR') # noqa except Exception as err: LOGGER.error(f'Error reading GTS mappings: {err}') return gts_mappings From a35cadbbd196d27d85f5348871c85d4851e6de4f Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 6 Jun 2024 14:45:41 +0200 Subject: [PATCH 03/21] add more sleep and logging to tests --- .github/workflows/tests-docker.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 621dc6db..2aa9de29 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -39,11 +39,11 @@ jobs: run: | python3 wis2box-ctl.py start python3 wis2box-ctl.py status -a - docker logs wis2box-management - - name: setup wis2box-management ⚙️ + - name: show environment and check logs ⚙️ run: | - sleep 20 + sleep 30 python3 wis2box-ctl.py execute wis2box environment show + docker logs wis2box-management - name: populate stations from CSV 📡 run: | python3 wis2box-ctl.py execute wis2box metadata station publish-collection @@ -176,6 +176,7 @@ jobs: - name: sleep 30 seconds then run integration tests ⚙️ run: | sleep 30 + docker logs wis2box-management pytest -s tests/integration - name: run flake8 ⚙️ run: | From 274e56c09ab0225f3dd5cbe1b528c5711b7dab31 Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 6 Jun 2024 15:10:31 +0200 Subject: [PATCH 04/21] fix missing key --- wis2box-management/wis2box/data_mappings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index e8c91b02..757f9d54 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -152,8 +152,8 @@ def data_defs(plugin): for key in gts_mappings.keys(): # check if string defined by key is contained in path if key in path: - data_defs['gts_ttaaii'] = gts_mappings['ttaaii'] - data_defs['gts_cccc'] = gts_mappings['cccc'] + data_defs['gts_ttaaii'] = gts_mappings[key]['ttaaii'] + data_defs['gts_cccc'] = gts_mappings[key]['cccc'] break return data_defs From a93b829221d7f2f83c48fe04336ad793f9d4061c Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 6 Jun 2024 16:36:28 +0200 Subject: [PATCH 05/21] adding documentation --- docs/source/index.rst | 1 + docs/source/user/gts-headers-in-wis2.rst | 63 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 docs/source/user/gts-headers-in-wis2.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index c721268c..742c22dd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -33,6 +33,7 @@ The user guide helps you setup your own wis2box instance. user/getting-started user/setup user/data-ingest + user/gts-headers-in-wis2 user/public-services-setup user/downloading-data diff --git a/docs/source/user/gts-headers-in-wis2.rst b/docs/source/user/gts-headers-in-wis2.rst new file mode 100644 index 00000000..fbf56576 --- /dev/null +++ b/docs/source/user/gts-headers-in-wis2.rst @@ -0,0 +1,63 @@ +.. _gts-headers-in-wis2: + +Optional: adding GTS headers to WIS2 notifications during the transition period +=============================================================================== + +Overview +-------- + +This section provides guidance how to add GTS headers to your WIS2 notifications. + +By adding GTS headers to your WIS2 notifications, you can stop your MSS and still have your data available on the GTS during the transition period. + +To enable the WIS2 to GTS Gateway to correctly identify the data to be republished on the GTS, you need to include the GTS property in the WIS2 Notification Message as follows: + +```json +{ + "properties": { + "gts": { + "ttaaii": "FTAE31", + "cccc": "VTBB" + } + } +} +``` + +The wis2box can add these to your WIS2 Notifications automatically, provided you specify the additional file `gts_headers_mappings.json` that contains the required information to map the GTS headers to the incoming filenames. + +Note that this is optional and only required if you want to turn off the system responsible for sending data to the GTS during the transition period. + +gts_headers_mappings.json +------------------------- + +If you want to add GTS headers to your WIS2 notifications, you need to create a JSON file that maps the GTS headers to the incoming filenames. + +The JSON file should be named `gts_headers_mappings.json` and should be placed in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable. + +The CSV should contain three columns: `string_in_filepath`, `TTAAii`, and `CCCC`. + +Example content for `gts_headers_mappings.json`: + +``` +string_in_filepath,TTAAii,CCCC +ISMD01LIIB,ISMD01,LIBB +ISMD02LIIB,ISMD02,LIBB +``` + +In this example, whenever `ISMD01LIIB` or `ISMD02LIIB` is contained in the file-path of the incoming file, +the corresponding GTS-headers will be added to the WIS2 Notification Message as a dictionary in the `properties` field: + +```json +{ + "properties": { + "gts": { + "ttaaii": "ISMD01", + "cccc": "LIBB" + } + } +} +``` + +If the `gts_headers_mappings.json` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, the wis2box will not add any GTS headers to the WIS2 Notification Message. + + From 11ca57e7a0ac876a7c813d16143e5db3d645d647 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 17:13:59 +0200 Subject: [PATCH 06/21] wis2download test and wis2box downloader commands to manage topics --- .github/workflows/tests-docker.yml | 5 + docker-compose.monitoring.yml | 2 +- docker-compose.yml | 6 +- ...s2-downloader.json => wis2downloader.json} | 48 ++++---- nginx/nginx-ssl.conf | 6 +- nginx/nginx.conf | 6 +- prometheus/prometheus.yml | 4 +- tests/integration/test_workflow.py | 17 +++ tests/test.env | 10 +- wis2-downloader/config.template | 15 --- wis2box-create-config.py | 11 ++ wis2box-management/docker/entrypoint.sh | 10 +- wis2box-management/wis2box/__init__.py | 2 + wis2box-management/wis2box/downloader.py | 110 ++++++++++++++++++ wis2box.env.example | 10 ++ .../Dockerfile | 20 ++-- .../clean.cron | 0 {wis2-downloader => wis2downloader}/clean.py | 0 wis2downloader/config.template | 15 +++ .../entrypoint.sh | 13 ++- 20 files changed, 241 insertions(+), 69 deletions(-) rename grafana/dashboards/{wis2-downloader.json => wis2downloader.json} (95%) delete mode 100644 wis2-downloader/config.template create mode 100644 wis2box-management/wis2box/downloader.py rename {wis2-downloader => wis2downloader}/Dockerfile (62%) rename {wis2-downloader => wis2downloader}/clean.cron (100%) rename {wis2-downloader => wis2downloader}/clean.py (100%) create mode 100644 wis2downloader/config.template rename {wis2-downloader => wis2downloader}/entrypoint.sh (50%) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index e10959e4..d62ae686 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -44,6 +44,11 @@ jobs: run: | sleep 30 python3 wis2box-ctl.py execute wis2box environment show + - name: downloader subscribe to topics + env: + SUBSCRIPTION_TOPIC: origin/a/wis2/+/data/core/# + run: | + python3 wis2box-ctl.py execute wis2box downloader add-subscription --topic $SUBSCRIPTION_TOPIC - name: populate stations from CSV 📡 run: | python3 wis2box-ctl.py execute wis2box metadata station publish-collection diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index c7f0dc64..2a25f4ee 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -114,7 +114,7 @@ services: <<: *logging web-proxy: <<: *logging - wis2-downloader: + wis2downloader: <<: *logging volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 4afb92b6..dce6636a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,9 +145,9 @@ services: depends_on: - wis2box-management - wis2-downloader: - container_name: wis2-downloader - build: ./wis2-downloader + wis2downloader: + container_name: wis2downloader + build: ./wis2downloader env_file: - wis2box.env volumes: diff --git a/grafana/dashboards/wis2-downloader.json b/grafana/dashboards/wis2downloader.json similarity index 95% rename from grafana/dashboards/wis2-downloader.json rename to grafana/dashboards/wis2downloader.json index a6136bcf..ec080d7c 100644 --- a/grafana/dashboards/wis2-downloader.json +++ b/grafana/dashboards/wis2downloader.json @@ -31,7 +31,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -85,7 +85,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -118,7 +118,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -168,7 +168,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "expr": "queue_size", @@ -183,7 +183,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -236,7 +236,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -255,7 +255,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -310,7 +310,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -329,7 +329,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -387,7 +387,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -402,7 +402,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -418,7 +418,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "code", "exemplar": false, @@ -438,7 +438,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -497,7 +497,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -512,7 +512,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "exemplar": false, @@ -528,7 +528,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "code", "exemplar": false, @@ -548,7 +548,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "description": "", "fieldConfig": { @@ -675,7 +675,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "expr": "sum(increase(downloaded_bytes_total[1m]))", @@ -691,7 +691,7 @@ { "datasource": { "type": "loki", - "uid": "P982945308D3682D1" + "uid": "P55348B596EBB51C3" }, "gridPos": { "h": 18, @@ -714,10 +714,10 @@ { "datasource": { "type": "loki", - "uid": "P982945308D3682D1" + "uid": "P55348B596EBB51C3" }, "editorMode": "builder", - "expr": "{container_name=\"wis2-downloader\"} != `Queue` != `GET` != `CLEAN`", + "expr": "{container_name=\"wis2downloader\"} != `Queue` != `GET` != `CLEAN`", "queryType": "range", "refId": "A" } @@ -729,7 +729,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "fieldConfig": { "defaults": { @@ -837,7 +837,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "PABAFD29CE247021E" }, "editorMode": "builder", "expr": "sum(increase(downloaded_files_total[1m]))", @@ -863,7 +863,7 @@ }, "timepicker": {}, "timezone": "", - "title": "wis2-downloader monitoring dashboard", + "title": "wis2downloader monitoring dashboard", "uid": "zB4GjVaIk", "version": 5, "weekStart": "" diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf index 64f9a044..81a38d16 100644 --- a/nginx/nginx-ssl.conf +++ b/nginx/nginx-ssl.conf @@ -59,12 +59,12 @@ rewrite ^/data(/.*)$ /wis2box-public$1 break; proxy_pass http://minio:9000; } - location /wis2-downloader { + location /wis2downloader { set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; - rewrite ^/wis2-downloader/(.*) /$1 break; - proxy_pass http://wis2-downloader:5000; + rewrite ^/wis2downloader/(.*) /$1 break; + proxy_pass http://wis2downloader:5000; } location /oapi { set $x_api_http_method $request_method; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 362e5887..fd19673f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -62,12 +62,12 @@ auth_request_set $auth_status $upstream_status; proxy_pass http://wis2box-api:80; } - location /wis2-downloader { + location /wis2downloader { set $x_api_http_method $request_method; auth_request /auth; auth_request_set $auth_status $upstream_status; - rewrite ^/wis2-downloader/(.*) /$1 break; - proxy_pass http://wis2-downloader:5000; + rewrite ^/wis2downloader/(.*) /$1 break; + proxy_pass http://wis2downloader:5000; } location /wis2box-webapp/ { auth_basic "Restricted Access"; diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 16d7d5c9..83bcc38f 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -26,8 +26,8 @@ scrape_configs: - targets: ['cadvisor:8080'] labels: alias: 'cadvisor' -- job_name: wis2-downloader +- job_name: wis2downloader metrics_path: /metrics scheme: http static_configs: - - targets: ['wis2-downloader:5000'] + - targets: ['wis2downloader:5000'] diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 9661b538..9b0c2a23 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -24,6 +24,8 @@ # .github/workflows/tests-docker.yml has been executed import csv +import os + from pathlib import Path from pywis_pubsub.validation import validate_message @@ -40,6 +42,21 @@ } +def test_wis2downloader(): + """Test if the wis2downloader has downloaded + the expected number of files in the download directory""" + + DOWNLOAD_DIR = DATADIR / 'downloads' + + # count the number of files received in the download directory + # over all subdirectories + total_files = 0 + for root, dirs, files in os.walk(DOWNLOAD_DIR): + print(f'Found {len(files)} files in {root}') + total_files += len(files) + + assert total_files == 385 + def test_metadata_station_cache(): """Test station metadata caching""" diff --git a/tests/test.env b/tests/test.env index 65b65037..1e23534f 100644 --- a/tests/test.env +++ b/tests/test.env @@ -60,4 +60,12 @@ MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 # wis2box webapp credentials WIS2BOX_WEBAPP_USERNAME=wis2box-user -WIS2BOX_WEBAPP_PASSWORD=wis2boxtest123 \ No newline at end of file +WIS2BOX_WEBAPP_PASSWORD=wis2boxtest123 + +# downloader settings, subscribe to local broker for test +DOWNLOAD_BROKER_HOST=mosquitto +DOWNLOAD_BROKER_PORT=1883 +DOWNLOAD_BROKER_USERNAME=everyone +DOWNLOAD_BROKER_PASSWORD=everyone +DOWNLOAD_BROKER_TRANSPORT=tcp +DOWNLOAD_MAX_VOLUME_MB=100 \ No newline at end of file diff --git a/wis2-downloader/config.template b/wis2-downloader/config.template deleted file mode 100644 index c5de79ac..00000000 --- a/wis2-downloader/config.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "broker_url": "${DOWNLOAD_BROKER_HOST}", - "broker_port": $DOWNLOAD_BROKER_PORT, - "username": "${DOWNLOAD_BROKER_USERNAME}", - "password": "${DOWNLOAD_BROKER_PASSWORD}", - "protocol": "${DOWNLOAD_BROKER_PROTOCOL}", - "topics": {}, - "download_dir": "${DOWNLOAD_DIR}", - "retention_period_hours": "${DOWNLOAD_RETENTION_PERIOD_HOURS}", - "flask_host": "${FLASK_HOST}", - "flask_port": "${FLASK_PORT}", - "download_workers": $DOWNLOAD_WORKERS, - "save_logs": false, - "log_dir": "logs" -} diff --git a/wis2box-create-config.py b/wis2box-create-config.py index 32be22df..c851c5d8 100644 --- a/wis2box-create-config.py +++ b/wis2box-create-config.py @@ -358,6 +358,17 @@ def create_wis2box_env(config_dir: str) -> None: fh.write('MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage\n') fh.write('MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1\n') fh.write('\n') + # downloader settings + fh.write('# downloader settings\n') + fh.write('DOWNLOAD_BROKER_HOST=globalbroker.meteo.fr\n') + fh.write('DOWNLOAD_BROKER_PORT=8883\n') + fh.write('DOWNLOAD_BROKER_USERNAME=everyone\n') + fh.write('DOWNLOAD_BROKER_PASSWORD=everyone\n') + fh.write('# download transport mechanism (tcp or websockets)\n') + fh.write('DOWNLOAD_BROKER_TRANSPORT=tcp\n') + fh.write('# maximum MB in download directory\n') + fh.write('DOWNLOAD_MAX_VOLUME_MB=100\n') + fh.write('\n') print('*' * 80) print('The file wis2box.env has been created in the current directory.') diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh index 5d065f8b..8baf4c22 100755 --- a/wis2box-management/docker/entrypoint.sh +++ b/wis2box-management/docker/entrypoint.sh @@ -91,14 +91,14 @@ else # Add the token wis2box auth add-token --path collections/stations -y fi -# repeat for wis2-downloader -is_restricted=$(wis2box auth is-restricted-path --path wis2-downloader) +# repeat for wis2downloader +is_restricted=$(wis2box auth is-restricted-path --path wis2downloader) if [ "$is_restricted" = "True" ]; then - echo "wis2-downloader is restricted" + echo "wis2downloader is restricted" else - echo "restricting wis2-downloader" + echo "restricting wis2downloader" # Add the token - wis2box auth add-token --path wis2-downloader -y + wis2box auth add-token --path wis2downloader -y fi echo "END /entrypoint.sh" diff --git a/wis2box-management/wis2box/__init__.py b/wis2box-management/wis2box/__init__.py index fa8a2f53..481951b1 100644 --- a/wis2box-management/wis2box/__init__.py +++ b/wis2box-management/wis2box/__init__.py @@ -30,6 +30,7 @@ from wis2box.metadata import metadata from wis2box.pubsub import pubsub from wis2box.auth import auth +from wis2box.downloader import downloader @click.group() @@ -46,3 +47,4 @@ def cli(): cli.add_command(metadata) cli.add_command(auth) cli.add_command(pubsub) +cli.add_command(downloader) diff --git a/wis2box-management/wis2box/downloader.py b/wis2box-management/wis2box/downloader.py new file mode 100644 index 00000000..32e7ea0e --- /dev/null +++ b/wis2box-management/wis2box/downloader.py @@ -0,0 +1,110 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging + +import click +import requests + +from wis2box import cli_helpers + +LOGGER = logging.getLogger(__name__) + +DOWNLOAD_URL = 'http://wis2downloader:5000' + +@click.group() +def downloader(): + """Interact with the wis2downloader""" + pass + +@click.command('list-subscriptions') +@click.pass_context +@cli_helpers.OPTION_VERBOSITY +def list_subscriptions(ctx, verbosity): + """list all subscriptions""" + + # make a GET requests to http://:/subscriptions + try: + response = requests.get(f'{DOWNLOAD_URL}/subscriptions') + # check response status + if response.status_code == 200: + click.echo('Current subscriptions:') + click.echo(response.text) + else: + click.echo(f'Error: {response.status_code}') + click.echo(response.text) + except requests.exceptions.ConnectionError: + click.echo(f'Error: Connection refused') + click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + +@click.command('add-subscription') +@click.pass_context +@click.option('--topic', '-t', help='The topic to subscribe to', required=True) +def add_subscription(ctx, topic): + """add a subscription""" + + topic.replace('#', '%23') + topic.replace('+', '%2B') + # make a POST request to http://:/subscriptions + try: + response = requests.post(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') + # check response status + if response.status_code == 200: + click.echo('Subscription added') + click.echo('Current subscriptions:') + click.echo(response.text) + else: + click.echo('Subscription not added') + click.echo(f'Error: {response.status_code}') + click.echo(response.text) + except requests.exceptions.ConnectionError: + click.echo(f'Error: Connection refused') + click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + + +@click.command('remove-subscription') +@click.pass_context +@click.option('--topic', '-t', help='The topic to subscribe to', required=True) +def remove_subscription(ctx, topic): + """remove a subscription""" + + topic.replace('#', '%23') + topic.replace('+', '%2B') + # make a DELETE request to http://:/subscriptions + try: + response = requests.delete(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') + # check response status + if response.status_code == 200: + click.echo('Subscription deleted') + click.echo('Current subscriptions:') + click.echo(response.text) + else: + click.echo('Subscription not deleted') + click.echo(f'Error: {response.status_code}') + click.echo(response.text) + except requests.exceptions.ConnectionError: + click.echo(f'Error: Connection refused') + click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + +downloader.add_command(list_subscriptions) +downloader.add_command(add_subscription) +downloader.add_command(remove_subscription) + diff --git a/wis2box.env.example b/wis2box.env.example index 7194e3cd..674e2409 100644 --- a/wis2box.env.example +++ b/wis2box.env.example @@ -59,3 +59,13 @@ MINIO_NOTIFY_MQTT_BROKER_WIS2BOX=tcp://${WIS2BOX_BROKER_HOST}:${WIS2BOX_BROKER_P MINIO_NOTIFY_MQTT_TOPIC_WIS2BOX=wis2box/storage MINIO_NOTIFY_MQTT_QOS_WIS2BOX=1 +# downloader settings +DOWNLOAD_BROKER_HOST=globalbroker.meteo.fr +DOWNLOAD_BROKER_PORT=443 +DOWNLOAD_BROKER_USERNAME=everyone +DOWNLOAD_BROKER_PASSWORD=everyone +# download transport mechanism (tcp or websockets) +DOWNLOAD_BROKER_TRANSPORT=websockets +# maximum MB in download directory +DOWNLOAD_MAX_VOLUME_MB=100 + diff --git a/wis2-downloader/Dockerfile b/wis2downloader/Dockerfile similarity index 62% rename from wis2-downloader/Dockerfile rename to wis2downloader/Dockerfile index a5ee6fd1..1cb30b4f 100644 --- a/wis2-downloader/Dockerfile +++ b/wis2downloader/Dockerfile @@ -10,17 +10,17 @@ ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr" ENV DOWNLOAD_BROKER_PORT 443 ENV DOWNLOAD_BROKER_USERNAME "everyone" ENV DOWNLOAD_BROKER_PASSWORD "everyone" -ENV DOWNLOAD_BROKER_PROTOCOL "websockets" +ENV DOWNLOAD_BROKER_TRANSPORT "websockets" ENV DOWNLOAD_WORKERS 8 -ENV FLASK_HOST "0.0.0.0" -ENV FLASK_PORT 5000 +ENV DOWNLOAD_MAX_VOLUME_MB 100 ENV SAVE_LOGS false -ENV LOGS_DIR /app/logs +ENV LOG_PATH /app/logs +ENV WIS2DOWNLOADER_CONFIG "/app/config.json" # update pyopenssl and pin requests and urllib3 to avoid SSL error RUN pip install pyopenssl --upgrade && pip install requests==2.26.0 urllib3==1.26.0 -# install cron and envsubst -RUN apt-get update && apt-get install -y cron gettext-base +# install cron and envsubst and guincorn +RUN apt-get update && apt-get install -y cron gettext-base gunicorn # copy all the code to the Docker image COPY . /app @@ -28,8 +28,8 @@ COPY . /app # Set the working directory to /app WORKDIR /app -# install wis2-downloader -RUN pip install https://github.com/wmo-im/wis2-downloader/archive/main.zip +# install wis2downloader from github branch=minor-fixes +RUN pip3 install https://github.com/wmo-im/wis2downloader/archive/minor-fixes.zip # add wis2box.cron to crontab COPY ./clean.cron /etc/cron.d/clean.cron @@ -42,5 +42,5 @@ RUN chmod +x /app/entrypoint.sh ENTRYPOINT [ "/app/entrypoint.sh" ] -# Run wis2-downloader when the container launches -CMD ["wis2downloader","--config","/app/config.json"] +# Run wis2downloader when the container launches +CMD ["gunicorn","--bind","0.0.0.0:5000","--pythonpath","/usr/local/lib/python3.9/site-packages/","--workers", "1", "wis2downloader.app:app"] \ No newline at end of file diff --git a/wis2-downloader/clean.cron b/wis2downloader/clean.cron similarity index 100% rename from wis2-downloader/clean.cron rename to wis2downloader/clean.cron diff --git a/wis2-downloader/clean.py b/wis2downloader/clean.py similarity index 100% rename from wis2-downloader/clean.py rename to wis2downloader/clean.py diff --git a/wis2downloader/config.template b/wis2downloader/config.template new file mode 100644 index 00000000..c32d3b2a --- /dev/null +++ b/wis2downloader/config.template @@ -0,0 +1,15 @@ +{ + "broker_hostname": "${DOWNLOAD_BROKER_HOST}", + "broker_port": $DOWNLOAD_BROKER_PORT, + "broker_username": "${DOWNLOAD_BROKER_USERNAME}", + "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", + "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", + "download_dir": "${DOWNLOAD_DIR}", + "flask_host": "0.0.0.0", + "flask_port": 5000, + "download_workers": $DOWNLOAD_WORKERS, + "save_logs": false, + "log_path": "logs", + "max_disk_usage": ${DOWNLOAD_MAX_VOLUME_MB}, + "mqtt_session_info": "${DOWNLOAD_DIR}/session-info.json" +} diff --git a/wis2-downloader/entrypoint.sh b/wis2downloader/entrypoint.sh similarity index 50% rename from wis2-downloader/entrypoint.sh rename to wis2downloader/entrypoint.sh index ff9a19d1..768c674b 100644 --- a/wis2-downloader/entrypoint.sh +++ b/wis2downloader/entrypoint.sh @@ -14,8 +14,6 @@ echo "END /entrypoint.sh" echo "Download directory in container: $DOWNLOAD_DIR" # print the retention period hours echo "Retention period in hours: $RETENTION_PERIOD_HOURS" -# print the maximum MB allowed in the download directory -echo "Maximum MB allowed in the download directory: $MAX_MB_DOWNLOAD_DIR" # ensure DOWNLOAD_DIR exists if [ ! -d $DOWNLOAD_DIR ]; then @@ -24,6 +22,17 @@ if [ ! -d $DOWNLOAD_DIR ]; then fi envsubst < config.template > config.json +# if session-info.json does not exists in $DOWNLOAD_DIR, create it +if [ ! -f $DOWNLOAD_DIR/session-info.json ]; then + echo "Creating session-info.json" + echo "{" > $DOWNLOAD_DIR/session-info.json + echo ' "topics": {},' >> $DOWNLOAD_DIR/session-info.json + # generate a random string for client_id + echo "Generating random client_id" + echo ' "client_id": "wis2box-wis2downloader-'$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)'"' >> $DOWNLOAD_DIR/session-info.json + echo "}" >> $DOWNLOAD_DIR/session-info.json +fi + # print the config echo "Config:" cat /app/config.json From 0871276bada49ebfa27e1e087b91d545b6494098 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 17:19:22 +0200 Subject: [PATCH 07/21] flake8 --- tests/integration/test_workflow.py | 3 ++- wis2box-management/wis2box/downloader.py | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 9b0c2a23..f7e92b18 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -43,7 +43,7 @@ def test_wis2downloader(): - """Test if the wis2downloader has downloaded + """Test if the wis2downloader has downloaded the expected number of files in the download directory""" DOWNLOAD_DIR = DATADIR / 'downloads' @@ -57,6 +57,7 @@ def test_wis2downloader(): assert total_files == 385 + def test_metadata_station_cache(): """Test station metadata caching""" diff --git a/wis2box-management/wis2box/downloader.py b/wis2box-management/wis2box/downloader.py index 32e7ea0e..292b09f4 100644 --- a/wis2box-management/wis2box/downloader.py +++ b/wis2box-management/wis2box/downloader.py @@ -30,18 +30,20 @@ DOWNLOAD_URL = 'http://wis2downloader:5000' + @click.group() def downloader(): """Interact with the wis2downloader""" pass + @click.command('list-subscriptions') @click.pass_context @cli_helpers.OPTION_VERBOSITY def list_subscriptions(ctx, verbosity): """list all subscriptions""" - # make a GET requests to http://:/subscriptions + # make a GET requests to http://{DOWNLOAD_URL}/subscriptions try: response = requests.get(f'{DOWNLOAD_URL}/subscriptions') # check response status @@ -52,18 +54,19 @@ def list_subscriptions(ctx, verbosity): click.echo(f'Error: {response.status_code}') click.echo(response.text) except requests.exceptions.ConnectionError: - click.echo(f'Error: Connection refused') + click.echo('Error: Connection refused') click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + @click.command('add-subscription') @click.pass_context @click.option('--topic', '-t', help='The topic to subscribe to', required=True) def add_subscription(ctx, topic): """add a subscription""" - + topic.replace('#', '%23') topic.replace('+', '%2B') - # make a POST request to http://:/subscriptions + # make a POST request to http://{DOWNLOAD_URL}/subscriptions try: response = requests.post(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') # check response status @@ -76,9 +79,9 @@ def add_subscription(ctx, topic): click.echo(f'Error: {response.status_code}') click.echo(response.text) except requests.exceptions.ConnectionError: - click.echo(f'Error: Connection refused') + click.echo('Error: Connection refused') click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') - + @click.command('remove-subscription') @click.pass_context @@ -88,9 +91,9 @@ def remove_subscription(ctx, topic): topic.replace('#', '%23') topic.replace('+', '%2B') - # make a DELETE request to http://:/subscriptions + # make a DELETE request to http://{DOWNLOAD_URL}/subscriptions try: - response = requests.delete(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') + response = requests.delete(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') # noqa # check response status if response.status_code == 200: click.echo('Subscription deleted') @@ -101,10 +104,10 @@ def remove_subscription(ctx, topic): click.echo(f'Error: {response.status_code}') click.echo(response.text) except requests.exceptions.ConnectionError: - click.echo(f'Error: Connection refused') + click.echo('Error: Connection refused') click.echo(f'Is the wis2downloader running on {DOWNLOAD_URL}?') + downloader.add_command(list_subscriptions) downloader.add_command(add_subscription) downloader.add_command(remove_subscription) - From 374e33bc19d7abf875d18973e21b62b80d1fa0d3 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 17:23:27 +0200 Subject: [PATCH 08/21] fix topic replace to deal with + and # --- wis2box-management/wis2box/downloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wis2box-management/wis2box/downloader.py b/wis2box-management/wis2box/downloader.py index 292b09f4..86a76a73 100644 --- a/wis2box-management/wis2box/downloader.py +++ b/wis2box-management/wis2box/downloader.py @@ -64,8 +64,8 @@ def list_subscriptions(ctx, verbosity): def add_subscription(ctx, topic): """add a subscription""" - topic.replace('#', '%23') - topic.replace('+', '%2B') + topic = topic.replace('#', '%23') + topic = topic.replace('+', '%2B') # make a POST request to http://{DOWNLOAD_URL}/subscriptions try: response = requests.post(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') @@ -89,8 +89,8 @@ def add_subscription(ctx, topic): def remove_subscription(ctx, topic): """remove a subscription""" - topic.replace('#', '%23') - topic.replace('+', '%2B') + topic = topic.replace('#', '%23') + topic = topic.replace('+', '%2B') # make a DELETE request to http://{DOWNLOAD_URL}/subscriptions try: response = requests.delete(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') # noqa From 53a243aaf1d9c68c1e06b01e0a6e7e242cf3d29f Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 21:58:33 +0200 Subject: [PATCH 09/21] replace localhost with public IP for download test to function ... --- .github/workflows/tests-docker.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index d62ae686..835140ac 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -26,9 +26,13 @@ jobs: docker version docker compose version python3 -V - - name: setup wis2box configuration + - name: Get Public IP + id: ip + uses: haythem/public-ip@v1.3 + - name: setup wis2box configuration, replace localhost with public IP ⚙️ run: | cp tests/test.env wis2box.env + sed -i "s/localhost/${{ steps.ip.outputs.ipv4 }}/g" wis2box.env cat wis2box.env python3 wis2box-ctl.py config - name: build wis2box From c5e2f504eafbf6c1b8ca5bdc175b38455e6e1d31 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 22:17:45 +0200 Subject: [PATCH 10/21] get IP via hostname -I --- .github/workflows/tests-docker.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 835140ac..30fd1047 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -26,13 +26,11 @@ jobs: docker version docker compose version python3 -V - - name: Get Public IP - id: ip - uses: haythem/public-ip@v1.3 - - name: setup wis2box configuration, replace localhost with public IP ⚙️ + - name: setup wis2box configuration, replace localhost with IP on host 📦 run: | + export IP=$(hostname -I | awk '{print $1}') cp tests/test.env wis2box.env - sed -i "s/localhost/${{ steps.ip.outputs.ipv4 }}/g" wis2box.env + sed -i "s/localhost/$IP/g" wis2box.env cat wis2box.env python3 wis2box-ctl.py config - name: build wis2box From 7a0a4873d96f2781fd9b9443b5b06f52fc953f2f Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 22:40:22 +0200 Subject: [PATCH 11/21] count files downloaded per topic --- tests/integration/test_workflow.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index f7e92b18..e422761c 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -48,14 +48,35 @@ def test_wis2downloader(): DOWNLOAD_DIR = DATADIR / 'downloads' + topic_nfiles_dict = { + 'origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop': 23, # noqa + 'origin/a/wis2/dz-alger_met_centre/data/core/weather/surface-based-observations/synop': 28, # noqa + 'origin/a/wis2/cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global': 10, # noqa + 'origin/a/wis2/ro-rnimh/data/core/weather/surface-based-observations/synop': 49, # noqa + 'origin/a/wis2/cd-brazza_met_centre/data/core/weather/surface-based-observations/synop': 14, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/buoy': 2, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind_profiler': 1, # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa + 'origin/a/wis2/it-roma_met_centre/data/core/weather/surface-based-observations/synop': 31 # noqa + } + + topic_nfiles_dict_found = {} + for key in topic_nfiles_dict.keys(): + topic_nfiles_dict_found[key] = 0 + # count the number of files received in the download directory # over all subdirectories total_files = 0 for root, dirs, files in os.walk(DOWNLOAD_DIR): - print(f'Found {len(files)} files in {root}') total_files += len(files) - - assert total_files == 385 + for key in topic_nfiles_dict.keys(): + if key in root: + topic_nfiles_dict_found[key] += len(files) + + # check if the number of files downloaded for each topic + # matches the expected number + for key in topic_nfiles_dict.keys(): + assert topic_nfiles_dict[key] == topic_nfiles_dict_found[key] def test_metadata_station_cache(): From 369df5967c27b0122fff58b182697a33650d8755 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 11 Jun 2024 08:51:09 +0200 Subject: [PATCH 12/21] updated docs --- docs/source/user/downloading-data.rst | 76 +++++++++++++-------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/docs/source/user/downloading-data.rst b/docs/source/user/downloading-data.rst index e3cadcde..048faea1 100644 --- a/docs/source/user/downloading-data.rst +++ b/docs/source/user/downloading-data.rst @@ -11,86 +11,82 @@ This section provides guidance how to download data from WIS2 Global Services. WIS2 Global Services include a Global Broker that provides users the ability to subscribe to data (via topics) and download to their local environment / workstation / decision support system from the WIS2 Global Cache. -wis2-downloader +wis2downloader --------------- -wis2box enables subscribe and data download workflow the WIS2 network, by using the ``wis2-downloader`` container, inside of which runs the `wis2-downloader tool " +``` +python3 wis2box.ctl.py execute wis2box downloader list-subscriptions ``` -The list of the currently active subscriptions should be returned as a JSON object. +This will return a JSON object with the current subscriptions. -Adding subscriptions -~~~~~~~~~~~~~~~~~~~~ +Adding a subscription +~~~~~~~~~~~~~~~~~~~~~ -Subscriptions can be added via a GET request to the `./add` endpoint that is proxied on /wis2-downloader on the wis2box host, with the following form: +To add a subscription, you can use the following command: -```bash -curl http://localhost/wis2-downloader/add?topic=&target= -H "Authorization: Bearer " ``` +python3 wis2box.ctl.py execute wis2box downloader add-subscription --topic +``` + +This will add a subscription to the topic you specify and return the JSON object with the current subscriptions. -- `topic` specifies the topic to subscribe to. *Special characters (+, #) must be URL encoded, i.e. `+` = `%2B`, `#` = `%23`.* -- `target` specifies the directory to save the downloads to, relative to `download_dir` from `config.json`. *If this is not provided, the directory will default to that of the topic hierarchy.* +Deleting a subscription +~~~~~~~~~~~~~~~~~~~~~~~ -For example: -```bash -curl http://localhost/wis2-downloader/add?topic=cache/a/wis2/%2B/data/core/weather/%23&target=example_data -H "Authorization: Bearer " +To delete a subscription, you can use the following command: + +``` +python3 wis2box.ctl.py execute wis2box downloader delete-subscription --topic ``` -The list of active subscriptions after addition should be returned as a JSON object. +This will delete the subscription to the topic you specify and return the JSON object with the current subscriptions. -Deleting subscriptions -~~~~~~~~~~~~~~~~~~~~~~ -Subscriptions are deleted similarly via a GET request to the `./delete` endpoint, with the following form: -```bash -curl http://:/delete?topic= -H "Authorization: Bearer " -``` +Managing subscriptions from outside the wis2box +---------------------------------------------- -For example: -```bash -curl http://localhost:8080/delete?topic=cache/a/wis2/%2B/data/core/weather/%23 -H "Authorization: Bearer " -``` +The wis2downloader API-endpoint is proxied on the path `/wis2downloader` on the wis2box host-url, allowing you to interact with it using curl or other HTTP clients from any machine that can reach the wis2box host. -The list of active subscriptions after deletion should be returned as a JSON object. +The wis2box-proxy by default secures the path `/wis2downloader` with a bearer token, which can be generated using the `wis2box auth` command as follows: +``` +python3 wis2box.ctl.py execute wis2box auth add-token --path wis2downloader -y +``` From a53f7cbeb2b5f43076b533ee4b2ff3153dc682b8 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 11 Jun 2024 22:07:43 +0200 Subject: [PATCH 13/21] grafana dashboard cleanup --- grafana/dashboards/cadvisor.json | 593 ---------------------- grafana/dashboards/home.json | 2 +- grafana/dashboards/storage_dashboard.json | 362 ------------- grafana/dashboards/wis2box_broker.json | 405 --------------- grafana/dashboards/wis2downloader.json | 5 +- 5 files changed, 4 insertions(+), 1363 deletions(-) delete mode 100644 grafana/dashboards/cadvisor.json delete mode 100644 grafana/dashboards/storage_dashboard.json delete mode 100644 grafana/dashboards/wis2box_broker.json diff --git a/grafana/dashboards/cadvisor.json b/grafana/dashboards/cadvisor.json deleted file mode 100644 index b5b88121..00000000 --- a/grafana/dashboards/cadvisor.json +++ /dev/null @@ -1,593 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 6, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "avg(container_memory_usage_bytes{name!=\"\"}/1024/1024) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory: used MB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{name!=\"\"}[1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Sent rate bytes/s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_errors_total{name!=\"\"} [1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Received error rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 9 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU load average", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 9 - }, - "id": 4, - "interval": "1m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{name!=\"\"}[1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Received rate bytes/s", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "always", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 9 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_errors_total{name!=\"\"} [1m])) by (name)", - "interval": "1m", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Network: Transmit error rate", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "cAdvisor dashboard", - "uid": "81cZrlS4z", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/grafana/dashboards/home.json b/grafana/dashboards/home.json index 51afba30..2bada33c 100644 --- a/grafana/dashboards/home.json +++ b/grafana/dashboards/home.json @@ -640,7 +640,7 @@ }, "timepicker": {}, "timezone": "", - "title": "wis2box workflow monitoring", + "title": "wis2box data publication dashboard", "uid": "KkBocEA4k", "version": 4, "weekStart": "" diff --git a/grafana/dashboards/storage_dashboard.json b/grafana/dashboards/storage_dashboard.json deleted file mode 100644 index 84920ddf..00000000 --- a/grafana/dashboards/storage_dashboard.json +++ /dev/null @@ -1,362 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "minio_bucket_usage_object_total", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Total objects stored per bucket", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "minio_bucket_usage_total_bytes", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Total MB stored per bucket", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 11, - "x": 8, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "rate(minio_bucket_traffic_sent_bytes[$__rate_interval])", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Traffic sent bytes rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 11, - "x": 8, - "y": 10 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "rate(minio_bucket_traffic_received_bytes[$__rate_interval])", - "legendFormat": "{{bucket}}", - "range": true, - "refId": "A" - } - ], - "title": "Traffic received bytes rate", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Storage statistics", - "uid": "XCoSx0g4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2box_broker.json b/grafana/dashboards/wis2box_broker.json deleted file mode 100644 index 9673553b..00000000 --- a/grafana/dashboards/wis2box_broker.json +++ /dev/null @@ -1,405 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "increase(wis2box_broker_msg_sent[1m])", - "legendFormat": "messages sent rate", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages sent per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 11, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "wis2box_broker_msg_dropped", - "legendFormat": "messages dropped", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages dropped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 0, - "y": 11 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "increase(wis2box_broker_msg_received[1m])", - "legendFormat": "messages received", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages received per minute", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 11, - "x": 11, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "wis2box_broker_msg_stored", - "legendFormat": "messages stored", - "range": true, - "refId": "A" - } - ], - "title": "wis2box-broker: messages stored", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "wis2box broker statistics $SYS/broker/messages", - "uid": "LNKi9EA4z", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/grafana/dashboards/wis2downloader.json b/grafana/dashboards/wis2downloader.json index ec080d7c..9c567c3e 100644 --- a/grafana/dashboards/wis2downloader.json +++ b/grafana/dashboards/wis2downloader.json @@ -822,6 +822,7 @@ "y": 13 }, "id": 2, + "interval": "60s", "options": { "legend": { "calcs": [], @@ -840,7 +841,7 @@ "uid": "PABAFD29CE247021E" }, "editorMode": "builder", - "expr": "sum(increase(downloaded_files_total[1m]))", + "expr": "increase(sum(downloaded_files_total or vector(0))[1m:])", "legendFormat": "Files downloaded per minute", "range": true, "refId": "A" @@ -863,7 +864,7 @@ }, "timepicker": {}, "timezone": "", - "title": "wis2downloader monitoring dashboard", + "title": "wis2downloader dashboard", "uid": "zB4GjVaIk", "version": 5, "weekStart": "" From 4ed6118c8d435b2844cc128383a022840446bc60 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 11 Jun 2024 22:21:01 +0200 Subject: [PATCH 14/21] updated downloaded request to align with Dave's changes --- wis2box-management/wis2box/downloader.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/wis2box-management/wis2box/downloader.py b/wis2box-management/wis2box/downloader.py index 86a76a73..a673bf71 100644 --- a/wis2box-management/wis2box/downloader.py +++ b/wis2box-management/wis2box/downloader.py @@ -64,15 +64,14 @@ def list_subscriptions(ctx, verbosity): def add_subscription(ctx, topic): """add a subscription""" - topic = topic.replace('#', '%23') - topic = topic.replace('+', '%2B') # make a POST request to http://{DOWNLOAD_URL}/subscriptions try: - response = requests.post(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') + response = requests.post(f'{DOWNLOAD_URL}/subscriptions', + json={'topic': topic}) # check response status - if response.status_code == 200: + if response.status_code == 201: click.echo('Subscription added') - click.echo('Current subscriptions:') + click.echo('Added subscription:') click.echo(response.text) else: click.echo('Subscription not added') @@ -89,11 +88,11 @@ def add_subscription(ctx, topic): def remove_subscription(ctx, topic): """remove a subscription""" - topic = topic.replace('#', '%23') - topic = topic.replace('+', '%2B') + topic = topic.replace('%23', '#') + topic = topic.replace('%2B', '+') # make a DELETE request to http://{DOWNLOAD_URL}/subscriptions try: - response = requests.delete(f'{DOWNLOAD_URL}/subscriptions?topic={topic}') # noqa + response = requests.delete(f'{DOWNLOAD_URL}/subscriptions/{topic}') # noqa # check response status if response.status_code == 200: click.echo('Subscription deleted') From 27043e98893a6e89a259605f4781a88b9729eb88 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 11 Jun 2024 22:21:45 +0200 Subject: [PATCH 15/21] rename to .session-info.json --- wis2downloader/config.template | 6 +++--- wis2downloader/entrypoint.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/wis2downloader/config.template b/wis2downloader/config.template index c32d3b2a..e202f0c0 100644 --- a/wis2downloader/config.template +++ b/wis2downloader/config.template @@ -1,15 +1,15 @@ { "broker_hostname": "${DOWNLOAD_BROKER_HOST}", - "broker_port": $DOWNLOAD_BROKER_PORT, + "broker_port": ${DOWNLOAD_BROKER_PORT}, "broker_username": "${DOWNLOAD_BROKER_USERNAME}", "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", "download_dir": "${DOWNLOAD_DIR}", "flask_host": "0.0.0.0", "flask_port": 5000, - "download_workers": $DOWNLOAD_WORKERS, + "download_workers": ${DOWNLOAD_WORKERS}, "save_logs": false, "log_path": "logs", "max_disk_usage": ${DOWNLOAD_MAX_VOLUME_MB}, - "mqtt_session_info": "${DOWNLOAD_DIR}/session-info.json" + "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json" } diff --git a/wis2downloader/entrypoint.sh b/wis2downloader/entrypoint.sh index 768c674b..61386e13 100644 --- a/wis2downloader/entrypoint.sh +++ b/wis2downloader/entrypoint.sh @@ -23,14 +23,14 @@ fi envsubst < config.template > config.json # if session-info.json does not exists in $DOWNLOAD_DIR, create it -if [ ! -f $DOWNLOAD_DIR/session-info.json ]; then - echo "Creating session-info.json" - echo "{" > $DOWNLOAD_DIR/session-info.json - echo ' "topics": {},' >> $DOWNLOAD_DIR/session-info.json +if [ ! -f $DOWNLOAD_DIR/.session-info.json ]; then + echo "Creating .session-info.json" + echo "{" > $DOWNLOAD_DIR/.session-info.json + echo ' "topics": {},' >> $DOWNLOAD_DIR/.session-info.json # generate a random string for client_id echo "Generating random client_id" - echo ' "client_id": "wis2box-wis2downloader-'$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)'"' >> $DOWNLOAD_DIR/session-info.json - echo "}" >> $DOWNLOAD_DIR/session-info.json + echo ' "client_id": "wis2box-wis2downloader-'$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)'"' >> $DOWNLOAD_DIR/.session-info.json + echo "}" >> $DOWNLOAD_DIR/.session-info.json fi # print the config From 73f5376de1d31601c879a2bea0185159c1de39ff Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 11 Jun 2024 22:29:08 +0200 Subject: [PATCH 16/21] adding vector(0) --- grafana/dashboards/wis2downloader.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/dashboards/wis2downloader.json b/grafana/dashboards/wis2downloader.json index 9c567c3e..5be0ec4a 100644 --- a/grafana/dashboards/wis2downloader.json +++ b/grafana/dashboards/wis2downloader.json @@ -678,7 +678,7 @@ "uid": "PABAFD29CE247021E" }, "editorMode": "builder", - "expr": "sum(increase(downloaded_bytes_total[1m]))", + "expr": "increase(sum(downloaded_bytes_total or vector(0))[1m:])", "hide": false, "legendFormat": "Data downloaded per minute", "range": true, From 9562aff3496441f3be5f443a9beaf381fd62fbb1 Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 12 Jun 2024 15:55:59 +0200 Subject: [PATCH 17/21] update dashboard, build from main, align ENV with expected config --- docs/source/user/downloading-data.rst | 2 +- grafana/dashboards/wis2downloader.json | 1510 +++++++++++------------- tests/test.env | 2 +- wis2box-create-config.py | 2 +- wis2box.env.example | 2 +- wis2downloader/Dockerfile | 8 +- wis2downloader/config.template | 11 +- 7 files changed, 719 insertions(+), 818 deletions(-) diff --git a/docs/source/user/downloading-data.rst b/docs/source/user/downloading-data.rst index 048faea1..dff74efb 100644 --- a/docs/source/user/downloading-data.rst +++ b/docs/source/user/downloading-data.rst @@ -27,7 +27,7 @@ The following environment variables are used by the ``wis2downloader``: - ``DOWNLOAD_BROKER_TRANSPORT``: ``websockets`` or ``tcp``, the transport-mechanism to use to connect to the MQTT-broker. Defaults to ``websockets``, - ``DOWNLOAD_RETENTION_PERIOD_HOURS``: The retention period in hours for the downloaded data. Defaults to ``24`` - ``DOWNLOAD_WORKERS``: The number of download workers to use. Defaults to ``8``. Determines the number of parallel downloads. -- ``DOWNLOAD_MAX_VOLUME_MB``: The maximum volume in MB to download, to avoid the host running out of disk space. Defaults to ``100``. +- ``DOWNLOAD_MIN_FREE_SPACE_GB``: The minimum free space in GB to keep on the volume hosting the downloads. Defaults to ``1``. To override the default configuration, you can set the environment variables in the wis2box.env file. diff --git a/grafana/dashboards/wis2downloader.json b/grafana/dashboards/wis2downloader.json index 5be0ec4a..690c2b5b 100644 --- a/grafana/dashboards/wis2downloader.json +++ b/grafana/dashboards/wis2downloader.json @@ -1,871 +1,769 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 0 + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" }, - "id": 14, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Topic" - } - ] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "pluginVersion": "", - "targets": [ + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" + "desc": true, + "displayName": "Topic" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(topic) (topic_subscription_status == 1)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "WIS2-downloader subscriptions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true }, - "editorMode": "builder", - "exemplar": false, - "expr": "max by(topic) (topic_subscription_status == 1)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" + "indexByName": {}, + "renameByName": { + "topic": "Topic" + } } - ], - "title": "WIS2-downloader subscriptions", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 25 }, - "indexByName": {}, - "renameByName": { - "topic": "Topic" + { + "color": "red", + "value": 50 } - } + ] } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" }, - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 25 - }, - { - "color": "red", - "value": 50 - } - ] - } + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 5, - "y": 0 - }, - "id": 18, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "editorMode": "builder", + "expr": "queue_size", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Download queue size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-green", + "mode": "fixed" }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "decimals": 0, + "mappings": [], + "max": 9997, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "expr": "queue_size", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Download queue size", - "type": "gauge" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-green", - "mode": "fixed" - }, - "mappings": [], - "max": 1000, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 100 - } - ] - } + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 16, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - "overrides": [] + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type=\"bufr\"} or vector(0))[1h:])", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 0 - }, - "id": 11, - "interval": "1h", - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - "showThresholdLabels": false, - "showThresholdMarkers": false + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type=\"grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "{{label_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Files downloaded last hour", - "type": "gauge" + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_files_total{file_type!~\"bufr|grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Files downloaded by file-type, last hour", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-blue", - "mode": "fixed" - }, - "decimals": 1, - "mappings": [], - "max": 1000, - "min": 0, - "noValue": "No data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 100 - } - ] - }, - "unit": "decbytes" + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "fixed" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 11, - "y": 0 - }, - "id": 12, - "interval": "1h", - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "decimals": 1, + "mappings": [], + "max": 1000000000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "showThresholdLabels": false, - "showThresholdMarkers": false + "unit": "decbytes" }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "{{label_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Data downloaded last hour", - "type": "gauge" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-green", - "mode": "fixed" - }, - "mappings": [], - "max": 9997, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 19, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 14, - "y": 0 + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type=\"bufr\"} or vector(0))[1h:])", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" }, - "id": 16, - "interval": "1h", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - "showUnfilled": true, - "text": { - "titleSize": 17, - "valueSize": 20 - } + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type=\"grib\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" }, - "pluginVersion": "", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type=\"bufr\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "BUFR", - "range": false, - "refId": "A" + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type=\"grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "GRIB", - "range": false, - "refId": "B" + "editorMode": "code", + "exemplar": false, + "expr": "increase(sum(downloaded_bytes_total{file_type!~\"grib|bufr\"} or vector(0))[1h:])", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Data downloaded by file-type, last hour", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(downloaded_files_total{file_type!~\"bufr|grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "Other", - "range": false, - "refId": "C" - } - ], - "title": "Files downloaded by file-type", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-blue", - "mode": "fixed" + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" }, - "decimals": 1, - "mappings": [], - "max": 1000000000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 19, - "y": 0 - }, - "id": 19, - "interval": "1h", - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "showUnfilled": true, - "text": { - "titleSize": 17, - "valueSize": 20 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "pluginVersion": "", - "targets": [ + "overrides": [ { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" + "matcher": { + "id": "byName", + "options": "download per minute" }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type=\"bufr\"}[1h]))", - "format": "time_series", - "instant": true, - "interval": "1h", - "legendFormat": "BUFR", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type=\"grib\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "GRIB", - "range": false, - "refId": "B" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] }, { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(downloaded_bytes_total{file_type!~\"grib|bufr\"}[1h]))", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "1h", - "legendFormat": "Other", - "range": false, - "refId": "C" - } - ], - "title": "Data downloaded by file-type", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 19, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "matcher": { + "id": "byName", + "options": "Files downloaded per hour" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-green", - "value": null + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" } - ] - }, - "unit": "decbytes" + } + ] }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Data downloaded per hour" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - }, - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Data downloaded per minute" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + { + "matcher": { + "id": "byName", + "options": "Failded downloads per minute" }, - { - "matcher": { - "id": "byName", - "options": "Data downloaded per minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 14, - "x": 0, - "y": 4 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + } + ] }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" + "matcher": { + "id": "byName", + "options": "Failed downloads per minute" }, - "editorMode": "builder", - "expr": "increase(sum(downloaded_bytes_total or vector(0))[1m:])", - "hide": false, - "legendFormat": "Data downloaded per minute", - "range": true, - "refId": "A" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] } - ], - "title": "Data downloaded per minute", - "type": "timeseries" + ] }, - { - "datasource": { - "type": "loki", - "uid": "P55348B596EBB51C3" - }, - "gridPos": { - "h": 18, - "w": 10, - "x": 14, - "y": 4 - }, - "id": 6, - "options": { - "dedupStrategy": "none", - "enableLogDetails": false, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P55348B596EBB51C3" - }, - "editorMode": "builder", - "expr": "{container_name=\"wis2downloader\"} != `Queue` != `GET` != `CLEAN`", - "queryType": "range", - "refId": "A" - } - ], - "title": "WIS2 downloader logs", - "transparent": true, - "type": "logs" + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 4 }, - { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" + "id": 2, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "expr": "increase(sum(downloaded_files_total or vector(0))[1m:])", + "legendFormat": "Successful downloads per minute", + "range": true, + "refId": "A" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "code", + "expr": "increase(sum(failed_downloads_total or vector(0))[1m:])", + "hide": false, + "legendFormat": "Failed downloads per minute", + "range": true, + "refId": "B" + } + ], + "title": "Successful / Failed downloads per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P55348B596EBB51C3" + }, + "gridPos": { + "h": 16, + "w": 9, + "x": 0, + "y": 6 + }, + "id": 6, + "options": { + "dedupStrategy": "none", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P55348B596EBB51C3" + }, + "editorMode": "builder", + "expr": "{container_name=\"wis2downloader\"} != `Queue` != `GET` != `CLEAN` |~ `ERROR|WARNING`", + "queryType": "range", + "refId": "A" + } + ], + "title": "WIS2 downloader logs: ERROR and WARNING", + "transparent": true, + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "download per minute" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Data downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" } - ] + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Data downloaded per minute" + ], + "prefix": "All except:", + "readOnly": true + } }, - { - "matcher": { - "id": "byName", - "options": "Files downloaded per hour" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-green", - "mode": "fixed" - } + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 14, - "x": 0, - "y": 13 - }, - "id": 2, - "interval": "60s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + } + ] }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PABAFD29CE247021E" + "matcher": { + "id": "byName", + "options": "Data downloaded per minute" }, - "editorMode": "builder", - "expr": "increase(sum(downloaded_files_total or vector(0))[1m:])", - "legendFormat": "Files downloaded per minute", - "range": true, - "refId": "A" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] } - ], - "title": "Files downloaded per minute", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "wis2downloader dashboard", - "uid": "zB4GjVaIk", - "version": 5, - "weekStart": "" - } \ No newline at end of file + ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 9, + "y": 13 + }, + "id": 9, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PABAFD29CE247021E" + }, + "editorMode": "builder", + "expr": "increase(sum(downloaded_bytes_total or vector(0))[1m:])", + "hide": false, + "legendFormat": "Data downloaded per minute", + "range": true, + "refId": "A" + } + ], + "title": "Data downloaded per minute", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "wis2downloader dashboard", + "uid": "zB4GjVaIk", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/tests/test.env b/tests/test.env index 1e23534f..82ae0dff 100644 --- a/tests/test.env +++ b/tests/test.env @@ -68,4 +68,4 @@ DOWNLOAD_BROKER_PORT=1883 DOWNLOAD_BROKER_USERNAME=everyone DOWNLOAD_BROKER_PASSWORD=everyone DOWNLOAD_BROKER_TRANSPORT=tcp -DOWNLOAD_MAX_VOLUME_MB=100 \ No newline at end of file +DOWNLOAD_MIN_FREE_SPACE_GB=0 \ No newline at end of file diff --git a/wis2box-create-config.py b/wis2box-create-config.py index c851c5d8..1e5d6226 100644 --- a/wis2box-create-config.py +++ b/wis2box-create-config.py @@ -367,7 +367,7 @@ def create_wis2box_env(config_dir: str) -> None: fh.write('# download transport mechanism (tcp or websockets)\n') fh.write('DOWNLOAD_BROKER_TRANSPORT=tcp\n') fh.write('# maximum MB in download directory\n') - fh.write('DOWNLOAD_MAX_VOLUME_MB=100\n') + fh.write('DOWNLOAD_MIN_FREE_SPACE_GB=1\n') fh.write('\n') print('*' * 80) diff --git a/wis2box.env.example b/wis2box.env.example index 674e2409..cbfdb6f9 100644 --- a/wis2box.env.example +++ b/wis2box.env.example @@ -67,5 +67,5 @@ DOWNLOAD_BROKER_PASSWORD=everyone # download transport mechanism (tcp or websockets) DOWNLOAD_BROKER_TRANSPORT=websockets # maximum MB in download directory -DOWNLOAD_MAX_VOLUME_MB=100 +DOWNLOAD_MIN_FREE_SPACE_GB=1 diff --git a/wis2downloader/Dockerfile b/wis2downloader/Dockerfile index 1cb30b4f..3b20c407 100644 --- a/wis2downloader/Dockerfile +++ b/wis2downloader/Dockerfile @@ -12,8 +12,8 @@ ENV DOWNLOAD_BROKER_USERNAME "everyone" ENV DOWNLOAD_BROKER_PASSWORD "everyone" ENV DOWNLOAD_BROKER_TRANSPORT "websockets" ENV DOWNLOAD_WORKERS 8 -ENV DOWNLOAD_MAX_VOLUME_MB 100 -ENV SAVE_LOGS false +ENV DOWNLOAD_MIN_FREE_SPACE_GB 1 +ENV DOWNLOAD_VALIDATE_TOPICS "false" ENV LOG_PATH /app/logs ENV WIS2DOWNLOADER_CONFIG "/app/config.json" @@ -28,8 +28,8 @@ COPY . /app # Set the working directory to /app WORKDIR /app -# install wis2downloader from github branch=minor-fixes -RUN pip3 install https://github.com/wmo-im/wis2downloader/archive/minor-fixes.zip +# install wis2downloader from github main branch +RUN pip3 install https://github.com/wmo-im/wis2downloader/archive/main.zip # add wis2box.cron to crontab COPY ./clean.cron /etc/cron.d/clean.cron diff --git a/wis2downloader/config.template b/wis2downloader/config.template index e202f0c0..7f11111e 100644 --- a/wis2downloader/config.template +++ b/wis2downloader/config.template @@ -4,12 +4,15 @@ "broker_username": "${DOWNLOAD_BROKER_USERNAME}", "broker_password": "${DOWNLOAD_BROKER_PASSWORD}", "broker_protocol": "${DOWNLOAD_BROKER_TRANSPORT}", + "download_workers": ${DOWNLOAD_WORKERS}, + "min_free_space": ${DOWNLOAD_MIN_FREE_SPACE_GB}, + "validate_topics": ${DOWNLOAD_VALIDATE_TOPICS}, + "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json", "download_dir": "${DOWNLOAD_DIR}", + "log_level": "INFO", + "base_url": "http://localhost:5000", "flask_host": "0.0.0.0", "flask_port": 5000, - "download_workers": ${DOWNLOAD_WORKERS}, "save_logs": false, - "log_path": "logs", - "max_disk_usage": ${DOWNLOAD_MAX_VOLUME_MB}, - "mqtt_session_info": "${DOWNLOAD_DIR}/.session-info.json" + "log_path": "logs" } From 370de7a2ce14d543640efea410e7694731e8365d Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 13 Jun 2024 12:55:53 +0200 Subject: [PATCH 18/21] revert __version__ change --- wis2box-management/wis2box/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wis2box-management/wis2box/__init__.py b/wis2box-management/wis2box/__init__.py index 2b482da1..fa8a2f53 100644 --- a/wis2box-management/wis2box/__init__.py +++ b/wis2box-management/wis2box/__init__.py @@ -19,7 +19,7 @@ # ############################################################################### -__version__ = '1.0b8' +__version__ = '1.0b7' import click From 3341e7edefda22e91c0425286cda61570e85b29e Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 13 Jun 2024 16:53:28 +0200 Subject: [PATCH 19/21] if None not in --- wis2box-management/wis2box/data/base.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index 0bcc05b0..e5ed363d 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -61,10 +61,12 @@ def __init__(self, defs: dict) -> None: self.output_data = {} self.discovery_metadata = {} self.gts = None - if defs.get('gts_ttaaii') and defs.get('gts_cccc'): + gts_ttaaii = defs.get('gts_ttaaii') + gts_cccc = defs.get('gts_cccc') + if None not in [gts_ttaaii, gts_cccc]: self.gts = { - 'ttaaii': defs.get('gts_ttaaii'), - 'cccc': defs.get('gts_cccc') + 'ttaaii': gts_ttaaii, + 'cccc': gts_cccc } # if discovery_metadata: From 3b6d78ddc9353baacbec18a9dc3e7f56c4272342 Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 13 Jun 2024 17:01:00 +0200 Subject: [PATCH 20/21] Update gts-headers-in-wis2.rst --- docs/source/user/gts-headers-in-wis2.rst | 34 +++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/docs/source/user/gts-headers-in-wis2.rst b/docs/source/user/gts-headers-in-wis2.rst index fbf56576..1d9af787 100644 --- a/docs/source/user/gts-headers-in-wis2.rst +++ b/docs/source/user/gts-headers-in-wis2.rst @@ -12,52 +12,48 @@ By adding GTS headers to your WIS2 notifications, you can stop your MSS and stil To enable the WIS2 to GTS Gateway to correctly identify the data to be republished on the GTS, you need to include the GTS property in the WIS2 Notification Message as follows: -```json -{ +.. code-block:: json + "properties": { "gts": { "ttaaii": "FTAE31", "cccc": "VTBB" } } -} -``` The wis2box can add these to your WIS2 Notifications automatically, provided you specify the additional file `gts_headers_mappings.json` that contains the required information to map the GTS headers to the incoming filenames. Note that this is optional and only required if you want to turn off the system responsible for sending data to the GTS during the transition period. -gts_headers_mappings.json -------------------------- +gts_headers_mapping.csv +----------------------- -If you want to add GTS headers to your WIS2 notifications, you need to create a JSON file that maps the GTS headers to the incoming filenames. +If you want to add GTS headers to your WIS2 notifications, you need to create a CSV file that maps the GTS headers to the incoming filenames. -The JSON file should be named `gts_headers_mappings.json` and should be placed in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable. +The CSV file should be named `gts_headers_mapping.csv` and should be placed in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable. The CSV should contain three columns: `string_in_filepath`, `TTAAii`, and `CCCC`. -Example content for `gts_headers_mappings.json`: +Example content for `gts_headers_mapping.csv`: + +.. code-block:: console -``` -string_in_filepath,TTAAii,CCCC -ISMD01LIIB,ISMD01,LIBB -ISMD02LIIB,ISMD02,LIBB -``` + string_in_filepath,TTAAii,CCCC + ISMD01LIIB,ISMD01,LIBB + ISMD02LIIB,ISMD02,LIBB In this example, whenever `ISMD01LIIB` or `ISMD02LIIB` is contained in the file-path of the incoming file, the corresponding GTS-headers will be added to the WIS2 Notification Message as a dictionary in the `properties` field: -```json -{ +.. code-block:: json + "properties": { "gts": { "ttaaii": "ISMD01", "cccc": "LIBB" } } -} -``` -If the `gts_headers_mappings.json` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, the wis2box will not add any GTS headers to the WIS2 Notification Message. +If the `gts_headers_mapping.csv` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, the wis2box will not add any GTS headers to the WIS2 Notification Message. From 7eb14cef35bd7dcb43670c87d62ae5178f2e8f6d Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 13 Jun 2024 13:08:59 -0400 Subject: [PATCH 21/21] Update gts-headers-in-wis2.rst --- docs/source/user/gts-headers-in-wis2.rst | 50 ++++++++++++------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/docs/source/user/gts-headers-in-wis2.rst b/docs/source/user/gts-headers-in-wis2.rst index 1d9af787..a5a9f12f 100644 --- a/docs/source/user/gts-headers-in-wis2.rst +++ b/docs/source/user/gts-headers-in-wis2.rst @@ -14,46 +14,44 @@ To enable the WIS2 to GTS Gateway to correctly identify the data to be republish .. code-block:: json - "properties": { - "gts": { - "ttaaii": "FTAE31", - "cccc": "VTBB" - } - } + "properties": { + "gts": { + "ttaaii": "FTAE31", + "cccc": "VTBB" + } + } -The wis2box can add these to your WIS2 Notifications automatically, provided you specify the additional file `gts_headers_mappings.json` that contains the required information to map the GTS headers to the incoming filenames. +wis2box can add these to WIS2 Notifications automatically, provided you specify the additional file `gts_headers_mappings.json` that contains the required information to map the GTS headers to the incoming filenames. -Note that this is optional and only required if you want to turn off the system responsible for sending data to the GTS during the transition period. +Note that this is optional and only required if you want to turn off the existing system responsible for sending data to the GTS during the transition period. gts_headers_mapping.csv ----------------------- -If you want to add GTS headers to your WIS2 notifications, you need to create a CSV file that maps the GTS headers to the incoming filenames. +If you want to add GTS headers to your WIS2 notifications, a CSV file is required that maps GTS headers to incoming filenames. -The CSV file should be named `gts_headers_mapping.csv` and should be placed in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable. +The CSV file should be named (exactly) `gts_headers_mapping.csv` and should be placed in the directory defined using the `WIS2BOX_HOST_DATADIR` environment variable. -The CSV should contain three columns: `string_in_filepath`, `TTAAii`, and `CCCC`. +The CSV should contain the following columns: `string_in_filepath`, `TTAAii`, and `CCCC`. Example content for `gts_headers_mapping.csv`: -.. code-block:: console +.. code-block:: csv - string_in_filepath,TTAAii,CCCC - ISMD01LIIB,ISMD01,LIBB - ISMD02LIIB,ISMD02,LIBB + string_in_filepath,TTAAii,CCCC + ISMD01LIIB,ISMD01,LIBB + ISMD02LIIB,ISMD02,LIBB -In this example, whenever `ISMD01LIIB` or `ISMD02LIIB` is contained in the file-path of the incoming file, -the corresponding GTS-headers will be added to the WIS2 Notification Message as a dictionary in the `properties` field: +In this example, whenever `ISMD01LIIB` or `ISMD02LIIB` is contained in the filepath of the incoming file, +the corresponding GTS headers will be added to the WIS2 Notification Message as a dictionary in the `properties` field: .. code-block:: json - "properties": { - "gts": { - "ttaaii": "ISMD01", - "cccc": "LIBB" - } - } - -If the `gts_headers_mapping.csv` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, the wis2box will not add any GTS headers to the WIS2 Notification Message. - + "properties": { + "gts": { + "ttaaii": "ISMD01", + "cccc": "LIBB" + } + } +If the `gts_headers_mapping.csv` file is not present in the directory you defined using the `WIS2BOX_HOST_DATADIR` environment variable, wis2box will not add any GTS headers to the WIS2 Notification Message.