From 1de4197f5dcf6eecca6ce819ca9f838bac1ea51e Mon Sep 17 00:00:00 2001 From: Anton Chub Date: Fri, 16 Aug 2024 14:33:13 +0200 Subject: [PATCH] #minor Add option to request only once per batch --- README.md | 3 ++- src/resources.py | 21 +++++++++++++++------ src/sidecar.py | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e97b659b..6ec3d171 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ All are identical multi-arch images built for `amd64`, `arm64`, `arm/v7`, `ppc64 - Values can also be base64 encoded URLs that download binary data e.g. executables - The key in the `ConfigMap`/`Secret` must end with "`.url`" ([see](https://github.com/kiwigrid/k8s-sidecar/blob/master/test/resources/resources.yaml#L84)) -# Usage +# Usage Example for a simple deployment can be found in [`example.yaml`](./examples/example.yaml). Depending on the cluster setup you have to grant yourself admin rights first: ```shell @@ -84,6 +84,7 @@ If the filename ends with `.url` suffix, the content will be processed as a URL | `REQ_USERNAME` | Username to use for basic authentication for requests to `REQ_URL` and for `*.url` triggered requests | false | - | string | | `REQ_PASSWORD` | Password to use for basic authentication for requests to `REQ_URL` and for `*.url` triggered requests | false | - | string | | `REQ_BASIC_AUTH_ENCODING` | Which encoding to use for username and password as [by default it's undefined](https://datatracker.ietf.org/doc/html/rfc7617) (e.g. `utf-8`). | false | `latin1` | string | +| `REQ_ONCE_PER_BATCH` | Send request to `REQ_URL` only once each `WATCH_SERVER_TIMEOUT`. By default request is made after each change. Applicable only to `METHOD=WATCH`. | `SCRIPT` | Absolute path to a script to execute after a configmap got reloaded. It runs before calls to `REQ_URI`. If the file is not executable it will be passed to `sh`. Otherwise it's executed as is. [Shebangs](https://en.wikipedia.org/wiki/Shebang_(Unix)) known to work are `#!/bin/sh` and `#!/usr/bin/env python` | false | - | string | | `ERROR_THROTTLE_SLEEP` | How many seconds to wait before watching resources again when an error occurs | false | `5` | integer | | `SKIP_TLS_VERIFY` | Set to `true` to skip tls verification for kube api calls | false | - | boolean | diff --git a/src/resources.py b/src/resources.py index ca3533f1..70d1e2f8 100755 --- a/src/resources.py +++ b/src/resources.py @@ -305,7 +305,7 @@ def _update_file(data_key, data_content, dest_folder, metadata, resource, def _watch_resource_iterator(label, label_value, target_folder, request_url, request_method, request_payload, namespace, folder_annotation, resource, unique_filenames, script, enable_5xx, - ignore_already_processed): + ignore_already_processed, request_once_per_batch): v1 = client.CoreV1Api() # Filter resources based on label and value or just label label_selector = f"{label}={label_value}" if label_value else label @@ -322,6 +322,9 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req stream = watch.Watch().stream(getattr(v1, _list_namespace[namespace][resource]), **additional_args) + # Used if request_once_per_batch is enabled + any_files_changed = False + # Process events for event in stream: item = event['object'] @@ -355,12 +358,18 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req else: files_changed |= _process_secret(dest_folder, item, resource, unique_filenames, enable_5xx, item_removed) + any_files_changed |= files_changed + if script and files_changed: execute(script) - if request_url and files_changed: + if request_url and files_changed and not request_once_per_batch: request(request_url, request_method, enable_5xx, request_payload) + if request_url and any_files_changed and request_once_per_batch: + logger.debug(f"Starting batch request") + request(request_url, request_method, enable_5xx, request_payload) + def _watch_resource_loop(mode, *args): while True: @@ -389,11 +398,11 @@ def _watch_resource_loop(mode, *args): def watch_for_changes(mode, label, label_value, target_folder, request_url, request_method, request_payload, current_namespace, folder_annotation, resources, unique_filenames, script, enable_5xx, - ignore_already_processed): + ignore_already_processed, request_once_per_batch): processes = _start_watcher_processes(current_namespace, folder_annotation, label, label_value, request_method, mode, request_payload, resources, target_folder, unique_filenames, script, request_url, enable_5xx, - ignore_already_processed) + ignore_already_processed, request_once_per_batch) while True: died = False @@ -413,14 +422,14 @@ def watch_for_changes(mode, label, label_value, target_folder, request_url, requ def _start_watcher_processes(namespace, folder_annotation, label, label_value, request_method, mode, request_payload, resources, target_folder, unique_filenames, script, request_url, - enable_5xx, ignore_already_processed): + enable_5xx, ignore_already_processed, request_once_per_batch): processes = [] for resource in resources: for ns in namespace.split(','): proc = Process(target=_watch_resource_loop, args=(mode, label, label_value, target_folder, request_url, request_method, request_payload, ns, folder_annotation, resource, unique_filenames, script, enable_5xx, - ignore_already_processed) + ignore_already_processed, request_once_per_batch) ) proc.daemon = True proc.start() diff --git a/src/sidecar.py b/src/sidecar.py index 6312b958..9ee2d654 100644 --- a/src/sidecar.py +++ b/src/sidecar.py @@ -24,6 +24,7 @@ REQ_PAYLOAD = "REQ_PAYLOAD" REQ_URL = "REQ_URL" REQ_METHOD = "REQ_METHOD" +REQ_ONCE_PER_BATCH = "REQ_ONCE_PER_BATCH" SCRIPT = "SCRIPT" ENABLE_5XX = "ENABLE_5XX" IGNORE_ALREADY_PROCESSED = "IGNORE_ALREADY_PROCESSED" @@ -72,10 +73,19 @@ def main(): request_method = os.getenv(REQ_METHOD) request_url = os.getenv(REQ_URL) - + request_payload = os.getenv(REQ_PAYLOAD) if request_payload: request_payload = prepare_payload(os.getenv(REQ_PAYLOAD)) + + request_once_per_batch = os.getenv(REQ_ONCE_PER_BATCH) + if request_once_per_batch is not None and request_once_per_batch.lower() == "true": + logger.info(f"Request once per batch will be enabled.") + request_once_per_batch = True + else: + logger.info(f"Request once per batch will not be enabled.") + request_once_per_batch = False + script = os.getenv(SCRIPT) _initialize_kubeclient_configuration() @@ -131,7 +141,8 @@ def main(): else: watch_for_changes(method, label, label_value, target_folder, request_url, request_method, request_payload, namespace, folder_annotation, resources, unique_filenames, script, enable_5xx, - ignore_already_processed) + ignore_already_processed, + request_once_per_batch) def _initialize_kubeclient_configuration():