Skip to content

Commit

Permalink
#minor Add option to request only once per batch
Browse files Browse the repository at this point in the history
  • Loading branch information
Anton Chub committed Aug 16, 2024
1 parent c6253d7 commit 1de4197
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 9 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ All are identical multi-arch images built for `amd64`, `arm64`, `arm/v7`, `ppc64
- Values can also be base64 encoded URLs that download binary data e.g. executables
- The key in the `ConfigMap`/`Secret` must end with "`.url`" ([see](https://github.com/kiwigrid/k8s-sidecar/blob/master/test/resources/resources.yaml#L84))

# Usage
# Usage

Example for a simple deployment can be found in [`example.yaml`](./examples/example.yaml). Depending on the cluster setup you have to grant yourself admin rights first:
```shell
Expand Down Expand Up @@ -84,6 +84,7 @@ If the filename ends with `.url` suffix, the content will be processed as a URL
| `REQ_USERNAME` | Username to use for basic authentication for requests to `REQ_URL` and for `*.url` triggered requests | false | - | string |
| `REQ_PASSWORD` | Password to use for basic authentication for requests to `REQ_URL` and for `*.url` triggered requests | false | - | string |
| `REQ_BASIC_AUTH_ENCODING` | Which encoding to use for username and password as [by default it's undefined](https://datatracker.ietf.org/doc/html/rfc7617) (e.g. `utf-8`). | false | `latin1` | string |
| `REQ_ONCE_PER_BATCH` | Send request to `REQ_URL` only once each `WATCH_SERVER_TIMEOUT`. By default request is made after each change. Applicable only to `METHOD=WATCH`.
| `SCRIPT` | Absolute path to a script to execute after a configmap got reloaded. It runs before calls to `REQ_URI`. If the file is not executable it will be passed to `sh`. Otherwise it's executed as is. [Shebangs](https://en.wikipedia.org/wiki/Shebang_(Unix)) known to work are `#!/bin/sh` and `#!/usr/bin/env python` | false | - | string |
| `ERROR_THROTTLE_SLEEP` | How many seconds to wait before watching resources again when an error occurs | false | `5` | integer |
| `SKIP_TLS_VERIFY` | Set to `true` to skip tls verification for kube api calls | false | - | boolean |
Expand Down
21 changes: 15 additions & 6 deletions src/resources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ def _update_file(data_key, data_content, dest_folder, metadata, resource,

def _watch_resource_iterator(label, label_value, target_folder, request_url, request_method, request_payload,
namespace, folder_annotation, resource, unique_filenames, script, enable_5xx,
ignore_already_processed):
ignore_already_processed, request_once_per_batch):
v1 = client.CoreV1Api()
# Filter resources based on label and value or just label
label_selector = f"{label}={label_value}" if label_value else label
Expand All @@ -322,6 +322,9 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req

stream = watch.Watch().stream(getattr(v1, _list_namespace[namespace][resource]), **additional_args)

# Used if request_once_per_batch is enabled
any_files_changed = False

# Process events
for event in stream:
item = event['object']
Expand Down Expand Up @@ -355,12 +358,18 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req
else:
files_changed |= _process_secret(dest_folder, item, resource, unique_filenames, enable_5xx, item_removed)

any_files_changed |= files_changed

if script and files_changed:
execute(script)

if request_url and files_changed:
if request_url and files_changed and not request_once_per_batch:
request(request_url, request_method, enable_5xx, request_payload)

if request_url and any_files_changed and request_once_per_batch:
logger.debug(f"Starting batch request")
request(request_url, request_method, enable_5xx, request_payload)


def _watch_resource_loop(mode, *args):
while True:
Expand Down Expand Up @@ -389,11 +398,11 @@ def _watch_resource_loop(mode, *args):

def watch_for_changes(mode, label, label_value, target_folder, request_url, request_method, request_payload,
current_namespace, folder_annotation, resources, unique_filenames, script, enable_5xx,
ignore_already_processed):
ignore_already_processed, request_once_per_batch):
processes = _start_watcher_processes(current_namespace, folder_annotation, label,
label_value, request_method, mode, request_payload, resources,
target_folder, unique_filenames, script, request_url, enable_5xx,
ignore_already_processed)
ignore_already_processed, request_once_per_batch)

while True:
died = False
Expand All @@ -413,14 +422,14 @@ def watch_for_changes(mode, label, label_value, target_folder, request_url, requ

def _start_watcher_processes(namespace, folder_annotation, label, label_value, request_method,
mode, request_payload, resources, target_folder, unique_filenames, script, request_url,
enable_5xx, ignore_already_processed):
enable_5xx, ignore_already_processed, request_once_per_batch):
processes = []
for resource in resources:
for ns in namespace.split(','):
proc = Process(target=_watch_resource_loop,
args=(mode, label, label_value, target_folder, request_url, request_method, request_payload,
ns, folder_annotation, resource, unique_filenames, script, enable_5xx,
ignore_already_processed)
ignore_already_processed, request_once_per_batch)
)
proc.daemon = True
proc.start()
Expand Down
15 changes: 13 additions & 2 deletions src/sidecar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
REQ_PAYLOAD = "REQ_PAYLOAD"
REQ_URL = "REQ_URL"
REQ_METHOD = "REQ_METHOD"
REQ_ONCE_PER_BATCH = "REQ_ONCE_PER_BATCH"
SCRIPT = "SCRIPT"
ENABLE_5XX = "ENABLE_5XX"
IGNORE_ALREADY_PROCESSED = "IGNORE_ALREADY_PROCESSED"
Expand Down Expand Up @@ -72,10 +73,19 @@ def main():

request_method = os.getenv(REQ_METHOD)
request_url = os.getenv(REQ_URL)

request_payload = os.getenv(REQ_PAYLOAD)
if request_payload:
request_payload = prepare_payload(os.getenv(REQ_PAYLOAD))

request_once_per_batch = os.getenv(REQ_ONCE_PER_BATCH)
if request_once_per_batch is not None and request_once_per_batch.lower() == "true":
logger.info(f"Request once per batch will be enabled.")
request_once_per_batch = True
else:
logger.info(f"Request once per batch will not be enabled.")
request_once_per_batch = False

script = os.getenv(SCRIPT)

_initialize_kubeclient_configuration()
Expand Down Expand Up @@ -131,7 +141,8 @@ def main():
else:
watch_for_changes(method, label, label_value, target_folder, request_url, request_method, request_payload,
namespace, folder_annotation, resources, unique_filenames, script, enable_5xx,
ignore_already_processed)
ignore_already_processed,
request_once_per_batch)


def _initialize_kubeclient_configuration():
Expand Down

0 comments on commit 1de4197

Please sign in to comment.