From 921d33b67f963c7273131e1c2322d7abb5660ce9 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sat, 4 May 2024 19:08:00 +0300 Subject: [PATCH] Time based cache refresh Previously to update the cache you had to clear the cache and fetch. This leaves a window where a build may use uncached value and fail. A better way is to refresh the cache periodically *without* clearing the cache, ensuring that builds will always use cached values. To support time based refresh, we fetch data only if the cached item it too old or missing. Upon successful fetch, the item age is set to zero, so the next access to the cache will skip the fetch. We use 2 threshold for refreshing the cache: - REFRESH_THRSHOLD: 12 hours - FETCH_THRESHOLD: 48 hours This allows refreshing the cache once per day (e.g. from a cron job). On errors, we can rerun the refresh until the refresh succeeds. Fresh items are skipped during the refresh. If refreshing the cache failed, jobs will trigger a refresh after 48 hours. The longer threshold ensures that this will never happen unless the refresh job is broken. The `fetch` command was renamed to `cache`. This describe better the purpose of the command. The hook was update as well to match the command name. The `drenv.cache` module provides now 2 operations: - cache.refresh() - rebuild the cache if older than cache.REFRESH_THRESHOLD (12 hours). Will is used in the `cache` hook. - cache.get() - rebuild the cache if older than cache.FETCH_THRESHOLD (48 hours) and return the path to the cached value. Will is used in `start` hooks to get a cached value. Signed-off-by: Nir Soffer --- test/addons/cdi/{fetch => cache} | 4 +-- test/addons/cdi/start | 4 +-- .../{rook-cephfs/fetch => csi-addons/cache} | 2 +- test/addons/csi-addons/start | 2 +- test/addons/kubevirt/cache | 11 +++++++ test/addons/kubevirt/start | 4 +-- test/addons/ocm-controller/cache | 10 +++++++ test/addons/ocm-controller/start | 2 +- test/addons/{kubevirt/fetch => olm/cache} | 4 +-- test/addons/olm/fetch | 11 ------- test/addons/olm/start | 4 +-- test/addons/recipe/{fetch => cache} | 2 +- test/addons/recipe/start | 2 +- .../{csi-addons/fetch => rook-cephfs/cache} | 2 +- test/addons/rook-cephfs/start | 2 +- .../fetch => rook-cluster/cache} | 2 +- test/addons/rook-cluster/fetch | 10 ------- test/addons/rook-cluster/start | 2 +- test/addons/rook-operator/cache | 10 +++++++ test/addons/rook-operator/fetch | 10 ------- test/addons/rook-operator/start | 2 +- test/addons/rook-toolbox/cache | 10 +++++++ test/addons/rook-toolbox/fetch | 10 ------- test/addons/rook-toolbox/start | 2 +- test/drenv/__main__.py | 16 +++++----- test/drenv/cache.py | 29 ++++++++++++++++--- 26 files changed, 95 insertions(+), 74 deletions(-) rename test/addons/cdi/{fetch => cache} (65%) rename test/addons/{rook-cephfs/fetch => csi-addons/cache} (80%) create mode 100755 test/addons/kubevirt/cache create mode 100755 test/addons/ocm-controller/cache rename test/addons/{kubevirt/fetch => olm/cache} (63%) delete mode 100755 test/addons/olm/fetch rename test/addons/recipe/{fetch => cache} (81%) rename test/addons/{csi-addons/fetch => rook-cephfs/cache} (79%) rename test/addons/{ocm-controller/fetch => rook-cluster/cache} (79%) delete mode 100755 test/addons/rook-cluster/fetch create mode 100755 test/addons/rook-operator/cache delete mode 100755 test/addons/rook-operator/fetch create mode 100755 test/addons/rook-toolbox/cache delete mode 100755 test/addons/rook-toolbox/fetch diff --git a/test/addons/cdi/fetch b/test/addons/cdi/cache similarity index 65% rename from test/addons/cdi/fetch rename to test/addons/cdi/cache index 9661e386f6..d45b306b9e 100755 --- a/test/addons/cdi/fetch +++ b/test/addons/cdi/cache @@ -7,5 +7,5 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch("operator", "addons/cdi-operator.yaml") -cache.fetch("cr", "addons/cdi-cr.yaml") +cache.refresh("operator", "addons/cdi-operator.yaml") +cache.refresh("cr", "addons/cdi-cr.yaml") diff --git a/test/addons/cdi/start b/test/addons/cdi/start index 336c9ad1f6..625beae2f1 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -14,7 +14,7 @@ NAMESPACE = "cdi" def deploy(cluster): print("Deploying cdi operator") - path = cache.fetch("operator", "addons/cdi-operator.yaml") + path = cache.get("operator", "addons/cdi-operator.yaml") kubectl.apply("--filename", path, context=cluster) print("Waiting until cdi-operator is rolled out") @@ -27,7 +27,7 @@ def deploy(cluster): ) print("Deploying cdi cr") - path = cache.fetch("cr", "addons/cdi-cr.yaml") + path = cache.get("cr", "addons/cdi-cr.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-cephfs/fetch b/test/addons/csi-addons/cache similarity index 80% rename from test/addons/rook-cephfs/fetch rename to test/addons/csi-addons/cache index dd0141dc80..7b37a59788 100755 --- a/test/addons/rook-cephfs/fetch +++ b/test/addons/csi-addons/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-cephfs.yaml") +cache.refresh(".", "addons/csi-addons.yaml") diff --git a/test/addons/csi-addons/start b/test/addons/csi-addons/start index d206387b22..7c1ffa7884 100755 --- a/test/addons/csi-addons/start +++ b/test/addons/csi-addons/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying csi addon for volume replication") - path = cache.fetch(".", "addons/csi-addons.yaml") + path = cache.get(".", "addons/csi-addons.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/kubevirt/cache b/test/addons/kubevirt/cache new file mode 100755 index 0000000000..9f3c235063 --- /dev/null +++ b/test/addons/kubevirt/cache @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh("operator", "addons/kubevirt-operator.yaml") +cache.refresh("cr", "addons/kubevirt-cr.yaml") diff --git a/test/addons/kubevirt/start b/test/addons/kubevirt/start index 14a022ab56..5cd6f32a0e 100755 --- a/test/addons/kubevirt/start +++ b/test/addons/kubevirt/start @@ -14,7 +14,7 @@ NAMESPACE = "kubevirt" def deploy(cluster): print("Deploying kubevirt operator") - path = cache.fetch("operator", "addons/kubevirt-operator.yaml") + path = cache.get("operator", "addons/kubevirt-operator.yaml") kubectl.apply("--filename", path, context=cluster) print("Waiting until virt-operator is rolled out") @@ -27,7 +27,7 @@ def deploy(cluster): ) print("Deploying kubevirt cr") - path = cache.fetch("cr", "addons/kubevirt-cr.yaml") + path = cache.get("cr", "addons/kubevirt-cr.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/ocm-controller/cache b/test/addons/ocm-controller/cache new file mode 100755 index 0000000000..bc4004e18c --- /dev/null +++ b/test/addons/ocm-controller/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/ocm-controller.yaml") diff --git a/test/addons/ocm-controller/start b/test/addons/ocm-controller/start index 431c5eca85..6b30c74f67 100755 --- a/test/addons/ocm-controller/start +++ b/test/addons/ocm-controller/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying ocm controller") - path = cache.fetch(".", "addons/ocm-controller.yaml") + path = cache.get(".", "addons/ocm-controller.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/kubevirt/fetch b/test/addons/olm/cache similarity index 63% rename from test/addons/kubevirt/fetch rename to test/addons/olm/cache index 304058b842..9e909d7fad 100755 --- a/test/addons/kubevirt/fetch +++ b/test/addons/olm/cache @@ -7,5 +7,5 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch("operator", "addons/kubevirt-operator.yaml") -cache.fetch("cr", "addons/kubevirt-cr.yaml") +cache.refresh("crds", "addons/olm-crds.yaml") +cache.refresh("operators", "addons/olm-operators.yaml") diff --git a/test/addons/olm/fetch b/test/addons/olm/fetch deleted file mode 100755 index 5d32a2f26e..0000000000 --- a/test/addons/olm/fetch +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch("crds", "addons/olm-crds.yaml") -cache.fetch("operators", "addons/olm-operators.yaml") diff --git a/test/addons/olm/start b/test/addons/olm/start index f2b7a0d2d4..c8a9fbb089 100755 --- a/test/addons/olm/start +++ b/test/addons/olm/start @@ -20,7 +20,7 @@ def deploy(cluster): # The CustomResourceDefinition "clusterserviceversions.operators.coreos.com" # is invalid: metadata.annotations: Too long: must have at most 262144 bytes # See https://medium.com/pareture/kubectl-install-crd-failed-annotations-too-long-2ebc91b40c7d - path = cache.fetch("crds", "addons/olm-crds.yaml") + path = cache.get("crds", "addons/olm-crds.yaml") kubectl.apply("--filename", path, "--server-side=true", context=cluster) print("Waiting until cdrs are established") @@ -32,7 +32,7 @@ def deploy(cluster): ) print("Deploying olm") - path = cache.fetch("operators", "addons/olm-operators.yaml") + path = cache.get("operators", "addons/olm-operators.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/recipe/fetch b/test/addons/recipe/cache similarity index 81% rename from test/addons/recipe/fetch rename to test/addons/recipe/cache index 221500f140..a094d54c5b 100755 --- a/test/addons/recipe/fetch +++ b/test/addons/recipe/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/recipe.yaml") +cache.refresh(".", "addons/recipe.yaml") diff --git a/test/addons/recipe/start b/test/addons/recipe/start index b1bb697cc6..bd0bdbec73 100755 --- a/test/addons/recipe/start +++ b/test/addons/recipe/start @@ -16,5 +16,5 @@ os.chdir(os.path.dirname(__file__)) cluster = sys.argv[1] print("Deploying recipe crd") -path = cache.fetch(".", "addons/recipe.yaml") +path = cache.get(".", "addons/recipe.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/csi-addons/fetch b/test/addons/rook-cephfs/cache similarity index 79% rename from test/addons/csi-addons/fetch rename to test/addons/rook-cephfs/cache index 71c61cc52e..56205b17e2 100755 --- a/test/addons/csi-addons/fetch +++ b/test/addons/rook-cephfs/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/csi-addons.yaml") +cache.refresh(".", "addons/rook-cephfs.yaml") diff --git a/test/addons/rook-cephfs/start b/test/addons/rook-cephfs/start index d8dcbaceb0..489b1b1e7e 100755 --- a/test/addons/rook-cephfs/start +++ b/test/addons/rook-cephfs/start @@ -13,7 +13,7 @@ from drenv import kubectl def deploy(cluster): print("Creating CephFS instance and storage/snapshot classes") - path = cache.fetch(".", "addons/rook-cephfs.yaml") + path = cache.get(".", "addons/rook-cephfs.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/ocm-controller/fetch b/test/addons/rook-cluster/cache similarity index 79% rename from test/addons/ocm-controller/fetch rename to test/addons/rook-cluster/cache index 37c55444fa..1da61bb7f9 100755 --- a/test/addons/ocm-controller/fetch +++ b/test/addons/rook-cluster/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/ocm-controller.yaml") +cache.refresh(".", "addons/rook-cluster.yaml") diff --git a/test/addons/rook-cluster/fetch b/test/addons/rook-cluster/fetch deleted file mode 100755 index ddf1c53d33..0000000000 --- a/test/addons/rook-cluster/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-cluster.yaml") diff --git a/test/addons/rook-cluster/start b/test/addons/rook-cluster/start index 530d9507a1..333b6b920a 100755 --- a/test/addons/rook-cluster/start +++ b/test/addons/rook-cluster/start @@ -20,7 +20,7 @@ TIMEOUT = 600 def deploy(cluster): print("Deploying rook ceph cluster") - path = cache.fetch(".", "addons/rook-cluster.yaml") + path = cache.get(".", "addons/rook-cluster.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-operator/cache b/test/addons/rook-operator/cache new file mode 100755 index 0000000000..d328f58042 --- /dev/null +++ b/test/addons/rook-operator/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/rook-operator.yaml") diff --git a/test/addons/rook-operator/fetch b/test/addons/rook-operator/fetch deleted file mode 100755 index 4b0d8bae9e..0000000000 --- a/test/addons/rook-operator/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-operator.yaml") diff --git a/test/addons/rook-operator/start b/test/addons/rook-operator/start index 62cf3e7a44..48275ca704 100755 --- a/test/addons/rook-operator/start +++ b/test/addons/rook-operator/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying rook ceph operator") - path = cache.fetch(".", "addons/rook-operator.yaml") + path = cache.get(".", "addons/rook-operator.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-toolbox/cache b/test/addons/rook-toolbox/cache new file mode 100755 index 0000000000..faf716ebec --- /dev/null +++ b/test/addons/rook-toolbox/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/rook-toolbox.yaml") diff --git a/test/addons/rook-toolbox/fetch b/test/addons/rook-toolbox/fetch deleted file mode 100755 index 572695efac..0000000000 --- a/test/addons/rook-toolbox/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-toolbox.yaml") diff --git a/test/addons/rook-toolbox/start b/test/addons/rook-toolbox/start index 8ebe287102..161d9ee9c5 100755 --- a/test/addons/rook-toolbox/start +++ b/test/addons/rook-toolbox/start @@ -13,7 +13,7 @@ from drenv import cache def deploy(cluster): print("Deploying rook ceph toolbox") - path = cache.fetch(".", "addons/rook-toolbox.yaml") + path = cache.get(".", "addons/rook-toolbox.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index c182774065..52e78216e6 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -85,7 +85,7 @@ def parse_args(): add_command(sp, "suspend", do_suspend, help="suspend virtual machines") add_command(sp, "resume", do_resume, help="resume virtual machines") add_command(sp, "dump", do_dump, help="dump an environment yaml") - add_command(sp, "fetch", do_fetch, help="cache environment resources") + add_command(sp, "cache", do_cache, help="cache environment resources") add_command(sp, "clear", do_clear, help="cleared cached resources", envfile=False) add_command(sp, "setup", do_setup, help="setup minikube for drenv", envfile=False) @@ -177,20 +177,20 @@ def do_clear(args): cache.clear() -def do_fetch(args): +def do_cache(args): env = load_env(args) start = time.monotonic() - logging.info("[%s] Fetching", env["name"]) + logging.info("[%s] Refreshing cached addons", env["name"]) addons = collect_addons(env) execute( - fetch_addon, + cache_addon, addons, - "fetch", + "cache", max_workers=args.max_workers, ctx=env["name"], ) logging.info( - "[%s] Fetching finishied in %.2f seconds", + "[%s] Cached addons refreshed in %.2f seconds", env["name"], time.monotonic() - start, ) @@ -491,13 +491,13 @@ def run_worker(worker, hooks=(), reverse=False, allow_failure=False): run_addon(addon, worker["name"], hooks=hooks, allow_failure=allow_failure) -def fetch_addon(addon, ctx="global"): +def cache_addon(addon, ctx="global"): addon_dir = os.path.join(ADDONS_DIR, addon["name"]) if not os.path.isdir(addon_dir): skip_addon(addon, ctx) return - hook = os.path.join(addon_dir, "fetch") + hook = os.path.join(addon_dir, "cache") if os.path.isfile(hook): run_hook(hook, (), ctx) diff --git a/test/drenv/cache.py b/test/drenv/cache.py index f4576bde82..25e1341aeb 100644 --- a/test/drenv/cache.py +++ b/test/drenv/cache.py @@ -4,9 +4,13 @@ import os import shutil import subprocess +import time from . import commands +REFRESH_THRESHOLD = 12 * 3600 +FETCH_THRESHOLD = 48 * 3600 + def clear(key=""): """ @@ -19,17 +23,34 @@ def clear(key=""): pass -def fetch(kustomization_dir, key, log=print): +def refresh(kustomization_dir, key, log=print): """ - Build kustomization and cache the output yaml. Retrun the path to the - cached yaml. + Rebuild the cache if cached item age is bigger than REFRESH_THRESHOLD. """ dest = _path(key) - if not os.path.exists(dest): + if _age(dest) > REFRESH_THRESHOLD: + _fetch(kustomization_dir, dest, log=log) + + +def get(kustomization_dir, key, log=print): + """ + Rebuild the cache if cached item age is bigger than FETCH_THRESHOLD. + Return the path to cached kustomization yaml. + """ + dest = _path(key) + if _age(dest) > FETCH_THRESHOLD: _fetch(kustomization_dir, dest, log=log) return dest +def _age(path): + try: + mtime = os.path.getmtime(path) + except FileNotFoundError: + mtime = 0 + return time.time() - mtime + + def _fetch(kustomization_dir, dest, log=print): log(f"Fetching {dest}") dest_dir = os.path.dirname(dest)