diff --git a/test/addons/cdi/fetch b/test/addons/cdi/cache similarity index 65% rename from test/addons/cdi/fetch rename to test/addons/cdi/cache index 9661e386f..d45b306b9 100755 --- a/test/addons/cdi/fetch +++ b/test/addons/cdi/cache @@ -7,5 +7,5 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch("operator", "addons/cdi-operator.yaml") -cache.fetch("cr", "addons/cdi-cr.yaml") +cache.refresh("operator", "addons/cdi-operator.yaml") +cache.refresh("cr", "addons/cdi-cr.yaml") diff --git a/test/addons/cdi/start b/test/addons/cdi/start index 336c9ad1f..625beae2f 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -14,7 +14,7 @@ NAMESPACE = "cdi" def deploy(cluster): print("Deploying cdi operator") - path = cache.fetch("operator", "addons/cdi-operator.yaml") + path = cache.get("operator", "addons/cdi-operator.yaml") kubectl.apply("--filename", path, context=cluster) print("Waiting until cdi-operator is rolled out") @@ -27,7 +27,7 @@ def deploy(cluster): ) print("Deploying cdi cr") - path = cache.fetch("cr", "addons/cdi-cr.yaml") + path = cache.get("cr", "addons/cdi-cr.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-cephfs/fetch b/test/addons/csi-addons/cache similarity index 80% rename from test/addons/rook-cephfs/fetch rename to test/addons/csi-addons/cache index dd0141dc8..7b37a5978 100755 --- a/test/addons/rook-cephfs/fetch +++ b/test/addons/csi-addons/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-cephfs.yaml") +cache.refresh(".", "addons/csi-addons.yaml") diff --git a/test/addons/csi-addons/start b/test/addons/csi-addons/start index d206387b2..7c1ffa788 100755 --- a/test/addons/csi-addons/start +++ b/test/addons/csi-addons/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying csi addon for volume replication") - path = cache.fetch(".", "addons/csi-addons.yaml") + path = cache.get(".", "addons/csi-addons.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/kubevirt/fetch b/test/addons/kubevirt/cache similarity index 62% rename from test/addons/kubevirt/fetch rename to test/addons/kubevirt/cache index 304058b84..9f3c23506 100755 --- a/test/addons/kubevirt/fetch +++ b/test/addons/kubevirt/cache @@ -7,5 +7,5 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch("operator", "addons/kubevirt-operator.yaml") -cache.fetch("cr", "addons/kubevirt-cr.yaml") +cache.refresh("operator", "addons/kubevirt-operator.yaml") +cache.refresh("cr", "addons/kubevirt-cr.yaml") diff --git a/test/addons/kubevirt/start b/test/addons/kubevirt/start index 14a022ab5..5cd6f32a0 100755 --- a/test/addons/kubevirt/start +++ b/test/addons/kubevirt/start @@ -14,7 +14,7 @@ NAMESPACE = "kubevirt" def deploy(cluster): print("Deploying kubevirt operator") - path = cache.fetch("operator", "addons/kubevirt-operator.yaml") + path = cache.get("operator", "addons/kubevirt-operator.yaml") kubectl.apply("--filename", path, context=cluster) print("Waiting until virt-operator is rolled out") @@ -27,7 +27,7 @@ def deploy(cluster): ) print("Deploying kubevirt cr") - path = cache.fetch("cr", "addons/kubevirt-cr.yaml") + path = cache.get("cr", "addons/kubevirt-cr.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/ocm-controller/cache b/test/addons/ocm-controller/cache new file mode 100755 index 000000000..bc4004e18 --- /dev/null +++ b/test/addons/ocm-controller/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/ocm-controller.yaml") diff --git a/test/addons/ocm-controller/start b/test/addons/ocm-controller/start index 431c5eca8..6b30c74f6 100755 --- a/test/addons/ocm-controller/start +++ b/test/addons/ocm-controller/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying ocm controller") - path = cache.fetch(".", "addons/ocm-controller.yaml") + path = cache.get(".", "addons/ocm-controller.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/recipe/fetch b/test/addons/recipe/cache similarity index 81% rename from test/addons/recipe/fetch rename to test/addons/recipe/cache index 221500f14..a094d54c5 100755 --- a/test/addons/recipe/fetch +++ b/test/addons/recipe/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/recipe.yaml") +cache.refresh(".", "addons/recipe.yaml") diff --git a/test/addons/recipe/start b/test/addons/recipe/start index b1bb697cc..bd0bdbec7 100755 --- a/test/addons/recipe/start +++ b/test/addons/recipe/start @@ -16,5 +16,5 @@ os.chdir(os.path.dirname(__file__)) cluster = sys.argv[1] print("Deploying recipe crd") -path = cache.fetch(".", "addons/recipe.yaml") +path = cache.get(".", "addons/recipe.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/csi-addons/fetch b/test/addons/rook-cephfs/cache similarity index 79% rename from test/addons/csi-addons/fetch rename to test/addons/rook-cephfs/cache index 71c61cc52..56205b17e 100755 --- a/test/addons/csi-addons/fetch +++ b/test/addons/rook-cephfs/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/csi-addons.yaml") +cache.refresh(".", "addons/rook-cephfs.yaml") diff --git a/test/addons/rook-cephfs/start b/test/addons/rook-cephfs/start index d8dcbaceb..489b1b1e7 100755 --- a/test/addons/rook-cephfs/start +++ b/test/addons/rook-cephfs/start @@ -13,7 +13,7 @@ from drenv import kubectl def deploy(cluster): print("Creating CephFS instance and storage/snapshot classes") - path = cache.fetch(".", "addons/rook-cephfs.yaml") + path = cache.get(".", "addons/rook-cephfs.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/ocm-controller/fetch b/test/addons/rook-cluster/cache similarity index 79% rename from test/addons/ocm-controller/fetch rename to test/addons/rook-cluster/cache index 37c55444f..1da61bb7f 100755 --- a/test/addons/ocm-controller/fetch +++ b/test/addons/rook-cluster/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/ocm-controller.yaml") +cache.refresh(".", "addons/rook-cluster.yaml") diff --git a/test/addons/rook-cluster/fetch b/test/addons/rook-cluster/fetch deleted file mode 100755 index ddf1c53d3..000000000 --- a/test/addons/rook-cluster/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-cluster.yaml") diff --git a/test/addons/rook-cluster/start b/test/addons/rook-cluster/start index 530d9507a..333b6b920 100755 --- a/test/addons/rook-cluster/start +++ b/test/addons/rook-cluster/start @@ -20,7 +20,7 @@ TIMEOUT = 600 def deploy(cluster): print("Deploying rook ceph cluster") - path = cache.fetch(".", "addons/rook-cluster.yaml") + path = cache.get(".", "addons/rook-cluster.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-operator/cache b/test/addons/rook-operator/cache new file mode 100755 index 000000000..d328f5804 --- /dev/null +++ b/test/addons/rook-operator/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/rook-operator.yaml") diff --git a/test/addons/rook-operator/fetch b/test/addons/rook-operator/fetch deleted file mode 100755 index 4b0d8bae9..000000000 --- a/test/addons/rook-operator/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-operator.yaml") diff --git a/test/addons/rook-operator/start b/test/addons/rook-operator/start index 62cf3e7a4..48275ca70 100755 --- a/test/addons/rook-operator/start +++ b/test/addons/rook-operator/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying rook ceph operator") - path = cache.fetch(".", "addons/rook-operator.yaml") + path = cache.get(".", "addons/rook-operator.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-toolbox/cache b/test/addons/rook-toolbox/cache new file mode 100755 index 000000000..faf716ebe --- /dev/null +++ b/test/addons/rook-toolbox/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/rook-toolbox.yaml") diff --git a/test/addons/rook-toolbox/fetch b/test/addons/rook-toolbox/fetch deleted file mode 100755 index 572695efa..000000000 --- a/test/addons/rook-toolbox/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-toolbox.yaml") diff --git a/test/addons/rook-toolbox/start b/test/addons/rook-toolbox/start index 8ebe28710..161d9ee9c 100755 --- a/test/addons/rook-toolbox/start +++ b/test/addons/rook-toolbox/start @@ -13,7 +13,7 @@ from drenv import cache def deploy(cluster): print("Deploying rook ceph toolbox") - path = cache.fetch(".", "addons/rook-toolbox.yaml") + path = cache.get(".", "addons/rook-toolbox.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index c18277406..52e78216e 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -85,7 +85,7 @@ def parse_args(): add_command(sp, "suspend", do_suspend, help="suspend virtual machines") add_command(sp, "resume", do_resume, help="resume virtual machines") add_command(sp, "dump", do_dump, help="dump an environment yaml") - add_command(sp, "fetch", do_fetch, help="cache environment resources") + add_command(sp, "cache", do_cache, help="cache environment resources") add_command(sp, "clear", do_clear, help="cleared cached resources", envfile=False) add_command(sp, "setup", do_setup, help="setup minikube for drenv", envfile=False) @@ -177,20 +177,20 @@ def do_clear(args): cache.clear() -def do_fetch(args): +def do_cache(args): env = load_env(args) start = time.monotonic() - logging.info("[%s] Fetching", env["name"]) + logging.info("[%s] Refreshing cached addons", env["name"]) addons = collect_addons(env) execute( - fetch_addon, + cache_addon, addons, - "fetch", + "cache", max_workers=args.max_workers, ctx=env["name"], ) logging.info( - "[%s] Fetching finishied in %.2f seconds", + "[%s] Cached addons refreshed in %.2f seconds", env["name"], time.monotonic() - start, ) @@ -491,13 +491,13 @@ def run_worker(worker, hooks=(), reverse=False, allow_failure=False): run_addon(addon, worker["name"], hooks=hooks, allow_failure=allow_failure) -def fetch_addon(addon, ctx="global"): +def cache_addon(addon, ctx="global"): addon_dir = os.path.join(ADDONS_DIR, addon["name"]) if not os.path.isdir(addon_dir): skip_addon(addon, ctx) return - hook = os.path.join(addon_dir, "fetch") + hook = os.path.join(addon_dir, "cache") if os.path.isfile(hook): run_hook(hook, (), ctx) diff --git a/test/drenv/cache.py b/test/drenv/cache.py index f4576bde8..25e1341ae 100644 --- a/test/drenv/cache.py +++ b/test/drenv/cache.py @@ -4,9 +4,13 @@ import os import shutil import subprocess +import time from . import commands +REFRESH_THRESHOLD = 12 * 3600 +FETCH_THRESHOLD = 48 * 3600 + def clear(key=""): """ @@ -19,17 +23,34 @@ def clear(key=""): pass -def fetch(kustomization_dir, key, log=print): +def refresh(kustomization_dir, key, log=print): """ - Build kustomization and cache the output yaml. Retrun the path to the - cached yaml. + Rebuild the cache if cached item age is bigger than REFRESH_THRESHOLD. """ dest = _path(key) - if not os.path.exists(dest): + if _age(dest) > REFRESH_THRESHOLD: + _fetch(kustomization_dir, dest, log=log) + + +def get(kustomization_dir, key, log=print): + """ + Rebuild the cache if cached item age is bigger than FETCH_THRESHOLD. + Return the path to cached kustomization yaml. + """ + dest = _path(key) + if _age(dest) > FETCH_THRESHOLD: _fetch(kustomization_dir, dest, log=log) return dest +def _age(path): + try: + mtime = os.path.getmtime(path) + except FileNotFoundError: + mtime = 0 + return time.time() - mtime + + def _fetch(kustomization_dir, dest, log=print): log(f"Fetching {dest}") dest_dir = os.path.dirname(dest)