feat(qemu): add checksum checks on qemu images

Before, images were not downloaded if a file on the given path already existed. Now, checksums are compared to determine whether the image should be downloaded. Configuration files must be expanded with a qemu.guest.<init>.url_checksum which points to the checksum file of the downloaded image. Solves issue #83 Signed-off-by: Nadja Brix Koch <[email protected]>
refenv · Dec 10, 2024 · 5c1dc8b · 5c1dc8b
1 parent 5564215
commit 5c1dc8b
Show file tree

Hide file tree

Showing 9 changed files with 127 additions and 17 deletions.
diff --git a/src/cijoe/core/misc.py b/src/cijoe/core/misc.py
@@ -1,5 +1,9 @@
 import errno
+import hashlib
+import logging as log
+import shutil
 from pathlib import Path
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 
 import requests
 
@@ -31,3 +35,82 @@ def download(url: str, path: Path):
                 local.write(chunk)
 
     return 0, path
+
+
+def get_checksums_from_url(url_checksum: str):
+    """
+    Downloads checksum(s) from given url to a temporary directory, returns
+    the hashing algorithm and the contents of the file. The algorithm is
+    found based on the filename of the downloaded checksum file.
+    Returns (err, checksums, algorithm).
+    """
+
+    dir = TemporaryDirectory()
+    err, path_checksum = download(url_checksum, Path(dir.name).resolve())
+    if err:
+        log.error(f"download({url_checksum}), {path_checksum}: failed")
+        return err, None, None
+
+    with open(path_checksum, "r") as checksum_f:
+        # Loop through valid hash algorithms to find the one that matches the
+        # filename of the checksum
+        for algorithm in hashlib.algorithms_guaranteed:
+            if algorithm in path_checksum.name.lower():
+                return 0, checksum_f.read(), algorithm
+
+    log.error(
+        "error: downloaded checksum does contain the name of a valid hash algorithm"
+    )
+    return 1, None, None
+
+
+def download_and_verify(url: str, url_checksum: str, path: Path):
+    """
+    Downloads a file over http(s). The file is only downloaded if checksums are
+    not equal and if the checksum at url_checksum matches the downloaded file,
+    returns (err, path).
+    """
+
+    path = Path(path).resolve()
+    if path.is_dir():
+        path = path / url.split("/")[-1]
+    if not (path.parent.is_dir() and path.parent.exists()):
+        return errno.EINVAL, path
+
+    err, verification, algorithm = get_checksums_from_url(url_checksum)
+    if err:
+        log.error(f"error when downloading checksum ({url_checksum})")
+
+    # The checksum of the existing file at path
+    checksum = None
+    checksum_path = path.with_suffix(path.suffix + f".{algorithm}sum")
+    if path.exists() and checksum_path.exists():
+        with open(checksum_path, "r") as f:
+            checksum = f.read()
+
+    # If the file does not already exists or if checksums do not match,
+    # download from url
+    if not checksum or checksum not in verification:
+        log.info(f"Downloading file from {url}")
+
+        with NamedTemporaryFile() as dwnld:
+            err, dwnld_path = download(url, Path(dwnld.name).resolve())
+            if err:
+                log.error(f"download({url}), {dwnld_path}: failed")
+                return err, None
+
+            # Check that downloaded checksum file contains the checksum of
+            # the downloaded file
+            new_checksum = hashlib.file_digest(dwnld, algorithm).hexdigest()
+            if new_checksum not in verification:
+                log.error(
+                    f"error while downloading file: checksum ({new_checksum}) not in checksum file:\n{verification}"
+                )
+                return 1, None
+
+            # Move contents of temporary file to the given path
+            shutil.copyfile(dwnld_path, path)
+            with open(checksum_path, "w") as f:
+                f.write(new_checksum)
+
+    return 0, path
diff --git a/src/cijoe/qemu/configs/debian-bookworm-amd64-kvm.toml b/src/cijoe/qemu/configs/debian-bookworm-amd64-kvm.toml
@@ -48,10 +48,12 @@ system_args.host_share = "{{ local.env.HOME }}/git"
 
 # Used by: qemu.guest_init_using_bootimage.py
 init_using_bootimage.url = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bookworm-amd64.qcow2"
+init_using_bootimage.url_checksum = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bookworm-amd64.qcow2.sha256"
 init_using_bootimage.img = "{{ local.env.HOME }}/images/boot_images/debian-bookworm-amd64.qcow2"
 
 # Used by: qemu.guest_init_using_cloudinit
 init_using_cloudinit.url = "https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-amd64-daily.qcow2"
+init_using_cloudinit.url_checksum = "https://cloud.debian.org/images/cloud/bookworm/daily/latest/SHA512SUMS"
 init_using_cloudinit.img = "{{ local.env.HOME }}/images/cloudinit/debian-12-generic-amd64-daily.qcow2"
 init_using_cloudinit.meta = "{{ resources.auxiliary['qemu.cloudinit-debian-meta'] }}"
 init_using_cloudinit.user = "{{ resources.auxiliary['qemu.cloudinit-debian-user-amd64'] }}"

diff --git a/src/cijoe/qemu/configs/debian-bookworm-arm64-hvf.toml b/src/cijoe/qemu/configs/debian-bookworm-arm64-hvf.toml
@@ -48,10 +48,12 @@ system_args.host_share = "{{ local.env.HOME }}/git"
 
 # Used by: qemu.guest_init_using_bootimage.py
 init_using_bootimage.url = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bookworm-arm64.qcow2"
+init_using_bootimage.url_checksum = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bookworm-arm64.qcow2.sha256"
 init_using_bootimage.img = "{{ local.env.HOME }}/images/boot_images/debian-bookworm-arm64.qcow2"
 
 # Used by: qemu.guest_init_using_cloudinit.py
 init_using_cloudinit.url = "https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-arm64-daily.qcow2"
+init_using_cloudinit.url_checksum = "https://cloud.debian.org/images/cloud/bookworm/daily/latest/SHA512SUMS"
 init_using_cloudinit.img = "{{ local.env.HOME }}/images/cloudinit/debian-12-generic-arm64-daily.qcow2"
 init_using_cloudinit.meta = "{{ resources.auxiliary['qemu.cloudinit-debian-meta'] }}"
 init_using_cloudinit.user = "{{ resources.auxiliary['qemu.cloudinit-debian-user-arm64'] }}"

diff --git a/src/cijoe/qemu/configs/debian-bullseye-amd64-kvm.toml b/src/cijoe/qemu/configs/debian-bullseye-amd64-kvm.toml
@@ -48,10 +48,12 @@ system_args.host_share = "{{ local.env.HOME }}/git"
 
 # Used by: qemu.guest_init_using_bootimage.py
 init_using_bootimage.url = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bullseye-amd64.qcow2"
+init_using_bootimage.url_checksum = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bullseye-amd64.qcow2.sha256"
 init_using_bootimage.img = "{{ local.env.HOME }}/images/boot_images/debian-bullseye-amd64.qcow2"
 
 # Used by: qemu.guest_init_using_cloudinit.py
 init_using_cloudinit.url = "https://cloud.debian.org/images/cloud/bullseye/daily/latest/debian-11-generic-amd64-daily.qcow2"
+init_using_cloudinit.url_checksum = "https://cloud.debian.org/images/cloud/bullseye/daily/latest/SHA512SUMS"
 init_using_cloudinit.img = "{{ local.env.HOME }}/images/cloudinit/debian-11-generic-amd64-daily.qcow2"
 init_using_cloudinit.meta = "{{ resources.auxiliary['qemu.cloudinit-debian-meta'] }}"
 init_using_cloudinit.user = "{{ resources.auxiliary['qemu.cloudinit-debian-user-amd64'] }}"

diff --git a/src/cijoe/qemu/configs/default-config.toml b/src/cijoe/qemu/configs/default-config.toml
@@ -48,10 +48,12 @@ system_args.host_share = "{{ local.env.HOME }}/git"
 
 # Used by: qemu.guest_init_using_bootimage.py
 init_using_bootimage.url = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bullseye-amd64.qcow2"
+init_using_bootimage.url_checksum = "https://refenv.fra1.digitaloceanspaces.com/boot_images/debian-bullseye-amd64.qcow2.sha256"
 init_using_bootimage.img = "{{ local.env.HOME }}/images/boot_images/debian-bullseye-amd64.qcow2"
 
 # Used by: qemu.guest_init_using_cloudinit.py
 init_using_cloudinit.url = "https://cloud.debian.org/images/cloud/bullseye/daily/latest/debian-11-generic-amd64-daily.qcow2"
+init_using_cloudinit.url_checksum = "https://cloud.debian.org/images/cloud/bullseye/daily/latest/SHA512SUMS"
 init_using_cloudinit.img = "{{ local.env.HOME }}/images/cloudinit/debian-11-generic-amd64-daily.qcow2"
 init_using_cloudinit.meta = "{{ resources.auxiliary['qemu.cloudinit-debian-meta'] }}"
 init_using_cloudinit.user = "{{ resources.auxiliary['qemu.cloudinit-debian-user-amd64'] }}"

diff --git a/src/cijoe/qemu/configs/freebsd-13-amd64-kvm.toml b/src/cijoe/qemu/configs/freebsd-13-amd64-kvm.toml
@@ -48,10 +48,12 @@ system_args.host_share = "{{ local.env.HOME }}/git"
 
 # Used by: qemu.guest_init_using_bootimage.py
 init_using_bootimage.url = "https://refenv.fra1.digitaloceanspaces.com/boot_images/freebsd-13.1-ksrc-amd64.qcow2"
+init_using_bootimage.url_checksum = "https://refenv.fra1.digitaloceanspaces.com/boot_images/freebsd-13.1-ksrc-amd64.qcow2.sha256"
 init_using_bootimage.img = "{{ local.env.HOME }}/images/boot_images/freebsd-13-amd64.qcow2"
 
 # Used by: qemu.guest_init_using_cloudinit.py
 init_using_cloudinit.url = "https://refenv.fra1.digitaloceanspaces.com/freebsd13-ufs-ksrc.qcow2"
+init_using_cloudinit.url_checksum = "https://refenv.fra1.digitaloceanspaces.com/freebsd13-ufs-ksrc.qcow2.sha256"
 init_using_cloudinit.img = "{{ local.env.HOME}}/images/cloudinit/freebsd13-ufs-ksrc.qcow2"
 init_using_cloudinit.meta = "{{ resources.auxiliary['qemu.cloudinit-freebsd-meta'] }}"
 init_using_cloudinit.user = "{{ resources.auxiliary['qemu.cloudinit-freebsd-user'] }}"

diff --git a/src/cijoe/qemu/scripts/guest_init_using_bootimage.py b/src/cijoe/qemu/scripts/guest_init_using_bootimage.py
@@ -11,6 +11,8 @@
 
     [qemu.guest.init_using_bootimage]
     url = # URL pointing to download location of the bootable disk-image
+    url_checksum = # URL pointing to download location of the checksum of the 
+                   # bootable disk-image
     img = # Absolute path to disk-image file"
 
 Retargetable: False

diff --git a/src/cijoe/qemu/scripts/guest_init_using_cloudinit.py b/src/cijoe/qemu/scripts/guest_init_using_cloudinit.py
@@ -14,6 +14,8 @@
 
     [qemu.guest.init_using_cloudinit]
     url = # URL of cloud-init image, e.g. on https://cloud.debian.org/images/cloud/
+    url_checksum = # URL pointing to download location of the checksum of the 
+                   # cloud-init image
     img = # Path to cloud-init image
     meta = # Path to cloud-init meta-file
     user = # Path to cloud-init user-file

diff --git a/src/cijoe/qemu/wrapper.py b/src/cijoe/qemu/wrapper.py
@@ -18,7 +18,7 @@
 
 import psutil
 
-from cijoe.core.misc import download
+from cijoe.core.misc import download_and_verify
 
 
 def qemu_img(cijoe, args=""):
@@ -233,22 +233,22 @@ def init_using_cloudinit(self):
             log.error("missing config([qemu.guest.init_using_cloudinit])")
             return 1
 
-        img = cloudinit.get("img", None)
-        if not img:
-            log.error("missing config([qemu.guest.init_using_cloudinit.img])")
+        if not all(key in cloudinit for key in ["img", "url", "url_checksum"]):
+            log.error(
+                'missing config. [qemu.guest.init_using_cloudinit] must have keys "img", "url", "url_checksum"'
+            )
             return 1
 
+        url = cloudinit["url"]
+        url_checksum = cloudinit["url_checksum"]
+        img = cloudinit["img"]
+
         img = Path(img).resolve()
         if not img.exists():
-            url = cloudinit.get("url", None)
-            if not url:
-                log.error("missing config([qemu.guest.init_using_cloudinit.url])")
-                return 1
-
             img.parent.mkdir(parents=True, exist_ok=True)
-            err, path = download(url, img)
+            err, img = download_and_verify(url, url_checksum, img)
             if err:
-                log.error(f"download({url}), {path}: failed")
+                log.error(f"download({url}), {img}: failed")
                 return err
 
         # Create the boot.img based on cloudinit_img
@@ -307,17 +307,30 @@ def init_using_bootimage(self):
 
         # Ensure the guest has an image available to boot from
         boot = self.guest_config.get("init_using_bootimage", {})
-        boot["img"] = Path(boot["img"]).resolve()
+        if not boot:
+            log.error("missing config([qemu.guest.init_using_bootimage])")
+            return 1
+
+        if not all(key in boot for key in ["img", "url", "url_checksum"]):
+            log.error(
+                'missing config. [qemu.guest.init_using_bootimage] must have keys "img", "url", "url_checksum"'
+            )
+            return 1
 
-        if not boot["img"].exists():
-            os.makedirs(boot["img"].parent, exist_ok=True)
-            err, path = download(boot["url"], str(boot["img"]))
+        url = boot["url"]
+        url_checksum = boot["url_checksum"]
+        img = boot["img"]
+
+        img = Path(img).resolve()
+        if not img.exists():
+            img.parent.mkdir(parents=True, exist_ok=True)
+            err, img = download_and_verify(url, url_checksum, img)
             if err:
-                log.error(f"download({boot['url']}), {path}: failed")
+                log.error(f"download({url}), {img}: failed")
                 return err
 
         # Create the boot.img based on cloudinit_img
-        shutil.copyfile(str(boot["img"]), str(self.boot_img))
+        shutil.copyfile(str(img), str(self.boot_img))
         qemu_img(self.cijoe, f"resize {self.boot_img} 10G")
 
         return 0