Force update script to download the image and calculate SHA512

Since we want to check the hash against the openstack backend,
we need to have the SHA512. Sadly, most images creators do not
provide us with that hash pre-computed. That means we will
compute the SHA512 for every new image update in the CI worker
by downloading the image.

Signed-off-by: Gondermann <[email protected]>

openstack_image_manager/update.py

@@ -2,6 +2,8 @@
 # source of latest URLs: https://gitlab.com/libosinfo/osinfo-db
 
 from datetime import datetime
+import hashlib
+import math
 import os
 import re
 import shutil
@@ -157,6 +159,39 @@ def mirror_image(
         client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename)
         os.remove(filename)
 
+def size_clean(size):
+    size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
+    i = int(math.floor(math.log(size, 1024)))
+    s = size / 1024 ** i
+    return f"{s:.2f} {size_name[i]}"
+
+def headers_and_sha512(download_url: str):
+    hash_obj = hashlib.new("sha512")
+
+    file_headers = None
+    with requests.get(url=download_url, stream=True, timeout=30) as response:
+        if response.status_code != 200:
+            logger.error(f"Downloading image '{download_url}' failed with error code {response.status_code}")
+            return None, None
+
+        file_headers = response.headers
+        file_size = int(file_headers["Content-Length"])
+        logger.info(f"Image size {size_clean(file_size)}")
+
+        downloadedBytes = 0
+        lastProgress = 0
+        for chunk in response.iter_content(chunk_size=8192):
+            downloadedBytes += 8192
+            progressPercent = (downloadedBytes / file_size) * 100
+            progress = round(min(max(progressPercent, 0), 100))
+            if progress - lastProgress >= 5:
+                logger.info(f"Downloading image: {progress}%")
+                lastProgress = progress
+
+            hash_obj.update(chunk)
+
+    sha512 = hash_obj.hexdigest()
+    return file_headers, f"sha512:{sha512}"
 
 def update_image(image, getter, minio_server, minio_bucket, minio_access_key, minio_secret_key):
     name = image["name"]
@@ -181,6 +216,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi
                 "checksum": None,
                 "url": None,
                 "version": None,
+                "verify_checksum": None
             }
         )
 
@@ -191,12 +227,15 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi
         logger.info(f"Image {name} is up-to-date, nothing to do")
         return 0
 
+    headers, verify_checksum = headers_and_sha512(current_url)
+    if verify_checksum == None:
+        return 0
+
     if current_version is None:
         logger.info(f"Checking {current_url}")
 
-        conn = urlopen(current_url, timeout=30)
         dt = datetime.strptime(
-            conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
+            headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
         )
         current_version = dt.strftime("%Y%m%d")
 
@@ -205,6 +244,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi
         "build_date": datetime.strptime(current_version, "%Y%m%d").date(),
         "checksum": current_checksum,
         "url": current_url,
+        "verify_checksum": verify_checksum,
     }
     logger.info(f"New values are {new_values}")
     image["versions"][0].update(new_values)