From 33e329fbff8c755552c5c8c9addc8742c3bf688d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 4 Oct 2024 08:51:47 +0200 Subject: [PATCH] Harmonize image selection across tests (#766) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse Co-authored-by: Kurt Garloff --- Tests/iaas/entropy/entropy-check.py | 2 +- Tests/iaas/image-metadata/image-md-check.py | 41 +++++++++++++++---- .../iaas/standard-images/images-openstack.py | 24 ++++++++--- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/Tests/iaas/entropy/entropy-check.py b/Tests/iaas/entropy/entropy-check.py index b24cfcf34..da4e8a7ef 100755 --- a/Tests/iaas/entropy/entropy-check.py +++ b/Tests/iaas/entropy/entropy-check.py @@ -437,7 +437,7 @@ def main(argv): all_flavors = conn.list_flavors(get_extra=True) if '*' not in image_visibility: - logger.debug(f"Images: filter for visibility {', '.join(image_visibility)}") + logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") all_images = [img for img in all_images if img.visibility in image_visibility] all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") diff --git a/Tests/iaas/image-metadata/image-md-check.py b/Tests/iaas/image-metadata/image-md-check.py index ec8b9fa35..77830d3a2 100755 --- a/Tests/iaas/image-metadata/image-md-check.py +++ b/Tests/iaas/image-metadata/image-md-check.py @@ -11,13 +11,18 @@ SPDX-License-Identifier: CC-BY-SA-4.0 """ +import calendar +from collections import Counter +import getopt +import logging import os import sys import time -import calendar -import getopt + import openstack -from collections import Counter + + +logger = logging.getLogger(__name__) def usage(ret): @@ -31,8 +36,10 @@ def usage(ret): print(" -v/--verbose : Be more verbose") print(" -s/--skip-completeness: Don't check whether we have all mandatory images") print(" -h/--help : Print this usage information") - print("If you pass images, only these will be validated, otherwise all (public unless") - print(" -p is specified) images from the catalog will be processed.") + print(" [-V/--image-visibility VIS_LIST] : filters images by visibility") + print(" (default: 'public,community'; use '*' to disable)") + print("If you pass images, only these will be validated, otherwise all images") + print("(filtered according to -p, -V) from the catalog will be processed.") sys.exit(ret) @@ -335,15 +342,19 @@ def miss_replacement_images(by_name, outd_list): def main(argv): "Main entry point" + # configure logging, disable verbose library logging + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + openstack.enable_logging(debug=False) # Option parsing global verbose + image_visibility = set() private = False skip = False cloud = os.environ.get("OS_CLOUD") err = 0 try: - opts, args = getopt.gnu_getopt(argv[1:], "phvc:s", - ("private", "help", "os-cloud=", "verbose", "skip-completeness")) + opts, args = getopt.gnu_getopt(argv[1:], "phvc:sV:", + ("private", "help", "os-cloud=", "verbose", "skip-completeness", "image-visibility=")) except getopt.GetoptError: # as exc: print("CRITICAL: Command-line syntax error", file=sys.stderr) usage(1) @@ -351,27 +362,39 @@ def main(argv): if opt[0] == "-h" or opt[0] == "--help": usage(0) elif opt[0] == "-p" or opt[0] == "--private": - private = True + private = True # only keep this for backwards compatibility (we have -V now) elif opt[0] == "-v" or opt[0] == "--verbose": verbose = True + logging.getLogger().setLevel(logging.DEBUG) elif opt[0] == "-s" or opt[0] == "--skip-completeness": skip = True elif opt[0] == "-c" or opt[0] == "--os-cloud": cloud = opt[1] + if opt[0] == "-V" or opt[0] == "--image-visibility": + image_visibility.update([v.strip() for v in opt[1].split(',')]) images = args if not cloud: print("CRITICAL: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr) usage(1) + if not image_visibility: + image_visibility.update(("public", "community")) + if private: + image_visibility.add("private") try: conn = openstack.connect(cloud=cloud, timeout=24) all_images = list(conn.image.images()) + if '*' not in image_visibility: + logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") + all_images = [img for img in all_images if img.visibility in image_visibility] + all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] + logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") by_name = {img.name: img for img in all_images} if len(by_name) != len(all_images): counter = Counter([img.name for img in all_images]) duplicates = [name for name, count in counter.items() if count > 1] print(f'WARNING: duplicate names detected: {", ".join(duplicates)}', file=sys.stderr) if not images: - images = [img.name for img in all_images if private or img.visibility == 'public'] + images = [img.name for img in all_images] # Analyse image metadata outdated_images = [] for imgnm in images: diff --git a/Tests/iaas/standard-images/images-openstack.py b/Tests/iaas/standard-images/images-openstack.py index 07a810b9b..6f192e5d0 100755 --- a/Tests/iaas/standard-images/images-openstack.py +++ b/Tests/iaas/standard-images/images-openstack.py @@ -40,6 +40,8 @@ def print_usage(file=sys.stderr): Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) [-d/--debug] enables DEBUG logging channel + [-V/--image-visibility VIS_LIST] filters images by visibility + (default: 'public,community'; use '*' to disable) """, end='', file=file) @@ -61,7 +63,7 @@ def main(argv): logger.addHandler(counting_handler) try: - opts, args = getopt.gnu_getopt(argv, "c:hd", ["os-cloud=", "help", "debug"]) + opts, args = getopt.gnu_getopt(argv, "c:hdV:", ["os-cloud=", "help", "debug", "image-visibility="]) except getopt.GetoptError as exc: logger.critical(f"{exc}") print_usage() @@ -74,6 +76,7 @@ def main(argv): yaml_path = args[0] cloud = os.environ.get("OS_CLOUD") + image_visibility = set() for opt in opts: if opt[0] == "-h" or opt[0] == "--help": print_usage() @@ -82,11 +85,16 @@ def main(argv): cloud = opt[1] if opt[0] == "-d" or opt[0] == "--debug": logging.getLogger().setLevel(logging.DEBUG) + if opt[0] == "-V" or opt[0] == "--image-visibility": + image_visibility.update([v.strip() for v in opt[1].split(',')]) if not cloud: logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") return 1 + if not image_visibility: + image_visibility.update(("public", "community")) + # we only support local files; but we allow specifying the following URLs for the sake of # better documentation prefix = next(p for p in ( @@ -113,11 +121,15 @@ def main(argv): logger.debug(f"Fetching image list from cloud '{cloud}'") with openstack.connect(cloud=cloud, timeout=32) as conn: present_images = conn.list_images(show_all=True) - by_name = { - image.name: image - for image in present_images - } - logger.debug(f"Images present: {', '.join(sorted(by_name))}") + if '*' not in image_visibility: + logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") + present_images = [img for img in present_images if img.visibility in image_visibility] + all_image_names = [f"{img.name} ({img.visibility})" for img in present_images] + logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") + by_name = { + image.name: image + for image in present_images + } logger.debug(f"Checking {len(image_specs)} image specs against {len(present_images)} images") for image_spec in image_specs: