From 7677d7fe2e28b85b7238505e89eeb12a77a243e2 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 2 Jul 2023 22:34:36 +0200 Subject: [PATCH] Cleanup base sources Signed-off-by: Christian Berendt --- files/cleanup-base-sources.sh | 16 +++++++ scripts/004-build.sh | 23 ++++++--- src/cleanup-images.py | 88 +++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 6 deletions(-) create mode 100755 files/cleanup-base-sources.sh create mode 100644 src/cleanup-images.py diff --git a/files/cleanup-base-sources.sh b/files/cleanup-base-sources.sh new file mode 100755 index 00000000..4e4fe65e --- /dev/null +++ b/files/cleanup-base-sources.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +for e in $(find /*-base-source -name '*config-generator*'); do + if [[ -d $e ]]; then + mkdir -p /x/$e + cp -r $e/* /x/$e + else + f=$(dirname $e) + mkdir -p /x/$f + cp -r $f/* /x/$f + fi +done + +rm -rf /*-base-source +mv /x/* / +rm -rf /x diff --git a/scripts/004-build.sh b/scripts/004-build.sh index a7db31a9..83ab6957 100755 --- a/scripts/004-build.sh +++ b/scripts/004-build.sh @@ -9,7 +9,6 @@ set -x # BUILD_TYPE # KOLLA_IMAGES # OPENSTACK_VERSION -# SQUASH # VERSION # Set default values @@ -18,7 +17,6 @@ BUILD_ID=${BUILD_ID:-$(date +%Y%m%d)} BUILD_TYPE=${BUILD_TYPE:-all} OPENSTACK_VERSION=${OPENSTACK_VERSION:-latest} VERSION=${VERSION:-latest} -SQUASH=${SQUASH:-false} KOLLA_CONF=kolla-build.conf @@ -33,10 +31,6 @@ fi export VERSION export OPENSTACK_VERSION -if [[ $SQUASH == "true" ]]; then - BUILD_OPTS+=" --squash" -fi - if [[ -z "$KOLLA_IMAGES" ]]; then KOLLA_IMAGES="$(python3 src/get-projects-from-versions-file.py)" fi @@ -51,6 +45,9 @@ if [[ $BUILD_TYPE == "base" ]]; then fi done + # The line can be commented out for tests to build keystone images only. + KOLLA_IMAGES_BASE=^keystone-base + kolla-build \ --template-override templates/$OPENSTACK_VERSION/template-overrides.j2 \ --config-file $KOLLA_CONF \ @@ -58,6 +55,9 @@ if [[ $BUILD_TYPE == "base" ]]; then $BUILD_OPTS \ $KOLLA_IMAGES_BASE 2>&1 | tee kolla-build-$BUILD_ID.log else + # The line can be commented out for tests to build keystone images only. + KOLLA_IMAGES=^keystone + kolla-build \ --template-override templates/$OPENSTACK_VERSION/template-overrides.j2 \ --config-file $KOLLA_CONF \ @@ -71,4 +71,15 @@ if grep -q "Failed with status: error" kolla-build-$BUILD_ID.log; then exit 1 fi +# Cleanup images + +export DOCKER_NAMESPACE=${DOCKER_NAMESPACE:-osism} +export DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io} + +python3 src/cleanup-images.py + +# List images + docker images + +docker inspect testing diff --git a/src/cleanup-images.py b/src/cleanup-images.py new file mode 100644 index 00000000..dca18fa6 --- /dev/null +++ b/src/cleanup-images.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +import os +import shutil +import sys +import tempfile + +from docker import APIClient, DockerClient +from loguru import logger + +DOCKER_NAMESPACE = os.environ.get("DOCKER_NAMESPACE", "osism") +DOCKER_REGISTRY = os.environ.get("DOCKER_REGISTRY", "quay.io") + +VERSION = os.environ.get("OPENSTACK_VERSION", "zed") +FILTERS = {"label": f"de.osism.release.openstack={VERSION}"} + +SUPPORTED_IMAGES = ["keystone"] + +level = "INFO" +log_fmt = ( + "{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | " + "{message}" +) + +logger.remove() +logger.add(sys.stderr, format=log_fmt, level=level, colorize=True) + +client = DockerClient() +client_api = APIClient() + +for image in client.images.list(filters=FILTERS): + # skip images without a tag + if not image.tags: + continue + + tag = image.tags[0] + + # Check if the image is supported + if not [x for x in SUPPORTED_IMAGES if x in image.labels["name"]]: + logger.info(f"Image {tag} not supported") + continue + + logger.info(f"Cleaning up image {tag}") + + # Image layers + + # 0: "sha256:59c56aee1fb4dbaeb334aef06088b49902105d1ea0c15a9e5a2a9ce560fa4c5d" <-- ubuntu:22.04 + # 1: "sha256:295ebbacd98b98d173f5fba0371e4fee8bbdf9ff0ad2676670ddb2b521482004" <-- base + # 2: "sha256:bd9f5e1f11d685268d42d7bf3a7850fdd7b6767bfcfbdb1bcc0b31227dae8f91" <-- openstack-base + # 3: "sha256:6ae01f1fd7a8c29837ddc7ed50e78ba70666f5967bdaf8438f07da2971d3d024" <-- service-base + # 4: "sha256:591d1c8305854fabc586786673d87f55eb142f0efed7a002f696c6a1c4c26880" <-- service + + with tempfile.TemporaryDirectory() as d: + shutil.copy("files/cleanup-base-sources.sh", d) + + dockerfile = os.path.join(d, "Dockerfile") + logger.info(f"Generating Dockerfile in {d}") + + # Get current username + result = client.containers.run(tag, "bash -c 'whoami'") + username = result.decode("utf-8") + + with open(dockerfile, "w+") as fp: + # Cleanup the sources + fp.write(f"FROM {tag} as cleanup\n") + fp.write("COPY cleanup-base-sources.sh /cleanup-base-sources.sh\n") + fp.write("USER root\n") + fp.write("RUN bash /cleanup-base-sources.sh\n") + fp.write("RUN rm /cleanup-base-sources.sh\n") + fp.write(f"USER {username}\n") + fp.write("\n") + + # Rebase on openstack-base + fp.write( + f"FROM {DOCKER_REGISTRY}/{DOCKER_NAMESPACE}/openstack-base:{tag.split(':')[1]}\n" + ) + fp.write("COPY --from=cleanup / /\n") + + with open(dockerfile, "r+") as fp: + logger.info(fp.read()) + + client.images.build(path=d, tag=f"{tag}-after-cleanup") + + # Re-tag the image + client.images.remove(tag, force=True) + client.api.tag(f"{tag}-after-cleanup", tag) + client.api.tag(f"{tag}-after-cleanup", "testing") + client.images.remove("{tag}-after-cleanup", force=True)