From d1dbf0a30721f0b232eb1b7f453139e5e19d15ed Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Thu, 24 Oct 2024 15:23:50 +0200 Subject: [PATCH 1/7] cleanup-images: remove --exit-code I'm not sure why this was there in the first place. AFAICT that flag doesn't exist and grep already sets the exit code depending on whether or not a match was found. --- tools/cleanup-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cleanup-images.sh b/tools/cleanup-images.sh index 846148526a..0b153527fd 100644 --- a/tools/cleanup-images.sh +++ b/tools/cleanup-images.sh @@ -37,7 +37,7 @@ for image in "${pauseImages[@]}"; do ctr "${ctrOpts[@]}" content fetch "${image}" done -if ctr "${ctrOpts[@]}" image check | grep --exit-code "incomplete"; then +if ctr "${ctrOpts[@]}" image check | grep "incomplete"; then echo "Incomplete images detected" exit 1 fi From 1c8093dde27bef775c13861934457755840ec1e5 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Thu, 24 Oct 2024 15:25:54 +0200 Subject: [PATCH 2/7] cleanup-bm: don't fail when no node-installer is found grep returns 1 if no match is found. Don't do that, we don't want to fail here. --- packages/cleanup-bm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cleanup-bm.sh b/packages/cleanup-bm.sh index 0fe87b1900..38975d5e3f 100755 --- a/packages/cleanup-bm.sh +++ b/packages/cleanup-bm.sh @@ -24,7 +24,7 @@ done kubectl get pods --all-namespaces -o jsonpath='{.items[?(@.metadata.annotations.contrast\.edgeless\.systems/pod-role=="contrast-node-installer")].spec.containers[0].args[1]}' | tr ' ' '\n' | - grep -o "contrast-cc-.\+" >>usedRuntimeClasses + grep -o "contrast-cc-.\+" >>usedRuntimeClasses || true sort -u usedRuntimeClasses -o usedRuntimeClasses mapfile -t unusedRuntimeClasses < <( From 9e3102baa9598deb64d2333570346ce194d75ddd Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Thu, 24 Oct 2024 15:27:52 +0200 Subject: [PATCH 3/7] scripts: add cleanup-images --- {tools => packages}/cleanup-images.sh | 0 packages/scripts.nix | 10 ++++++++++ 2 files changed, 10 insertions(+) rename {tools => packages}/cleanup-images.sh (100%) diff --git a/tools/cleanup-images.sh b/packages/cleanup-images.sh similarity index 100% rename from tools/cleanup-images.sh rename to packages/cleanup-images.sh diff --git a/packages/scripts.nix b/packages/scripts.nix index 050c6e3e01..a7670027fc 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -500,4 +500,14 @@ ]; text = builtins.readFile ./cleanup-bm.sh; }; + + cleanup-images = writeShellApplication { + name = "cleanup-images"; + runtimeInputs = with pkgs; [ + gnugrep + busybox + containerd + ]; + text = builtins.readFile ./cleanup-images.sh; + }; } From 0aeaff745b6355268ace15bc150299524ebe7d9a Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Thu, 24 Oct 2024 15:28:51 +0200 Subject: [PATCH 4/7] cleanup-bm: run cleanup-images Removing the data directories for the snapshotters isn't quite enough, we also need to tell containerd to clean up its state. --- packages/cleanup-bm.sh | 3 +++ packages/containers.nix | 3 +++ packages/scripts.nix | 1 + tools/bm-maintenance/deployment_tdx_snp.yml | 6 ++++++ 4 files changed, 13 insertions(+) diff --git a/packages/cleanup-bm.sh b/packages/cleanup-bm.sh index 38975d5e3f..c284d1efd9 100755 --- a/packages/cleanup-bm.sh +++ b/packages/cleanup-bm.sh @@ -56,3 +56,6 @@ for runtimeClass in "${unusedRuntimeClasses[@]}"; do dasel delete --file "${CONFIG}" --indent 0 --read toml --write toml "plugins.io\.containerd\.grpc\.v1\.cri.containerd.runtimes.${runtimeClass}" 2>/dev/null dasel delete --file "${CONFIG}" --indent 0 --read toml --write toml "proxy_plugins.${SNAPSHOTTER}-${runtimeClass}" 2>/dev/null done + +# Fix the state for removed snapshotters. +cleanup-images diff --git a/packages/containers.nix b/packages/containers.nix index 8fbb462c19..f3504d7422 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -159,6 +159,9 @@ let cleanup-bm = dockerTools.buildImage { name = "cleanup-bm"; tag = "v0.0.1"; + copyToRoot = with pkgs; [ + cacert + ]; config = { Cmd = [ "${lib.getExe pkgs.scripts.cleanup-bm}" ]; }; diff --git a/packages/scripts.nix b/packages/scripts.nix index a7670027fc..a02ce4608d 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -497,6 +497,7 @@ busybox kubectl dasel + scripts.cleanup-images ]; text = builtins.readFile ./cleanup-bm.sh; }; diff --git a/tools/bm-maintenance/deployment_tdx_snp.yml b/tools/bm-maintenance/deployment_tdx_snp.yml index 9503aee83c..509d8f855e 100644 --- a/tools/bm-maintenance/deployment_tdx_snp.yml +++ b/tools/bm-maintenance/deployment_tdx_snp.yml @@ -62,6 +62,8 @@ spec: mountPath: /var/lib/nydus-snapshotter - name: containerd-config mountPath: /var/lib/rancher/k3s/agent/etc/containerd + - name: containerd-run + mountPath: /run/k3s/containerd/ volumes: - name: opt-edgeless hostPath: @@ -75,4 +77,8 @@ spec: hostPath: path: /var/lib/rancher/k3s/agent/etc/containerd type: Directory + - name: containerd-run + hostPath: + path: /run/k3s/containerd/ + type: Directory restartPolicy: OnFailure From e73c8de7497e6e68495438eca0fd020c02cae1a5 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Thu, 24 Oct 2024 15:35:51 +0200 Subject: [PATCH 5/7] workflows: add workflow to push cleanup image --- .github/workflows/cleanup.yml | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/cleanup.yml diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000000..7ccf03e363 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,39 @@ +name: push cleanup-bm image + +on: + pull_request: + paths: + - .github/workflows/cleaup.yml + - packages/** + +env: + container_registry: ghcr.io/edgelesssys + +jobs: + push: + name: "push cleanup image" + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: ./.github/actions/setup_nix + with: + githubToken: ${{ secrets.GITHUB_TOKEN }} + cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Log in to ghcr.io Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 + - name: Create justfile.env + run: | + cat < justfile.env + container_registry=${{ env.container_registry }} + EOF + - name: Build and push cleanup-bm image + run: | + just push cleanup-bm From 8ad6fef7446c000a0905d42ee563e182a6e94b30 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Fri, 25 Oct 2024 08:50:53 +0200 Subject: [PATCH 6/7] containers: rename cleanup-bm to cleanup-bare-metal For some reason, ghcr.io won't let use upload a package named cleanup-bm, but it let's us upload other packages. Renaming the package fixes the issue. --- .github/workflows/cleanup.yml | 6 +++--- packages/{cleanup-bm.sh => cleanup-bare-metal.sh} | 0 packages/containers.nix | 6 +++--- packages/scripts.nix | 6 +++--- tools/bm-maintenance/deployment_tdx_snp.yml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) rename packages/{cleanup-bm.sh => cleanup-bare-metal.sh} (100%) diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 7ccf03e363..44e3c5add2 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -1,4 +1,4 @@ -name: push cleanup-bm image +name: push cleanup-bare-metal image on: pull_request: @@ -34,6 +34,6 @@ jobs: cat < justfile.env container_registry=${{ env.container_registry }} EOF - - name: Build and push cleanup-bm image + - name: Build and push cleanup-bare-metal image run: | - just push cleanup-bm + just push cleanup-bare-metal diff --git a/packages/cleanup-bm.sh b/packages/cleanup-bare-metal.sh similarity index 100% rename from packages/cleanup-bm.sh rename to packages/cleanup-bare-metal.sh diff --git a/packages/containers.nix b/packages/containers.nix index f3504d7422..80f2d79f46 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -156,14 +156,14 @@ let }; }; - cleanup-bm = dockerTools.buildImage { - name = "cleanup-bm"; + cleanup-bare-metal = dockerTools.buildImage { + name = "cleanup-bare-metal"; tag = "v0.0.1"; copyToRoot = with pkgs; [ cacert ]; config = { - Cmd = [ "${lib.getExe pkgs.scripts.cleanup-bm}" ]; + Cmd = [ "${lib.getExe pkgs.scripts.cleanup-bare-metal}" ]; }; }; }; diff --git a/packages/scripts.nix b/packages/scripts.nix index a02ce4608d..830181a100 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -491,15 +491,15 @@ ''; }; - cleanup-bm = writeShellApplication { - name = "cleanup-bm"; + cleanup-bare-metal = writeShellApplication { + name = "cleanup-bare-metal"; runtimeInputs = with pkgs; [ busybox kubectl dasel scripts.cleanup-images ]; - text = builtins.readFile ./cleanup-bm.sh; + text = builtins.readFile ./cleanup-bare-metal.sh; }; cleanup-images = writeShellApplication { diff --git a/tools/bm-maintenance/deployment_tdx_snp.yml b/tools/bm-maintenance/deployment_tdx_snp.yml index 509d8f855e..03a9535cec 100644 --- a/tools/bm-maintenance/deployment_tdx_snp.yml +++ b/tools/bm-maintenance/deployment_tdx_snp.yml @@ -47,7 +47,7 @@ spec: serviceAccountName: cleanup-sa containers: - name: cleanup - image: ghcr.io/edgelesssys/contrast/cleanup-bm:v0.0.1 + image: ghcr.io/edgelesssys/contrast/cleanup-bare-metal:v0.0.1 env: - name: OPTEDGELESS value: /opt/edgeless From 5d1fbcb7d8d9dd4dec368edc2747127bf420b849 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Mon, 28 Oct 2024 09:26:26 +0100 Subject: [PATCH 7/7] containers: use latest tag for cleanup-bare-metal We don't care about properly versioning the cleanup-bare-metal image, so let's just use latest. Kubernetes will always pull the image if the tag is `latest`. --- packages/containers.nix | 2 +- tools/bm-maintenance/deployment_tdx_snp.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/containers.nix b/packages/containers.nix index 80f2d79f46..6304f1a52e 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -158,7 +158,7 @@ let cleanup-bare-metal = dockerTools.buildImage { name = "cleanup-bare-metal"; - tag = "v0.0.1"; + tag = "latest"; copyToRoot = with pkgs; [ cacert ]; diff --git a/tools/bm-maintenance/deployment_tdx_snp.yml b/tools/bm-maintenance/deployment_tdx_snp.yml index 03a9535cec..f663afc6cc 100644 --- a/tools/bm-maintenance/deployment_tdx_snp.yml +++ b/tools/bm-maintenance/deployment_tdx_snp.yml @@ -47,7 +47,7 @@ spec: serviceAccountName: cleanup-sa containers: - name: cleanup - image: ghcr.io/edgelesssys/contrast/cleanup-bare-metal:v0.0.1 + image: ghcr.io/edgelesssys/contrast/cleanup-bare-metal env: - name: OPTEDGELESS value: /opt/edgeless