From 9a8cb7d79cc1986ff5636ad4510d4aaf88cb2c7d Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 6 Sep 2023 11:14:23 +0800 Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked containerd Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. 1)image sharing on the host without dm-verity. 2)image sharing on the host with dm-verity. 3)two pods with pulling image only once. 4)image pulling in the guest with nydus-snapshotter. Fixes #5763 Depends-on: github.com/kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 --- .ci/install_nydus_snapshotter.sh | 90 +++++++++++++++++++ Makefile | 2 +- integration/confidential/lib.sh | 81 +++++++++++++++-- .../kubernetes/confidential/agent_image.bats | 6 +- .../confidential/agent_image_encrypted.bats | 9 +- .../confidential/fixtures/pod-config.yaml.in | 2 +- .../image_pulling_with_snapshotter.bats | 90 +++++++++++++++++++ integration/kubernetes/confidential/lib.sh | 3 +- .../confidential/sealed_secret.bats | 9 +- integration/kubernetes/confidential/sev.bats | 3 +- integration/kubernetes/confidential/snp.bats | 1 + .../kubernetes/confidential/tests_common.sh | 30 +++++-- 12 files changed, 294 insertions(+), 32 deletions(-) create mode 100644 .ci/install_nydus_snapshotter.sh create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats diff --git a/.ci/install_nydus_snapshotter.sh b/.ci/install_nydus_snapshotter.sh new file mode 100644 index 000000000..5e2da6c6f --- /dev/null +++ b/.ci/install_nydus_snapshotter.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +cidir=$(dirname "$0") +source "${cidir}/lib.sh" + +target_dir="/opt/confidential-containers" + +nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"} +nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git" +nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"} +nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}" +nydus_snapshotter_binary_target_dir="$target_dir/bin" +nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter" + +nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"} +nydus_version=${nydus_version:-"v2.2.3"} + +arch="$(uname -m)" + +clone_nydus_snapshotter_repo() { + add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}" + + if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then + sudo mkdir -p "${nydus_snapshotter_repo_dir}" + git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true + pushd "${nydus_snapshotter_repo_dir}" + git checkout "${nydus_snapshotter_version}" + popd + fi +} + +build_nydus_snapshotter() { + pushd "${nydus_snapshotter_repo_dir}" + if [ "$arch" = "s390x" ]; then + export GOARCH=$arch + fi + sudo -E PATH=$PATH make + + if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then + sudo mkdir -p $nydus_snapshotter_binary_target_dir + fi + sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc" + sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" + if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then + echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it." + sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs" + fi + sudo rm -rf "$nydus_snapshotter_repo_dir/bin" + popd >/dev/null +} + +download_nydus_snapshotter_config() { + if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then + mkdir -p "$nydus_snapshotter_config_target_dir" + fi + sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml" + sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml" + sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml" + sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml" + +} + +download_nydus_from_tarball() { + if [ "$arch" = "s390x" ]; then + echo "Skip to download nydus for $arch, it doesn't work for $arch now." + return + fi + local goarch="$(${cidir}/kata-arch.sh --golang)" + local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz" + echo "Download tarball from ${tarball_url}" + tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX) + curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1 + sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/" +} + +download_nydus_from_tarball +clone_nydus_snapshotter_repo +build_nydus_snapshotter +download_nydus_snapshotter_config +echo "install nydus-snapshotter successful" diff --git a/Makefile b/Makefile index 48b892219..b2a5c07e8 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ cc-containerd: # Run the Confidential Containers tests for kubernetes. cc-kubernetes: bash -f .ci/install_bats.sh - K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \ + K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \ bash integration/kubernetes/run_kubernetes_tests.sh # Run the Confidential Containers AMD SEV specific tests. diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..2f652f2df 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -180,6 +184,13 @@ disable_full_debug() { sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH" } +restart_containerd(){ + sudo systemctl restart containerd + if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then + die "containerd seems not operational after restartedW" + fi +} + # Configure containerd for confidential containers. Among other things, it ensures # the CRI handler is configured to deal with confidential container. # @@ -198,6 +209,7 @@ configure_cc_containerd() { # installed via operator it will assume containerd is in right state # already. [ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0 + sudo iptables -w -P FORWARD ACCEPT # Even if we are not saving the original file it is a good idea to # restart containerd because it might be in an inconsistent state here. @@ -205,8 +217,7 @@ configure_cc_containerd() { sleep 5 [ -n "$saved_containerd_conf_file" ] && \ sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file" - sudo systemctl start containerd - waitForProcess 30 5 "sudo crictl info >/dev/null" + restart_containerd # Ensure the cc CRI handler is set. local cri_handler=$(sudo crictl info | \ @@ -223,11 +234,6 @@ configure_cc_containerd() { sudo tee -a "$containerd_conf_file" fi - sudo systemctl restart containerd - if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then - die "containerd seems not operational after reconfigured" - fi - sudo iptables -w -P FORWARD ACCEPT } # @@ -445,3 +451,64 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +configure_remote_snapshotter() { + case "${SNAPSHOTTER:-}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} + +is_containerd_support_per_runtime_snapshotter () { + containerd_version=$(containerd --version | awk '{print $3}') + required_version="v1.7.0" + printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V +} + +configure_containerd_for_nydus_snapshotter() { + if [ "${SNAPSHOTTER:-}" = "nydus" ]; then + local containerd_config="$1" + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config" + sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config" + fi +} + +kill_nydus_snapshotter_process() { + echo "Kill nydus snapshotter" + bin="containerd-nydus-grpc" + sudo kill -9 $(pidof $bin) || true + sudo rm -rf "/var/lib/containerd-nydus" || true +} + +remove_test_image() { + local test_image="$1" + crictl rmi "$1" + pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))') + crictl rmi "$pause_name" +} + +restart_nydus_snapshotter() { + kill_nydus_snapshotter_process || true + echo "Restart nydus snapshotter" + sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 & +} + +configure_nydus_snapshotter() { + if [ "${SNAPSHOTTER:-}" = "nydus" ]; then + echo "Configure nydus snapshotter" + if [ "$EXPORT_MODE" == "image_guest_pull" ]; then + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG" + else + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" + sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG" + fi + restart_nydus_snapshotter + fi +} \ No newline at end of file diff --git a/integration/kubernetes/confidential/agent_image.bats b/integration/kubernetes/confidential/agent_image.bats index de5c815e8..f4196affc 100644 --- a/integration/kubernetes/confidential/agent_image.bats +++ b/integration/kubernetes/confidential/agent_image.bats @@ -4,6 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${BATS_TEST_DIRNAME}/tests_common.sh" tag_suffix="" @@ -31,7 +33,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}" test_tag="[cc][agent][kubernetes][containerd]" setup() { - setup_common + setup_containerd + restart_containerd + reconfigure_kata } @test "$test_tag Test can launch pod with measured boot enabled" { diff --git a/integration/kubernetes/confidential/agent_image_encrypted.bats b/integration/kubernetes/confidential/agent_image_encrypted.bats index b8d842ed9..ce1f21459 100644 --- a/integration/kubernetes/confidential/agent_image_encrypted.bats +++ b/integration/kubernetes/confidential/agent_image_encrypted.bats @@ -25,13 +25,8 @@ setup() { SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" - echo "Reconfigure Kata Containers" - switch_image_service_offload on - clear_kernel_params - add_kernel_params "${original_kernel_params}" - - setup_proxy - switch_measured_rootfs_verity_scheme none + reconfigure_kata + restart_containerd } @test "$test_tag Test can pull an encrypted image inside the guest with decryption key" { diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..929a8ad81 --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,90 @@ +#!/usr/bin/env bats +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +tag_suffix="" +if [ "$(uname -m)" != "x86_64" ]; then + tag_suffix="-$(uname -m)" +fi + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +setup() { + remove_test_image "$image_unsigned_protected" || true + setup_containerd + configure_containerd_for_nydus_snapshotter "$containerd_conf_file" + restart_containerd + reconfigure_kata +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod + fi +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once with dm-verity enabled" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config="$(new_pod_config "$image_unsigned_protected" "1")" + echo $pod_config + create_test_pod + pod_config="$(new_pod_config "$image_unsigned_protected" "2")" + echo $pod_config + create_test_pod + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l) + [ ${pull_times} -eq 1 ] + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod + fi +} + +teardown() { + teardown_common + remove_test_image "$image_unsigned_protected" || true + kill_nydus_snapshotter_process +} diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh index 9e9d904cc..d85f0ce00 100755 --- a/integration/kubernetes/confidential/lib.sh +++ b/integration/kubernetes/confidential/lib.sh @@ -46,7 +46,8 @@ kubernetes_create_cc_pod() { fi kubectl apply -f ${config_file} - if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then + pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name') + if ! kubectl get pod "$pod_name" &> /dev/null; then echo "Failed to create the pod" return 1 fi diff --git a/integration/kubernetes/confidential/sealed_secret.bats b/integration/kubernetes/confidential/sealed_secret.bats index 282005547..80d72ce60 100644 --- a/integration/kubernetes/confidential/sealed_secret.bats +++ b/integration/kubernetes/confidential/sealed_secret.bats @@ -22,13 +22,8 @@ setup() { SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" - echo "Reconfigure Kata Containers" - switch_image_service_offload on - clear_kernel_params - add_kernel_params "${original_kernel_params}" - - setup_proxy - switch_measured_rootfs_verity_scheme none + reconfigure_kata + restart_containerd kubectl delete secret sealed-secret --ignore-not-found # Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault diff --git a/integration/kubernetes/confidential/sev.bats b/integration/kubernetes/confidential/sev.bats index 9f515a8bb..f47c4da5a 100644 --- a/integration/kubernetes/confidential/sev.bats +++ b/integration/kubernetes/confidential/sev.bats @@ -55,7 +55,8 @@ setup_file() { # Configure CoCo settings in containerd config local saved_containerd_conf_file="/etc/containerd/config.toml.$$" configure_cc_containerd "${saved_containerd_conf_file}" - + restart_containerd + # KBS setup and run echo "Setting up simple-kbs..." simple_kbs_run diff --git a/integration/kubernetes/confidential/snp.bats b/integration/kubernetes/confidential/snp.bats index 3f03ebda2..530a82110 100644 --- a/integration/kubernetes/confidential/snp.bats +++ b/integration/kubernetes/confidential/snp.bats @@ -38,6 +38,7 @@ setup_file() { # Configure CoCo settings in containerd config local saved_containerd_conf_file="/etc/containerd/config.toml.$$" configure_cc_containerd "${saved_containerd_conf_file}" + restart_containerd # Pull unencrypted image and retrieve ssh keys echo "Pulling unencrypted image and retrieve ssh key..." diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh index 177ed450e..807fb3b7d 100644 --- a/integration/kubernetes/confidential/tests_common.sh +++ b/integration/kubernetes/confidential/tests_common.sh @@ -11,30 +11,41 @@ load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" original_kernel_params=$(get_kernel_params) -# Common setup for tests. +# Common setup for tests. # # Global variables exported: # $test_start_date - test start time. # $pod_config - path to default pod configuration file. -# $original_kernel_params - saved the original list of kernel parameters. # -setup_common() { +setup_common(){ test_start_date=$(date +"%Y-%m-%d %H:%M:%S") pod_config="$(new_pod_config "$image_simple_signed")" pod_id="" kubernetes_delete_all_cc_pods_if_any_exists || true +} + +# Setup containerd for tests. +# +setup_containerd() { + setup_common echo "Prepare containerd for Confidential Container" SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" +} +# Reconfigure Kata for tests. +# +# Global variables exported: +# $original_kernel_params - saved the original list of kernel parameters. +# +reconfigure_kata() { echo "Reconfigure Kata Containers" switch_image_service_offload on clear_kernel_params add_kernel_params "${original_kernel_params}" - setup_proxy switch_measured_rootfs_verity_scheme none } @@ -58,6 +69,13 @@ teardown_common() { add_kernel_params "${original_kernel_params}" switch_image_service_offload off disable_full_debug + # Restore containerd to pre-test state. + if [ -f "$SAVED_CONTAINERD_CONF_FILE" ]; then + systemctl stop containerd || true + sleep 5 + mv -f "$SAVED_CONTAINERD_CONF_FILE" "/etc/containerd/config.toml" + systemctl start containerd || true + fi } @@ -76,7 +94,6 @@ create_test_pod() { enable_full_debug enable_agent_console fi - echo "Create the test sandbox" echo "Pod config is: $pod_config" kubernetes_create_cc_pod $pod_config @@ -97,8 +114,9 @@ create_test_pod() { new_pod_config() { local base_config="${FIXTURES_DIR}/pod-config.yaml.in" local image="$1" + local index="${2:-}" local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") - IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config" + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$index" envsubst < "$base_config" > "$new_config" echo "$new_config" }