chore: add integration tests for image-cache

Provide separate `integration/image-cache` tag. Closes #9860 Signed-off-by: Dmitriy Matrenichev <[email protected]>
siderolabs · Dec 6, 2024 · c4724fc · c4724fc
1 parent 07220fe
commit c4724fc
Show file tree

Hide file tree

Showing 9 changed files with 258 additions and 39 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-12-04T15:25:22Z by kres 232fe63.
+# Generated on 2024-12-06T17:21:12Z by kres 1ebe796.
 
 name: default
 concurrency:
@@ -1606,6 +1606,105 @@ jobs:
           TF_SCRIPT_DIR: _out/contrib
         run: |
           make e2e-cloud-tf
+  integration-image-cache:
+    permissions:
+      actions: read
+      contents: write
+      issues: read
+      packages: write
+      pull-requests: read
+    runs-on:
+      - self-hosted
+      - talos
+    if: contains(fromJSON(needs.default.outputs.labels), 'integration/image-cache')
+    needs:
+      - default
+    steps:
+      - name: gather-system-info
+        id: system-info
+        uses: kenchan0130/[email protected]
+        continue-on-error: true
+      - name: print-system-info
+        run: |
+          MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
+
+          OUTPUTS=(
+            "CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
+            "CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
+            "Hostname: ${{ steps.system-info.outputs.hostname }}"
+            "NodeName: ${NODE_NAME}"
+            "Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
+            "Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
+            "Name: ${{ steps.system-info.outputs.name }}"
+            "Platform: ${{ steps.system-info.outputs.platform }}"
+            "Release: ${{ steps.system-info.outputs.release }}"
+            "Total memory: ${MEMORY_GB} GB"
+          )
+
+          for OUTPUT in "${OUTPUTS[@]}";do
+            echo "${OUTPUT}"
+          done
+        continue-on-error: true
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
+      - name: Download artifacts
+        if: github.event_name != 'schedule'
+        uses: actions/download-artifact@v4
+        with:
+          name: talos-artifacts
+          path: _out
+      - name: Fix artifact permissions
+        if: github.event_name != 'schedule'
+        run: |
+          xargs -a _out/executable-artifacts -I {} chmod +x {}
+      - name: ci-temp-release-tag
+        if: github.event_name != 'schedule'
+        run: |
+          make ci-temp-release-tag
+      - name: uki-certs
+        if: github.event_name == 'schedule'
+        env:
+          PLATFORM: linux/amd64
+        run: |
+          make uki-certs
+      - name: image-cache
+        env:
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          MORE_IMAGES: alpine;registry.k8s.io/conformance:v1.32.0-rc.1;registry.k8s.io/e2e-test-images/busybox:1.36.1-1;registry.k8s.io/e2e-test-images/agnhost:2.53;registry.k8s.io/e2e-test-images/httpd:2.4.38-4;registry.k8s.io/e2e-test-images/nonewprivs:1.3;registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7;registry.k8s.io/e2e-test-images/nautilus:1.7;registry.k8s.io/e2e-test-images/sample-apiserver:1.29.2;registry.k8s.io/e2e-test-images/nginx:1.14-4;registry.k8s.io/etcd:3.5.16-0;registry.k8s.io/e2e-test-images/httpd:2.4.39-4
+          PLATFORM: linux/amd64,linux/arm64
+          PUSH: "true"
+        run: |
+          make cache-create
+      - name: e2e-image-cache
+        env:
+          GITHUB_STEP_NAME: ${{ github.job}}-e2e-image-cache
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          REGISTRY_MIRROR_FLAGS: "no"
+          SHORT_INTEGRATION_TEST: "yes"
+          VIA_MAINTENANCE_MODE: "true"
+          WITH_CONFIG_PATCH: '@hack/test/patches/image-cache.yaml'
+          WITH_ISO: "true"
+        run: |
+          sudo -E make e2e-qemu
+      - name: save artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: talos-logs-integration-image-cache
+          path: |-
+            /tmp/logs-*.tar.gz
+            /tmp/support-*.zip
+          retention-days: "5"
   integration-image-factory:
     permissions:
       actions: read

diff --git a/.kres.yaml b/.kres.yaml
@@ -72,6 +72,7 @@ spec:
     - integration-images
     - integration-reproducibility-test
     - integration-cloud-images
+    - integration-image-cache
     - integration-image-factory
     - integration-aws
     - integration-aws-nvidia-oss
@@ -1514,6 +1515,60 @@ spec:
             PLATFORM: linux/amd64,linux/arm64
             IMAGE_REGISTRY: registry.dev.siderolabs.io
         - name: cloud-images
+    - name: integration-image-cache
+      buildxOptions:
+        enabled: true
+      depends:
+        - default
+      runners:
+        - self-hosted
+        - talos
+      triggerLabels:
+        - integration/image-cache
+      steps:
+        - name: download-artifacts
+          conditions:
+            - not-on-schedule
+          artifactStep:
+            type: download
+            artifactName: talos-artifacts
+            artifactPath: _out
+        - name: ci-temp-release-tag
+          conditions:
+            - not-on-schedule
+        - name: uki-certs
+          conditions:
+            - only-on-schedule
+          environment:
+            PLATFORM: linux/amd64
+        - name: image-cache
+          command: cache-create
+          environment:
+            PLATFORM: linux/amd64,linux/arm64
+            IMAGE_REGISTRY: registry.dev.siderolabs.io
+            PUSH: true
+            MORE_IMAGES: "alpine;registry.k8s.io/conformance:v1.32.0-rc.1;registry.k8s.io/e2e-test-images/busybox:1.36.1-1;registry.k8s.io/e2e-test-images/agnhost:2.53;registry.k8s.io/e2e-test-images/httpd:2.4.38-4;registry.k8s.io/e2e-test-images/nonewprivs:1.3;registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7;registry.k8s.io/e2e-test-images/nautilus:1.7;registry.k8s.io/e2e-test-images/sample-apiserver:1.29.2;registry.k8s.io/e2e-test-images/nginx:1.14-4;registry.k8s.io/etcd:3.5.16-0;registry.k8s.io/e2e-test-images/httpd:2.4.39-4"
+        - name: e2e-image-cache
+          command: e2e-qemu
+          withSudo: true
+          environment:
+            GITHUB_STEP_NAME: ${{ github.job}}-e2e-image-cache
+            IMAGE_REGISTRY: registry.dev.siderolabs.io
+            REGISTRY_MIRROR_FLAGS: no
+            SHORT_INTEGRATION_TEST: yes
+            VIA_MAINTENANCE_MODE: true
+            WITH_CONFIG_PATCH: '@hack/test/patches/image-cache.yaml'
+            WITH_ISO: true
+        - name: save-talos-logs
+          conditions:
+            - always
+          artifactStep:
+            type: upload
+            artifactName: talos-logs-integration-image-cache
+            disableExecutableListGeneration: true
+            artifactPath: /tmp/logs-*.tar.gz
+            additionalArtifacts:
+              - "/tmp/support-*.zip"
     - name: integration-image-factory
       buildxOptions:
         enabled: true

diff --git a/Makefile b/Makefile
@@ -127,6 +127,7 @@ SHORT_INTEGRATION_TEST ?=
 CUSTOM_CNI_URL ?=
 INSTALLER_ARCH ?= all
 IMAGER_ARGS ?=
+MORE_IMAGES ?=
 
 CGO_ENABLED ?= 0
 GO_BUILDFLAGS ?=
@@ -463,6 +464,12 @@ uki-certs: talosctl ## Generate test certificates for SecureBoot/PCR Signing
 	@$(TALOSCTL_EXECUTABLE) gen secureboot pcr
 	@$(TALOSCTL_EXECUTABLE) gen secureboot database
 
+.PHONY: cache-create
+cache-create: installer imager ## Generate image cache.
+	@( $(TALOSCTL_EXECUTABLE) images default | grep -v 'siderolabs/installer'; echo "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)"; echo "$(MORE_IMAGES)" | tr ';' '\n' ) | $(TALOSCTL_EXECUTABLE) images cache-create --image-cache-path=/tmp/cache.tar --images=- --force
+	@crane push /tmp/cache.tar $(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG)
+	@$(MAKE) image-iso IMAGER_ARGS="--image-cache=$(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG) --extra-kernel-arg='console=ttyS0'"
+
 # Code Quality
 
 api-descriptors: ## Generates API descriptors used to detect breaking API changes.

diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh
@@ -131,6 +131,14 @@ case "${WITH_CONFIG_PATCH:-false}" in
     ;;
 esac
 
+case "${WITH_ISO:-false}" in
+  false)
+    ;;
+  *)
+    QEMU_FLAGS+=("--iso-path=${ARTIFACTS}/metal-amd64.iso")
+    ;;
+esac
+
 case "${WITH_CONFIG_PATCH_WORKER:-false}" in
   false)
     ;;

diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh
@@ -142,6 +142,12 @@ function dump_cluster_state {
 }
 
 function build_registry_mirrors {
+  if [[ "${REGISTRY_MIRROR_FLAGS:-yes}" == "no" ]]; then
+    REGISTRY_MIRROR_FLAGS=()
+
+    return
+  fi
+
   if [[ "${CI:-false}" == "true" ]]; then
     REGISTRY_MIRROR_FLAGS=()
 
@@ -151,9 +157,6 @@ function build_registry_mirrors {
 
       REGISTRY_MIRROR_FLAGS+=("--registry-mirror=${registry}=http://${addr}:5000")
     done
-  else
-    # use the value from the environment, if present
-    REGISTRY_MIRROR_FLAGS=("${REGISTRY_MIRROR_FLAGS:-}")
   fi
 }
 

diff --git a/hack/test/patches/image-cache.yaml b/hack/test/patches/image-cache.yaml
@@ -0,0 +1,25 @@
+machine:
+  features:
+    imageCache:
+      localEnabled: true
+  registries:
+    mirrors:
+      "*":
+        skipFallback: true
+        endpoints:
+        - http://172.20.0.251:65000
+      k8s.gcr.io:
+        skipFallback: true
+        endpoints:
+        - http://172.20.0.251:65000
+      registry.k8s.io:
+        skipFallback: true
+        endpoints:
+        - http://172.20.0.251:65000
+---
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: IMAGECACHE
+provisioning:
+  diskSelector:
+    match: 'system_disk'
diff --git a/internal/integration/cli/image.go b/internal/integration/cli/image.go
@@ -47,10 +47,19 @@ func (suite *ImageSuite) TestList() {
 	)
 }
 
+var imageCacheQuery = []string{"get", "imagecacheconfig", "--output", "jsonpath='{.spec.copyStatus}'"}
+
 // TestPull verifies pulling images to the CRI.
 func (suite *ImageSuite) TestPull() {
+	const image = "registry.k8s.io/kube-apiserver:v1.27.0"
+
 	node := suite.RandomDiscoveredNodeInternalIP()
-	image := "registry.k8s.io/kube-apiserver:v1.27.0"
+
+	if stdout, _ := suite.RunCLI(imageCacheQuery); strings.Contains(stdout, "ready") {
+		suite.T().Logf("skipping as the image cache is present")
+
+		return
+	}
 
 	suite.RunCLI([]string{"image", "pull", "--nodes", node, image},
 		base.StdoutEmpty(),

diff --git a/internal/integration/cli/list.go b/internal/integration/cli/list.go
@@ -7,7 +7,6 @@
 package cli
 
 import (
-	"fmt"
 	"os"
 	"regexp"
 	"strings"
@@ -40,10 +39,17 @@ func (suite *ListSuite) TestSuccess() {
 
 // TestDepth tests various combinations of --recurse and --depth flags.
 func (suite *ListSuite) TestDepth() {
-	suite.T().Parallel()
-
 	node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)
 
+	// Expected maximum number of separators in the output
+	// In plain terms, it's the maximum depth of the directory tree
+	maxSeps := 5
+
+	if stdout, _ := suite.RunCLI(imageCacheQuery); strings.Contains(stdout, "ready") {
+		// Image cache paths parts are longer
+		maxSeps = 8
+	}
+
 	// checks that enough separators are encountered in the output
 	runAndCheck := func(t *testing.T, expectedSeparators int, flags ...string) {
 		args := append([]string{"list", "--nodes", node, "/system"}, flags...)
@@ -59,44 +65,44 @@ func (suite *ListSuite) TestDepth() {
 		for _, line := range lines[2:] {
 			actualSeparators := strings.Count(strings.Fields(line)[1], string(os.PathSeparator))
 
-			msg := fmt.Sprintf(
-				"too many separators (actualSeparators = %d, expectedSeparators = %d)\nflags: %s\nlines:\n%s",
-				actualSeparators, expectedSeparators, strings.Join(flags, " "), strings.Join(lines, "\n"),
-			)
-			if !assert.LessOrEqual(t, actualSeparators, expectedSeparators, msg) {
+			if !assert.LessOrEqual(
+				t,
+				actualSeparators,
+				expectedSeparators,
+				"too many separators, flags: %s\nlines:\n%s",
+				strings.Join(flags, " "),
+				stdout,
+			) {
 				return
 			}
 
-			if maxActualSeparators < actualSeparators {
-				maxActualSeparators = actualSeparators
-			}
+			maxActualSeparators = max(maxActualSeparators, actualSeparators)
 		}
 
-		msg := fmt.Sprintf(
-			"not enough separators (maxActualSeparators = %d, expectedSeparators = %d)\nflags: %s\nlines:\n%s",
-			maxActualSeparators, expectedSeparators, strings.Join(flags, " "), strings.Join(lines, "\n"),
+		assert.Equal(
+			t,
+			expectedSeparators,
+			maxActualSeparators,
+			"not enough separators, \nflags: %s\nlines:\n%s",
+			strings.Join(flags, " "),
+			stdout,
 		)
-		assert.Equal(t, maxActualSeparators, expectedSeparators, msg)
 	}
 
 	for _, test := range []struct {
 		separators int
 		flags      []string
 	}{
 		{separators: 0},
-
 		{separators: 0, flags: []string{"--recurse=false"}},
-
 		{separators: 0, flags: []string{"--depth=-1"}},
 		{separators: 0, flags: []string{"--depth=0"}},
 		{separators: 0, flags: []string{"--depth=1"}},
 		{separators: 1, flags: []string{"--depth=2"}},
 		{separators: 2, flags: []string{"--depth=3"}},
-
-		{separators: 5, flags: []string{"--recurse=true"}},
+		{separators: maxSeps, flags: []string{"--recurse=true"}},
 	} {
 		suite.Run(strings.Join(test.flags, ","), func() {
-			suite.T().Parallel()
 			runAndCheck(suite.T(), test.separators, test.flags...)
 		})
 	}
-Original file line number
+Diff line change
@@ Expand Up / @@ -131,6 +131,14 @@ case "${WITH_CONFIG_PATCH:-false}" in @@
         ;;
     esac
+    case "${WITH_ISO:-false}" in
+      false)
+        ;;
+      *)
+        QEMU_FLAGS+=("--iso-path=${ARTIFACTS}/metal-amd64.iso")
+        ;;
+    esac
     case "${WITH_CONFIG_PATCH_WORKER:-false}" in
       false)
         ;;
@@ Expand Down @@