chore: de-hardcode list of extra images for image-cache test

Get the image list using `registry.k8s.io/conformance` image instead of hardcoding it. Add new command `talosctl image integration` to create a proper list of k8s integration images for `talosctl images cache-create` command. Signed-off-by: Dmitriy Matrenichev <[email protected]>
siderolabs · Jan 21, 2025 · b9a8ad6 · b9a8ad6
1 parent 683153a
commit b9a8ad6
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 16 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1564,7 +1564,6 @@ jobs:
       - name: image-cache
         env:
           IMAGE_REGISTRY: registry.dev.siderolabs.io
-          MORE_IMAGES: alpine;registry.k8s.io/conformance:v1.32.1;registry.k8s.io/e2e-test-images/busybox:1.36.1-1;registry.k8s.io/e2e-test-images/agnhost:2.53;registry.k8s.io/e2e-test-images/httpd:2.4.38-4;registry.k8s.io/e2e-test-images/nonewprivs:1.3;registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7;registry.k8s.io/e2e-test-images/nautilus:1.7;registry.k8s.io/e2e-test-images/sample-apiserver:1.29.2;registry.k8s.io/e2e-test-images/nginx:1.14-4;registry.k8s.io/etcd:3.5.16-0;registry.k8s.io/e2e-test-images/httpd:2.4.39-4
           PLATFORM: linux/amd64,linux/arm64
           PUSH: "true"
         run: |

diff --git a/.kres.yaml b/.kres.yaml
@@ -1651,7 +1651,6 @@ spec:
             PLATFORM: linux/amd64,linux/arm64
             IMAGE_REGISTRY: registry.dev.siderolabs.io
             PUSH: true
-            MORE_IMAGES: "alpine;registry.k8s.io/conformance:v1.32.1;registry.k8s.io/e2e-test-images/busybox:1.36.1-1;registry.k8s.io/e2e-test-images/agnhost:2.53;registry.k8s.io/e2e-test-images/httpd:2.4.38-4;registry.k8s.io/e2e-test-images/nonewprivs:1.3;registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7;registry.k8s.io/e2e-test-images/nautilus:1.7;registry.k8s.io/e2e-test-images/sample-apiserver:1.29.2;registry.k8s.io/e2e-test-images/nginx:1.14-4;registry.k8s.io/etcd:3.5.16-0;registry.k8s.io/e2e-test-images/httpd:2.4.39-4"
         - name: e2e-image-cache
           command: e2e-qemu
           withSudo: true

diff --git a/Makefile b/Makefile
@@ -150,8 +150,6 @@ IMAGER_EXTRA_PKGS ?= \
 INSTALLER_PKGS ?= $(INSTALLER_ONLY_PKGS) $(IMAGER_EXTRA_PKGS)
 IMAGER_ARGS ?=
 
-MORE_IMAGES ?=
-
 CGO_ENABLED ?= 0
 GO_BUILDFLAGS ?=
 GO_BUILDTAGS ?= tcell_minimal,grpcnotrace
@@ -491,7 +489,9 @@ uki-certs: talosctl ## Generate test certificates for SecureBoot/PCR Signing
 
 .PHONY: cache-create
 cache-create: installer imager ## Generate image cache.
-	@( $(TALOSCTL_EXECUTABLE) images default | grep -v 'siderolabs/installer'; echo "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)"; echo "$(MORE_IMAGES)" | tr ';' '\n' ) | $(TALOSCTL_EXECUTABLE) images cache-create --image-cache-path=/tmp/cache.tar --images=- --force
+	@docker run --entrypoint /usr/local/bin/e2e.test registry.k8s.io/conformance:$(KUBECTL_VERSION) --list-images | \
+		$(TALOSCTL_EXECUTABLE) images integration --installer-tag=$(IMAGE_TAG) --registry-and-user=$(REGISTRY_AND_USERNAME) | \
+		$(TALOSCTL_EXECUTABLE) images cache-create --image-cache-path=/tmp/cache.tar --images=- --force
 	@crane push /tmp/cache.tar $(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG)
 	@$(MAKE) image-iso IMAGER_ARGS="--image-cache=$(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG) --extra-kernel-arg='console=ttyS0'"
 

diff --git a/cmd/talosctl/cmd/talos/image.go b/cmd/talosctl/cmd/talos/image.go
@@ -5,10 +5,13 @@
 package talos
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"strings"
 	"text/tabwriter"
 	"time"
@@ -153,6 +156,89 @@ var imageDefaultCmd = &cobra.Command{
 	},
 }
 
+// imageIntegrationCmd represents the integration image command.
+var imageIntegrationCmd = &cobra.Command{
+	Use:    "integration",
+	Short:  "List the integration images used by k8s in Talos",
+	Args:   cobra.NoArgs,
+	Hidden: true,
+	RunE: func(*cobra.Command, []string) error {
+		if stat, _ := os.Stdin.Stat(); (stat.Mode() & os.ModeCharDevice) != 0 { //nolint:errcheck
+			return errors.New("input must be piped")
+		}
+
+		if imageIntegrationCmdFlags.installerTag == "" {
+			return errors.New("installer tag is required")
+		}
+
+		if imageIntegrationCmdFlags.registryAndUser == "" {
+			return errors.New("registry and user string is required")
+		}
+
+		imgs := images.List(container.NewV1Alpha1(&v1alpha1.Config{
+			MachineConfig: &v1alpha1.MachineConfig{
+				MachineKubelet: &v1alpha1.KubeletConfig{},
+			},
+			ClusterConfig: &v1alpha1.ClusterConfig{
+				EtcdConfig:              &v1alpha1.EtcdConfig{},
+				APIServerConfig:         &v1alpha1.APIServerConfig{},
+				ControllerManagerConfig: &v1alpha1.ControllerManagerConfig{},
+				SchedulerConfig:         &v1alpha1.SchedulerConfig{},
+				CoreDNSConfig:           &v1alpha1.CoreDNS{},
+				ProxyConfig:             &v1alpha1.ProxyConfig{},
+			},
+		}))
+
+		imageNames := []string{
+			imgs.Flannel,
+			imgs.CoreDNS,
+			imgs.Etcd,
+			imgs.KubeAPIServer,
+			imgs.KubeControllerManager,
+			imgs.KubeScheduler,
+			imgs.KubeProxy,
+			imgs.Kubelet,
+			imgs.Pause,
+			"registry.k8s.io/conformance:v" + constants.DefaultKubernetesVersion,
+			"docker.io/library/alpine:latest",
+			imageIntegrationCmdFlags.registryAndUser + "/installer:" +
+				imageIntegrationCmdFlags.installerTag,
+		}
+
+		sc := bufio.NewScanner(os.Stdin)
+
+		for sc.Scan() {
+			switch sc := sc.Text(); {
+			case strings.Contains(sc, "authenticated-"):
+			// skip authenticated images
+			case strings.HasPrefix(sc, "invalid.registry.k8s.io"):
+			// skip invalid images
+			default:
+				imageNames = append(imageNames, sc)
+			}
+		}
+
+		if err := sc.Err(); err != nil {
+			return fmt.Errorf("error reading from stdin: %w", err)
+		}
+
+		slices.Sort(imageNames)
+
+		imageNames = slices.Compact(imageNames)
+
+		for _, img := range imageNames {
+			fmt.Println(img)
+		}
+
+		return nil
+	},
+}
+
+var imageIntegrationCmdFlags struct {
+	installerTag    string
+	registryAndUser string
+}
+
 // imageCacheCreate represents the image cache create command.
 var imageCacheCreateCmd = &cobra.Command{
 	Use:   "cache-create",
@@ -207,7 +293,7 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t
 	},
 }
 
-type imageCacheCreateCmdFlagsType struct {
+var imageCacheCreateCmdFlags struct {
 	imageCachePath      string
 	imageLayerCachePath string
 	platform            string
@@ -218,8 +304,6 @@ type imageCacheCreateCmdFlagsType struct {
 	force    bool
 }
 
-var imageCacheCreateCmdFlags imageCacheCreateCmdFlagsType
-
 func init() {
 	imageCmd.PersistentFlags().StringVar(&imageCmdFlags.namespace, "namespace", "cri", "namespace to use: `system` (etcd and kubelet images) or `cri` for all Kubernetes workloads")
 	addCommand(imageCmd)
@@ -228,6 +312,7 @@ func init() {
 	imageCmd.AddCommand(imageListCmd)
 	imageCmd.AddCommand(imagePullCmd)
 	imageCmd.AddCommand(imageCacheCreateCmd)
+	imageCmd.AddCommand(imageIntegrationCmd)
 
 	imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.imageCachePath, "image-cache-path", "", "directory to save the image cache in OCI format")
 	imageCacheCreateCmd.MarkPersistentFlagRequired("image-cache-path") //nolint:errcheck
@@ -237,4 +322,9 @@ func init() {
 	imageCacheCreateCmd.MarkPersistentFlagRequired("images") //nolint:errcheck
 	imageCacheCreateCmd.PersistentFlags().BoolVar(&imageCacheCreateCmdFlags.insecure, "insecure", false, "allow insecure registries")
 	imageCacheCreateCmd.PersistentFlags().BoolVar(&imageCacheCreateCmdFlags.force, "force", false, "force overwrite of existing image cache")
+
+	imageIntegrationCmd.PersistentFlags().StringVar(&imageIntegrationCmdFlags.installerTag, "installer-tag", "", "tag of the installer image to use")
+	imageIntegrationCmd.MarkPersistentFlagRequired("installer-tag") //nolint:errcheck
+	imageIntegrationCmd.PersistentFlags().StringVar(&imageIntegrationCmdFlags.registryAndUser, "registry-and-user", "", "registry and user to use for the images")
+	imageIntegrationCmd.MarkPersistentFlagRequired("registry-and-user") //nolint:errcheck
 }
diff --git a/hack/test/patches/image-cache.yaml b/hack/test/patches/image-cache.yaml
@@ -23,4 +23,5 @@ name: IMAGECACHE
 provisioning:
   diskSelector:
     match: 'system_disk'
+  maxSize: 3GiB
   grow: true
diff --git a/internal/app/machined/pkg/controllers/cri/image_cache_config.go b/internal/app/machined/pkg/controllers/cri/image_cache_config.go
@@ -366,9 +366,11 @@ func (ctrl *ImageCacheConfigController) analyzeImageCacheVolumes(ctx context.Con
 			case VolumeImageCacheISO:
 				isoReady = true
 				copySource = root.ValueOr("")
+				logger = logger.With(zap.String("iso_size", volumeStatus.TypedSpec().PrettySize))
 			case VolumeImageCacheDISK:
 				diskReady = true
 				copyTarget = root.ValueOr("")
+				logger = logger.With(zap.String("disk_size", volumeStatus.TypedSpec().PrettySize))
 			}
 		}
 
@@ -379,6 +381,8 @@ func (ctrl *ImageCacheConfigController) analyzeImageCacheVolumes(ctx context.Con
 		}
 	}
 
+	logger = logger.With(zap.Bool("all_ready", allReady))
+
 	var copyStatus cri.ImageCacheCopyStatus
 
 	switch {
@@ -462,12 +466,8 @@ func (ctrl *ImageCacheConfigController) copyImageCache(ctx context.Context, logg
 	if err := filepath.WalkDir(source, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return fmt.Errorf("error walking source directory: %w", err)
-		}
-
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
+		} else if err = ctx.Err(); err != nil {
+			return err
 		}
 
 		relPath, err := filepath.Rel(source, path)
@@ -537,7 +537,7 @@ func copyFileSafe(src, dst string) error {
 	defer dstFile.Close() //nolint:errcheck
 
 	if _, err = io.Copy(dstFile, srcFile); err != nil {
-		return fmt.Errorf("error copying file: %w", err)
+		return fmt.Errorf("error copying file: %w, source size is %d", err, srcStat.Size())
 	}
 
 	if err = dstFile.Close(); err != nil {

diff --git a/internal/integration/cli/list.go b/internal/integration/cli/list.go
@@ -51,7 +51,7 @@ func (suite *ListSuite) TestDepth() {
 
 	if stdout, _ := suite.RunCLI(imageCacheQuery); strings.Contains(stdout, "ready") {
 		// Image cache paths parts are longer
-		maxSeps = 8
+		maxSeps = 9
 	}
 
 	// checks that enough separators are encountered in the output