Enables e2e tests using k3d services export to external ips

This PR changes some e2e environment files to be able to run e2e tests
in systems in which we cannot access load balancers IPs from clusters.

This is specially intented to be able to execute the e2e tests locally
in MAC OS systems in which docker IPs cannot be accessed.

* Get your IP address (localhost is not valid. See
  [this](helm/helm#11683 (comment))
* Export your IP with export external_ip=YOUR_IP or edit the
  dev/env.single-cluster-defaults file and add it.
* Run e2e tests normally

Note: when running tests with external_ip set ports passed to `k3d` are
not random. This in intentional as we need to access the services in a
well known port.

Signed-off-by: Xavi Garcia <[email protected]>

tt

dev/README.md

@@ -223,7 +223,7 @@ The local infra setup creates pods for:
 
 - git server, using nginx with git-http-backend, port 8080/tcp
 - OCI repo server, using Zot, port 8081/tcp
-- Helm registry, using chartmuseum, port 5000/tcp
+- Helm registry, using chartmuseum, port 8082/tcp
 
 To build and run the infra setup command do:
 

dev/setup-k3d

@@ -20,4 +20,4 @@ EOF
   args="$args --registry-config $TMP_CONFIG"
 fi
 
-k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):80@server:0" -p "$(( 8443 + i )):443@server:0" $args
+k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):8080@server:0" -p "$(( 8081 + i )):8081@server:0" -p "$(( 8082 + i )):8082@server:0" -p "$(( 8443 + i )):443@server:0" $args

dev/setup-single-cluster

@@ -9,7 +9,13 @@ source ./dev/setup-cluster-config
 # Cleans with settings sourced, so it should be rather selective.
 ./dev/k3d-act-clean
 
-./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" $(( RANDOM % 10001 ))
+RANDOM_PORT=0
+if [ -z ${external_ip} ];
+then
+ RANDOM_PORT=$(( RANDOM % 10001 ))
+fi
+
+./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" "${RANDOM_PORT}"
 ./dev/build-fleet
 ./dev/import-images-k3d
 ./dev/setup-fleet "${FLEET_E2E_CLUSTER#k3d-}" '[

e2e/assets/helm/repo/oci-with-auth/fleet.yaml

@@ -12,7 +12,7 @@ helm:
   # The directory of the chart in the repo.  Any valid go-getter supported
   # URL can also be used here to specify where to download the chart from.
   # If repo below is set, this value is the chart name in the repo.
-  chart: "oci://zot-service.fleet-local.svc.cluster.local:5000/sleeper-chart"
+  chart: "oci://zot-service.fleet-local.svc.cluster.local:8082/sleeper-chart"
 
   # Used if repo is set to look up the version of the chart
   version: "0.1.0"

e2e/assets/helm/zot_configmap.yaml

@@ -24,7 +24,7 @@ data:
                 }
             },
             "address": "0.0.0.0",
-            "port": "5000"
+            "port": "8082"
         },
         "log": { "level": "debug" },
         "storage": {

e2e/assets/helm/zot_deployment.yaml

@@ -19,7 +19,7 @@ spec:
           image: ghcr.io/project-zot/zot-linux-amd64:v1.4.3
           imagePullPolicy: IfNotPresent
           ports:
-          - containerPort: 5000
+          - containerPort: 8082
           volumeMounts:
           - name: config-file
             mountPath: /etc/zot/config.json

e2e/assets/helm/zot_service.yaml

@@ -7,6 +7,6 @@ spec:
     app: zot
   ports:
     - protocol: TCP
-      port: 5000
-      targetPort: 5000
+      port: 8082
+      targetPort: 8082
   type: LoadBalancer

e2e/single-cluster/imagescan_test.go

@@ -85,7 +85,7 @@ var _ = Describe("Image Scan dynamic tests pushing to ttl.sh", Label("infra-setu
 		k = env.Kubectl.Namespace(env.Namespace)
 		tmpdir := GinkgoT().TempDir()
 		clonedir = path.Join(tmpdir, "clone")
-		repository = setupRepo(k, tmpdir, clonedir, tmpRepoDir)
+		repository, gh = setupRepo(k, tmpdir, clonedir, tmpRepoDir)
 	})
 
 	AfterEach(func() {
@@ -206,7 +206,7 @@ func getGitjobControllerReady(k kubectl.Command) bool {
 	return boolValue
 }
 
-func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) *git.Repository {
+func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) (*git.Repository, *githelper.Git) {
 	// Create git secret
 	out, err := k.Create(
 		"secret", "generic", "git-auth", "--type", "kubernetes.io/basic-auth",
@@ -241,7 +241,7 @@ func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) *git.Reposit
 
 	out, err = k.Apply("-f", gitrepo)
 	Expect(err).ToNot(HaveOccurred(), out)
-	return repo
+	return repo, gh
 }
 
 func tagAndPushImage(baseImage, image, tag string) string {

e2e/single-cluster/oci_registry_test.go

@@ -132,7 +132,7 @@ var _ = Describe("Single Cluster Deployments using OCI registry", Label("oci-reg
 		externalIP, err := k.Get("service", "zot-service", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}")
 		Expect(err).ToNot(HaveOccurred(), externalIP)
 		Expect(net.ParseIP(externalIP)).ShouldNot(BeNil())
-		ociRegistry = fmt.Sprintf("%s:5000", externalIP)
+		ociRegistry = fmt.Sprintf("%s:8082", externalIP)
 
 		// store the actual value of the experimental env value
 		// we'll restore in the AfterEach statement if needed

e2e/testenv/infra/cmd/setup.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"crypto/tls"
 	"fmt"
@@ -13,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/chartmuseum/helm-push/pkg/chartmuseum"
 	"github.com/rancher/fleet/e2e/testenv"
 	"github.com/rancher/fleet/e2e/testenv/kubectl"
 	"github.com/spf13/cobra"
@@ -102,17 +102,9 @@ var setupCmd = &cobra.Command{
 			go spinUpGitServer(k, &wgGit)
 		}
 
-		var chartArchive []byte
-		var externalIP string
+		externalIP := os.Getenv("external_ip")
 
 		if withHelmRegistry || withOCIRegistry {
-			chartArchive, err = os.ReadFile("sleeper-chart-0.1.0.tgz")
-			if err != nil {
-				fail(fmt.Errorf("read packaged Helm chart: %v", err))
-			}
-
-			externalIP = os.Getenv("external_ip")
-
 			out, err := k.Create(
 				"secret", "tls", "helm-tls",
 				"--cert", path.Join(os.Getenv("CI_OCI_CERTS_DIR"), "helm.crt"),
@@ -139,15 +131,26 @@ var setupCmd = &cobra.Command{
 			wgOCI.Wait()
 
 			// Login and push a Helm chart to our local OCI registry
-			OCIClient, err := registry.NewClient()
+			tlsConf := &tls.Config{
+				InsecureSkipVerify: true,
+			}
+			OCIClient, err := registry.NewClient(
+				registry.ClientOptHTTPClient(&http.Client{
+					Transport: &http.Transport{
+						TLSClientConfig: tlsConf,
+						Proxy:           http.ProxyFromEnvironment,
+					},
+				}),
+			)
 			if err != nil {
 				fail(fmt.Errorf("create OCI registry client: %v", err))
 			}
 
 			if externalIP == "" {
 				externalIP = waitForLoadbalancer(k, "zot-service")
 			}
-			OCIHost := fmt.Sprintf("%s:5000", externalIP)
+
+			OCIHost := fmt.Sprintf("%s:8082", externalIP)
 
 			fmt.Printf("logging into OCI registry at %s...\n", OCIHost)
 			_ = eventually(func() (string, error) {
@@ -163,30 +166,14 @@ var setupCmd = &cobra.Command{
 				return "", nil
 			})
 
-			fmt.Println("determining Helm binary path...")
-			helmPath := os.Getenv("HELM_PATH")
-			if helmPath == "" {
-				helmPath = "/usr/bin/helm" // prevents eg. ~/.rd/bin/helm from being used, without support for skipping TLS
+			// TODO enable this when the Helm library supports `--insecure-skip-tls-verify`
+			chartArchive, err := os.ReadFile("sleeper-chart-0.1.0.tgz")
+			if err != nil {
+				fail(fmt.Errorf("reading helm chart: %v", err))
 			}
-
-			fmt.Println("pushing Helm chart to registry...")
-			pushCmd := exec.Command(
-				helmPath,
-				"push",
-				"sleeper-chart-0.1.0.tgz",
-				fmt.Sprintf("oci://%s", OCIHost),
-				"--insecure-skip-tls-verify",
-			)
-			if _, err := pushCmd.Output(); err != nil {
-				fail(fmt.Errorf("push to Helm registry: %v with output %s", err, pushCmd.Stderr))
+			if _, err := OCIClient.Push(chartArchive, fmt.Sprintf("%s/sleeper-chart:0.1.0", OCIHost)); err != nil {
+				fail(fmt.Errorf("push to OCI registry: %v", err))
 			}
-
-			/*
-				// TODO enable this when the Helm library supports `--insecure-skip-tls-verify`
-				if _, err := OCIClient.Push(chartArchive, fmt.Sprintf("%s/sleeper-chart:0.1.0", OCIHost)); err != nil {
-					fail(fmt.Errorf("push to OCI registry: %v", err))
-				}
-			*/
 		}
 
 		if withHelmRegistry {
@@ -201,30 +188,16 @@ var setupCmd = &cobra.Command{
 			}
 
 			_ = eventually(func() (string, error) {
-				SSLCfg := &tls.Config{
-					// works around having to install or reference a CA cert
-					InsecureSkipVerify: true, // nolint:gosec
-				}
-
-				client := http.Client{
-					Timeout: 10 * time.Second,
-					Transport: &http.Transport{
-						TLSClientConfig:       SSLCfg,
-						IdleConnTimeout:       10 * time.Second,
-						ExpectContinueTimeout: 1 * time.Second,
-					},
-				}
-
-				cmAddr := fmt.Sprintf("https://%s:8081/api/charts", externalIP)
-
-				req, err := http.NewRequest(http.MethodPost, cmAddr, bytes.NewReader(chartArchive))
+				c, err := chartmuseum.NewClient(
+					chartmuseum.URL(fmt.Sprintf("https://%s:8081", externalIP)),
+					chartmuseum.Username(os.Getenv("CI_OCI_USERNAME")),
+					chartmuseum.Password(os.Getenv("CI_OCI_PASSWORD")),
+					chartmuseum.InsecureSkipVerify(true),
+				)
 				if err != nil {
-					fail(fmt.Errorf("create POST request to ChartMuseum: %v", err))
+					return "", fmt.Errorf("creating chartmuseum client: %v", err)
 				}
-
-				req.SetBasicAuth(os.Getenv("CI_OCI_USERNAME"), os.Getenv("CI_OCI_PASSWORD"))
-
-				resp, err := client.Do(req)
+				resp, err := c.UploadChartPackage("sleeper-chart-0.1.0.tgz", true)
 				if err != nil {
 					return "", fmt.Errorf("POST request to ChartMuseum failed: %v", err)
 				}

go.mod

@@ -12,6 +12,7 @@ replace (
 require (
 	github.com/Masterminds/semver/v3 v3.3.0
 	github.com/Masterminds/sprig/v3 v3.3.0
+	github.com/chartmuseum/helm-push v0.10.4
 	github.com/cheggaaa/pb v1.0.29
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/docker/docker v27.3.1+incompatible
@@ -248,6 +249,7 @@ require (
 	k8s.io/component-base v0.31.2 // indirect
 	k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7 // indirect
 	k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect
+	k8s.io/helm v2.17.0+incompatible // indirect
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

go.sum

@@ -255,6 +255,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/chartmuseum/helm-push v0.10.4 h1:vJTEwhQQ/c+puHSZv+n/3qoHQEAkjjX2b7ODvFVL14s=
+github.com/chartmuseum/helm-push v0.10.4/go.mod h1:T+g3wEExKHZADxEU3ZwCRZm5Wa5crecdqJWBfETGTKw=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
 github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
@@ -1475,6 +1477,8 @@ k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7 h1:HCbtr1pVu/ElMcTTs18KdMtH5y6f7
 k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 h1:cErOOTkQ3JW19o4lo91fFurouhP8NcoBvb7CkvhZZpk=
 k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU=
+k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao=
+k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=