From 95c7b3789f615be30f81299f62bf25bf03c381a7 Mon Sep 17 00:00:00 2001 From: Xavi Garcia Date: Thu, 21 Nov 2024 10:40:08 +0100 Subject: [PATCH] Enables e2e tests using k3d services export to external ips This PR changes some e2e environment files to be able to run e2e tests in systems in which we cannot access load balancers IPs from clusters. This is specially intented to be able to execute the e2e tests locally in MAC OS systems in which docker IPs cannot be accessed. * Get your IP address (localhost is not valid. See [this](https://github.com/helm/helm/issues/11683#issuecomment-2183304555) * Export your IP with export external_ip=YOUR_IP or edit the dev/env.single-cluster-defaults file and add it. * Run e2e tests normally Note: when running tests with external_ip set ports passed to `k3d` are not random. This in intentional as we need to access the services in a well known port. Signed-off-by: Xavi Garcia tt --- dev/README.md | 2 +- dev/setup-k3d | 2 +- dev/setup-single-cluster | 8 +- e2e/assets/helm/repo/oci-with-auth/fleet.yaml | 2 +- e2e/assets/helm/zot_configmap.yaml | 2 +- e2e/assets/helm/zot_deployment.yaml | 2 +- e2e/assets/helm/zot_service.yaml | 4 +- e2e/single-cluster/imagescan_test.go | 6 +- e2e/single-cluster/oci_registry_test.go | 2 +- e2e/testenv/infra/cmd/setup.go | 85 +++++++------------ go.mod | 2 + go.sum | 4 + 12 files changed, 53 insertions(+), 68 deletions(-) diff --git a/dev/README.md b/dev/README.md index e9bba42228..76f3ba2c5f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -223,7 +223,7 @@ The local infra setup creates pods for: - git server, using nginx with git-http-backend, port 8080/tcp - OCI repo server, using Zot, port 8081/tcp -- Helm registry, using chartmuseum, port 5000/tcp +- Helm registry, using chartmuseum, port 8082/tcp To build and run the infra setup command do: diff --git a/dev/setup-k3d b/dev/setup-k3d index 24aeeed6d3..be429fd454 100755 --- a/dev/setup-k3d +++ b/dev/setup-k3d @@ -20,4 +20,4 @@ EOF args="$args --registry-config $TMP_CONFIG" fi -k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):80@server:0" -p "$(( 8443 + i )):443@server:0" $args +k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):8080@server:0" -p "$(( 8081 + i )):8081@server:0" -p "$(( 8082 + i )):8082@server:0" -p "$(( 8443 + i )):443@server:0" $args diff --git a/dev/setup-single-cluster b/dev/setup-single-cluster index 332964478c..fe31221fa3 100755 --- a/dev/setup-single-cluster +++ b/dev/setup-single-cluster @@ -9,7 +9,13 @@ source ./dev/setup-cluster-config # Cleans with settings sourced, so it should be rather selective. ./dev/k3d-act-clean -./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" $(( RANDOM % 10001 )) +RANDOM_PORT=0 +if [ -z ${external_ip} ]; +then + RANDOM_PORT=$(( RANDOM % 10001 )) +fi + +./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" "${RANDOM_PORT}" ./dev/build-fleet ./dev/import-images-k3d ./dev/setup-fleet "${FLEET_E2E_CLUSTER#k3d-}" '[ diff --git a/e2e/assets/helm/repo/oci-with-auth/fleet.yaml b/e2e/assets/helm/repo/oci-with-auth/fleet.yaml index 87fdf98826..c8a5d362eb 100644 --- a/e2e/assets/helm/repo/oci-with-auth/fleet.yaml +++ b/e2e/assets/helm/repo/oci-with-auth/fleet.yaml @@ -12,7 +12,7 @@ helm: # The directory of the chart in the repo. Any valid go-getter supported # URL can also be used here to specify where to download the chart from. # If repo below is set, this value is the chart name in the repo. - chart: "oci://zot-service.fleet-local.svc.cluster.local:5000/sleeper-chart" + chart: "oci://zot-service.fleet-local.svc.cluster.local:8082/sleeper-chart" # Used if repo is set to look up the version of the chart version: "0.1.0" diff --git a/e2e/assets/helm/zot_configmap.yaml b/e2e/assets/helm/zot_configmap.yaml index 4cab91d01e..15474c2f7d 100644 --- a/e2e/assets/helm/zot_configmap.yaml +++ b/e2e/assets/helm/zot_configmap.yaml @@ -24,7 +24,7 @@ data: } }, "address": "0.0.0.0", - "port": "5000" + "port": "8082" }, "log": { "level": "debug" }, "storage": { diff --git a/e2e/assets/helm/zot_deployment.yaml b/e2e/assets/helm/zot_deployment.yaml index 930359a33a..7fc7407565 100644 --- a/e2e/assets/helm/zot_deployment.yaml +++ b/e2e/assets/helm/zot_deployment.yaml @@ -19,7 +19,7 @@ spec: image: ghcr.io/project-zot/zot-linux-amd64:v1.4.3 imagePullPolicy: IfNotPresent ports: - - containerPort: 5000 + - containerPort: 8082 volumeMounts: - name: config-file mountPath: /etc/zot/config.json diff --git a/e2e/assets/helm/zot_service.yaml b/e2e/assets/helm/zot_service.yaml index 61d19c4718..6c52ffb255 100644 --- a/e2e/assets/helm/zot_service.yaml +++ b/e2e/assets/helm/zot_service.yaml @@ -7,6 +7,6 @@ spec: app: zot ports: - protocol: TCP - port: 5000 - targetPort: 5000 + port: 8082 + targetPort: 8082 type: LoadBalancer diff --git a/e2e/single-cluster/imagescan_test.go b/e2e/single-cluster/imagescan_test.go index 596b7e084e..795c054ab0 100644 --- a/e2e/single-cluster/imagescan_test.go +++ b/e2e/single-cluster/imagescan_test.go @@ -85,7 +85,7 @@ var _ = Describe("Image Scan dynamic tests pushing to ttl.sh", Label("infra-setu k = env.Kubectl.Namespace(env.Namespace) tmpdir := GinkgoT().TempDir() clonedir = path.Join(tmpdir, "clone") - repository = setupRepo(k, tmpdir, clonedir, tmpRepoDir) + repository, gh = setupRepo(k, tmpdir, clonedir, tmpRepoDir) }) AfterEach(func() { @@ -206,7 +206,7 @@ func getGitjobControllerReady(k kubectl.Command) bool { return boolValue } -func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) *git.Repository { +func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) (*git.Repository, *githelper.Git) { // Create git secret out, err := k.Create( "secret", "generic", "git-auth", "--type", "kubernetes.io/basic-auth", @@ -241,7 +241,7 @@ func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) *git.Reposit out, err = k.Apply("-f", gitrepo) Expect(err).ToNot(HaveOccurred(), out) - return repo + return repo, gh } func tagAndPushImage(baseImage, image, tag string) string { diff --git a/e2e/single-cluster/oci_registry_test.go b/e2e/single-cluster/oci_registry_test.go index a0bebe6c97..f16e07419a 100644 --- a/e2e/single-cluster/oci_registry_test.go +++ b/e2e/single-cluster/oci_registry_test.go @@ -132,7 +132,7 @@ var _ = Describe("Single Cluster Deployments using OCI registry", Label("oci-reg externalIP, err := k.Get("service", "zot-service", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}") Expect(err).ToNot(HaveOccurred(), externalIP) Expect(net.ParseIP(externalIP)).ShouldNot(BeNil()) - ociRegistry = fmt.Sprintf("%s:5000", externalIP) + ociRegistry = fmt.Sprintf("%s:8082", externalIP) // store the actual value of the experimental env value // we'll restore in the AfterEach statement if needed diff --git a/e2e/testenv/infra/cmd/setup.go b/e2e/testenv/infra/cmd/setup.go index f86f46c6cb..d3632c71a9 100644 --- a/e2e/testenv/infra/cmd/setup.go +++ b/e2e/testenv/infra/cmd/setup.go @@ -1,7 +1,6 @@ package cmd import ( - "bytes" "context" "crypto/tls" "fmt" @@ -13,6 +12,7 @@ import ( "sync" "time" + "github.com/chartmuseum/helm-push/pkg/chartmuseum" "github.com/rancher/fleet/e2e/testenv" "github.com/rancher/fleet/e2e/testenv/kubectl" "github.com/spf13/cobra" @@ -102,17 +102,9 @@ var setupCmd = &cobra.Command{ go spinUpGitServer(k, &wgGit) } - var chartArchive []byte - var externalIP string + externalIP := os.Getenv("external_ip") if withHelmRegistry || withOCIRegistry { - chartArchive, err = os.ReadFile("sleeper-chart-0.1.0.tgz") - if err != nil { - fail(fmt.Errorf("read packaged Helm chart: %v", err)) - } - - externalIP = os.Getenv("external_ip") - out, err := k.Create( "secret", "tls", "helm-tls", "--cert", path.Join(os.Getenv("CI_OCI_CERTS_DIR"), "helm.crt"), @@ -139,7 +131,17 @@ var setupCmd = &cobra.Command{ wgOCI.Wait() // Login and push a Helm chart to our local OCI registry - OCIClient, err := registry.NewClient() + tlsConf := &tls.Config{ + InsecureSkipVerify: true, + } + OCIClient, err := registry.NewClient( + registry.ClientOptHTTPClient(&http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConf, + Proxy: http.ProxyFromEnvironment, + }, + }), + ) if err != nil { fail(fmt.Errorf("create OCI registry client: %v", err)) } @@ -147,7 +149,8 @@ var setupCmd = &cobra.Command{ if externalIP == "" { externalIP = waitForLoadbalancer(k, "zot-service") } - OCIHost := fmt.Sprintf("%s:5000", externalIP) + + OCIHost := fmt.Sprintf("%s:8082", externalIP) fmt.Printf("logging into OCI registry at %s...\n", OCIHost) _ = eventually(func() (string, error) { @@ -163,30 +166,14 @@ var setupCmd = &cobra.Command{ return "", nil }) - fmt.Println("determining Helm binary path...") - helmPath := os.Getenv("HELM_PATH") - if helmPath == "" { - helmPath = "/usr/bin/helm" // prevents eg. ~/.rd/bin/helm from being used, without support for skipping TLS + // TODO enable this when the Helm library supports `--insecure-skip-tls-verify` + chartArchive, err := os.ReadFile("sleeper-chart-0.1.0.tgz") + if err != nil { + fail(fmt.Errorf("reading helm chart: %v", err)) } - - fmt.Println("pushing Helm chart to registry...") - pushCmd := exec.Command( - helmPath, - "push", - "sleeper-chart-0.1.0.tgz", - fmt.Sprintf("oci://%s", OCIHost), - "--insecure-skip-tls-verify", - ) - if _, err := pushCmd.Output(); err != nil { - fail(fmt.Errorf("push to Helm registry: %v with output %s", err, pushCmd.Stderr)) + if _, err := OCIClient.Push(chartArchive, fmt.Sprintf("%s/sleeper-chart:0.1.0", OCIHost)); err != nil { + fail(fmt.Errorf("push to OCI registry: %v", err)) } - - /* - // TODO enable this when the Helm library supports `--insecure-skip-tls-verify` - if _, err := OCIClient.Push(chartArchive, fmt.Sprintf("%s/sleeper-chart:0.1.0", OCIHost)); err != nil { - fail(fmt.Errorf("push to OCI registry: %v", err)) - } - */ } if withHelmRegistry { @@ -201,30 +188,16 @@ var setupCmd = &cobra.Command{ } _ = eventually(func() (string, error) { - SSLCfg := &tls.Config{ - // works around having to install or reference a CA cert - InsecureSkipVerify: true, // nolint:gosec - } - - client := http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: SSLCfg, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } - - cmAddr := fmt.Sprintf("https://%s:8081/api/charts", externalIP) - - req, err := http.NewRequest(http.MethodPost, cmAddr, bytes.NewReader(chartArchive)) + c, err := chartmuseum.NewClient( + chartmuseum.URL(fmt.Sprintf("https://%s:8081", externalIP)), + chartmuseum.Username(os.Getenv("CI_OCI_USERNAME")), + chartmuseum.Password(os.Getenv("CI_OCI_PASSWORD")), + chartmuseum.InsecureSkipVerify(true), + ) if err != nil { - fail(fmt.Errorf("create POST request to ChartMuseum: %v", err)) + return "", fmt.Errorf("creating chartmuseum client: %v", err) } - - req.SetBasicAuth(os.Getenv("CI_OCI_USERNAME"), os.Getenv("CI_OCI_PASSWORD")) - - resp, err := client.Do(req) + resp, err := c.UploadChartPackage("sleeper-chart-0.1.0.tgz", true) if err != nil { return "", fmt.Errorf("POST request to ChartMuseum failed: %v", err) } diff --git a/go.mod b/go.mod index f42dfb9fae..457c5fa72d 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ replace ( require ( github.com/Masterminds/semver/v3 v3.3.0 github.com/Masterminds/sprig/v3 v3.3.0 + github.com/chartmuseum/helm-push v0.10.4 github.com/cheggaaa/pb v1.0.29 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/docker/docker v27.3.1+incompatible @@ -248,6 +249,7 @@ require ( k8s.io/component-base v0.31.2 // indirect k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7 // indirect k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect + k8s.io/helm v2.17.0+incompatible // indirect oras.land/oras-go v1.2.5 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 182f78c814..cf816607bf 100644 --- a/go.sum +++ b/go.sum @@ -255,6 +255,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/chartmuseum/helm-push v0.10.4 h1:vJTEwhQQ/c+puHSZv+n/3qoHQEAkjjX2b7ODvFVL14s= +github.com/chartmuseum/helm-push v0.10.4/go.mod h1:T+g3wEExKHZADxEU3ZwCRZm5Wa5crecdqJWBfETGTKw= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= @@ -1475,6 +1477,8 @@ k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7 h1:HCbtr1pVu/ElMcTTs18KdMtH5y6f7 k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 h1:cErOOTkQ3JW19o4lo91fFurouhP8NcoBvb7CkvhZZpk= k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao= +k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=