diff --git a/dev/README.md b/dev/README.md index e9bba42228..76f3ba2c5f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -223,7 +223,7 @@ The local infra setup creates pods for: - git server, using nginx with git-http-backend, port 8080/tcp - OCI repo server, using Zot, port 8081/tcp -- Helm registry, using chartmuseum, port 5000/tcp +- Helm registry, using chartmuseum, port 8082/tcp To build and run the infra setup command do: diff --git a/dev/setup-k3d b/dev/setup-k3d index 24aeeed6d3..be429fd454 100755 --- a/dev/setup-k3d +++ b/dev/setup-k3d @@ -20,4 +20,4 @@ EOF args="$args --registry-config $TMP_CONFIG" fi -k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):80@server:0" -p "$(( 8443 + i )):443@server:0" $args +k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):8080@server:0" -p "$(( 8081 + i )):8081@server:0" -p "$(( 8082 + i )):8082@server:0" -p "$(( 8443 + i )):443@server:0" $args diff --git a/dev/setup-single-cluster b/dev/setup-single-cluster index 332964478c..fe31221fa3 100755 --- a/dev/setup-single-cluster +++ b/dev/setup-single-cluster @@ -9,7 +9,13 @@ source ./dev/setup-cluster-config # Cleans with settings sourced, so it should be rather selective. ./dev/k3d-act-clean -./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" $(( RANDOM % 10001 )) +RANDOM_PORT=0 +if [ -z ${external_ip} ]; +then + RANDOM_PORT=$(( RANDOM % 10001 )) +fi + +./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" "${RANDOM_PORT}" ./dev/build-fleet ./dev/import-images-k3d ./dev/setup-fleet "${FLEET_E2E_CLUSTER#k3d-}" '[ diff --git a/e2e/assets/helm/repo/oci-with-auth/fleet.yaml b/e2e/assets/helm/repo/oci-with-auth/fleet.yaml index 87fdf98826..c8a5d362eb 100644 --- a/e2e/assets/helm/repo/oci-with-auth/fleet.yaml +++ b/e2e/assets/helm/repo/oci-with-auth/fleet.yaml @@ -12,7 +12,7 @@ helm: # The directory of the chart in the repo. Any valid go-getter supported # URL can also be used here to specify where to download the chart from. # If repo below is set, this value is the chart name in the repo. - chart: "oci://zot-service.fleet-local.svc.cluster.local:5000/sleeper-chart" + chart: "oci://zot-service.fleet-local.svc.cluster.local:8082/sleeper-chart" # Used if repo is set to look up the version of the chart version: "0.1.0" diff --git a/e2e/assets/helm/zot_configmap.yaml b/e2e/assets/helm/zot_configmap.yaml index 4cab91d01e..15474c2f7d 100644 --- a/e2e/assets/helm/zot_configmap.yaml +++ b/e2e/assets/helm/zot_configmap.yaml @@ -24,7 +24,7 @@ data: } }, "address": "0.0.0.0", - "port": "5000" + "port": "8082" }, "log": { "level": "debug" }, "storage": { diff --git a/e2e/assets/helm/zot_deployment.yaml b/e2e/assets/helm/zot_deployment.yaml index 930359a33a..7fc7407565 100644 --- a/e2e/assets/helm/zot_deployment.yaml +++ b/e2e/assets/helm/zot_deployment.yaml @@ -19,7 +19,7 @@ spec: image: ghcr.io/project-zot/zot-linux-amd64:v1.4.3 imagePullPolicy: IfNotPresent ports: - - containerPort: 5000 + - containerPort: 8082 volumeMounts: - name: config-file mountPath: /etc/zot/config.json diff --git a/e2e/assets/helm/zot_service.yaml b/e2e/assets/helm/zot_service.yaml index 61d19c4718..6c52ffb255 100644 --- a/e2e/assets/helm/zot_service.yaml +++ b/e2e/assets/helm/zot_service.yaml @@ -7,6 +7,6 @@ spec: app: zot ports: - protocol: TCP - port: 5000 - targetPort: 5000 + port: 8082 + targetPort: 8082 type: LoadBalancer diff --git a/e2e/single-cluster/imagescan_test.go b/e2e/single-cluster/imagescan_test.go index 596b7e084e..795c054ab0 100644 --- a/e2e/single-cluster/imagescan_test.go +++ b/e2e/single-cluster/imagescan_test.go @@ -85,7 +85,7 @@ var _ = Describe("Image Scan dynamic tests pushing to ttl.sh", Label("infra-setu k = env.Kubectl.Namespace(env.Namespace) tmpdir := GinkgoT().TempDir() clonedir = path.Join(tmpdir, "clone") - repository = setupRepo(k, tmpdir, clonedir, tmpRepoDir) + repository, gh = setupRepo(k, tmpdir, clonedir, tmpRepoDir) }) AfterEach(func() { @@ -206,7 +206,7 @@ func getGitjobControllerReady(k kubectl.Command) bool { return boolValue } -func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) *git.Repository { +func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) (*git.Repository, *githelper.Git) { // Create git secret out, err := k.Create( "secret", "generic", "git-auth", "--type", "kubernetes.io/basic-auth", @@ -241,7 +241,7 @@ func setupRepo(k kubectl.Command, tmpdir, clonedir, repoDir string) *git.Reposit out, err = k.Apply("-f", gitrepo) Expect(err).ToNot(HaveOccurred(), out) - return repo + return repo, gh } func tagAndPushImage(baseImage, image, tag string) string { diff --git a/e2e/single-cluster/oci_registry_test.go b/e2e/single-cluster/oci_registry_test.go index a0bebe6c97..f16e07419a 100644 --- a/e2e/single-cluster/oci_registry_test.go +++ b/e2e/single-cluster/oci_registry_test.go @@ -132,7 +132,7 @@ var _ = Describe("Single Cluster Deployments using OCI registry", Label("oci-reg externalIP, err := k.Get("service", "zot-service", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}") Expect(err).ToNot(HaveOccurred(), externalIP) Expect(net.ParseIP(externalIP)).ShouldNot(BeNil()) - ociRegistry = fmt.Sprintf("%s:5000", externalIP) + ociRegistry = fmt.Sprintf("%s:8082", externalIP) // store the actual value of the experimental env value // we'll restore in the AfterEach statement if needed diff --git a/e2e/testenv/infra/cmd/setup.go b/e2e/testenv/infra/cmd/setup.go index f86f46c6cb..d3632c71a9 100644 --- a/e2e/testenv/infra/cmd/setup.go +++ b/e2e/testenv/infra/cmd/setup.go @@ -1,7 +1,6 @@ package cmd import ( - "bytes" "context" "crypto/tls" "fmt" @@ -13,6 +12,7 @@ import ( "sync" "time" + "github.com/chartmuseum/helm-push/pkg/chartmuseum" "github.com/rancher/fleet/e2e/testenv" "github.com/rancher/fleet/e2e/testenv/kubectl" "github.com/spf13/cobra" @@ -102,17 +102,9 @@ var setupCmd = &cobra.Command{ go spinUpGitServer(k, &wgGit) } - var chartArchive []byte - var externalIP string + externalIP := os.Getenv("external_ip") if withHelmRegistry || withOCIRegistry { - chartArchive, err = os.ReadFile("sleeper-chart-0.1.0.tgz") - if err != nil { - fail(fmt.Errorf("read packaged Helm chart: %v", err)) - } - - externalIP = os.Getenv("external_ip") - out, err := k.Create( "secret", "tls", "helm-tls", "--cert", path.Join(os.Getenv("CI_OCI_CERTS_DIR"), "helm.crt"), @@ -139,7 +131,17 @@ var setupCmd = &cobra.Command{ wgOCI.Wait() // Login and push a Helm chart to our local OCI registry - OCIClient, err := registry.NewClient() + tlsConf := &tls.Config{ + InsecureSkipVerify: true, + } + OCIClient, err := registry.NewClient( + registry.ClientOptHTTPClient(&http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConf, + Proxy: http.ProxyFromEnvironment, + }, + }), + ) if err != nil { fail(fmt.Errorf("create OCI registry client: %v", err)) } @@ -147,7 +149,8 @@ var setupCmd = &cobra.Command{ if externalIP == "" { externalIP = waitForLoadbalancer(k, "zot-service") } - OCIHost := fmt.Sprintf("%s:5000", externalIP) + + OCIHost := fmt.Sprintf("%s:8082", externalIP) fmt.Printf("logging into OCI registry at %s...\n", OCIHost) _ = eventually(func() (string, error) { @@ -163,30 +166,14 @@ var setupCmd = &cobra.Command{ return "", nil }) - fmt.Println("determining Helm binary path...") - helmPath := os.Getenv("HELM_PATH") - if helmPath == "" { - helmPath = "/usr/bin/helm" // prevents eg. ~/.rd/bin/helm from being used, without support for skipping TLS + // TODO enable this when the Helm library supports `--insecure-skip-tls-verify` + chartArchive, err := os.ReadFile("sleeper-chart-0.1.0.tgz") + if err != nil { + fail(fmt.Errorf("reading helm chart: %v", err)) } - - fmt.Println("pushing Helm chart to registry...") - pushCmd := exec.Command( - helmPath, - "push", - "sleeper-chart-0.1.0.tgz", - fmt.Sprintf("oci://%s", OCIHost), - "--insecure-skip-tls-verify", - ) - if _, err := pushCmd.Output(); err != nil { - fail(fmt.Errorf("push to Helm registry: %v with output %s", err, pushCmd.Stderr)) + if _, err := OCIClient.Push(chartArchive, fmt.Sprintf("%s/sleeper-chart:0.1.0", OCIHost)); err != nil { + fail(fmt.Errorf("push to OCI registry: %v", err)) } - - /* - // TODO enable this when the Helm library supports `--insecure-skip-tls-verify` - if _, err := OCIClient.Push(chartArchive, fmt.Sprintf("%s/sleeper-chart:0.1.0", OCIHost)); err != nil { - fail(fmt.Errorf("push to OCI registry: %v", err)) - } - */ } if withHelmRegistry { @@ -201,30 +188,16 @@ var setupCmd = &cobra.Command{ } _ = eventually(func() (string, error) { - SSLCfg := &tls.Config{ - // works around having to install or reference a CA cert - InsecureSkipVerify: true, // nolint:gosec - } - - client := http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: SSLCfg, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } - - cmAddr := fmt.Sprintf("https://%s:8081/api/charts", externalIP) - - req, err := http.NewRequest(http.MethodPost, cmAddr, bytes.NewReader(chartArchive)) + c, err := chartmuseum.NewClient( + chartmuseum.URL(fmt.Sprintf("https://%s:8081", externalIP)), + chartmuseum.Username(os.Getenv("CI_OCI_USERNAME")), + chartmuseum.Password(os.Getenv("CI_OCI_PASSWORD")), + chartmuseum.InsecureSkipVerify(true), + ) if err != nil { - fail(fmt.Errorf("create POST request to ChartMuseum: %v", err)) + return "", fmt.Errorf("creating chartmuseum client: %v", err) } - - req.SetBasicAuth(os.Getenv("CI_OCI_USERNAME"), os.Getenv("CI_OCI_PASSWORD")) - - resp, err := client.Do(req) + resp, err := c.UploadChartPackage("sleeper-chart-0.1.0.tgz", true) if err != nil { return "", fmt.Errorf("POST request to ChartMuseum failed: %v", err) } diff --git a/go.mod b/go.mod index a481f7cbab..d0829cf542 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ replace ( require ( github.com/Masterminds/semver/v3 v3.3.1 github.com/Masterminds/sprig/v3 v3.3.0 + github.com/chartmuseum/helm-push v0.10.4 github.com/cheggaaa/pb v1.0.29 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/docker/docker v27.3.1+incompatible @@ -248,6 +249,7 @@ require ( k8s.io/component-base v0.31.2 // indirect k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7 // indirect k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect + k8s.io/helm v2.17.0+incompatible // indirect oras.land/oras-go v1.2.5 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index f571f263c2..39229549b1 100644 --- a/go.sum +++ b/go.sum @@ -255,6 +255,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/chartmuseum/helm-push v0.10.4 h1:vJTEwhQQ/c+puHSZv+n/3qoHQEAkjjX2b7ODvFVL14s= +github.com/chartmuseum/helm-push v0.10.4/go.mod h1:T+g3wEExKHZADxEU3ZwCRZm5Wa5crecdqJWBfETGTKw= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= @@ -1475,6 +1477,8 @@ k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7 h1:HCbtr1pVu/ElMcTTs18KdMtH5y6f7 k8s.io/gengo v0.0.0-20240826214909-a7b603a56eb7/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 h1:cErOOTkQ3JW19o4lo91fFurouhP8NcoBvb7CkvhZZpk= k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao= +k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=