From 2238aad648de352d82365cad74592124f619c945 Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Thu, 24 Mar 2022 00:24:55 +0100 Subject: [PATCH 1/3] Add prune option to `bundle` binary Add command line flag `--prune` to delete the image after it was pulled. Add delete logic for DockerHub as well as standard registries. Refactor authentification retrieval code to be more generic. --- cmd/bundle/main.go | 209 ++++++++++++++++++++++++++++++++++++++-- cmd/bundle/main_test.go | 125 ++++++++++++++++++++++-- pkg/bundle/bundle.go | 4 +- 3 files changed, 320 insertions(+), 18 deletions(-) diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 878d01798c..c6d39e3b90 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -5,10 +5,13 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" "io/ioutil" "log" + "net/http" "os" "path/filepath" "strings" @@ -16,6 +19,7 @@ import ( "github.com/docker/cli/cli/config" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/pflag" @@ -25,6 +29,7 @@ import ( type settings struct { help bool image string + prune bool target string secretPath string resultFileImageDigest string @@ -42,6 +47,7 @@ func init() { pflag.StringVar(&flagValues.resultFileImageDigest, "result-file-image-digest", "", "A file to write the image digest") pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains access credentials (optional)") + pflag.BoolVar(&flagValues.prune, "prune", false, "Delete bundle image from registry after it was pulled") } func main() { @@ -75,7 +81,6 @@ func Do(ctx context.Context) error { } log.Printf("Pulling image %q", ref) - img, err := bundle.PullAndUnpack( ref, flagValues.target, @@ -87,13 +92,28 @@ func Do(ctx context.Context) error { log.Printf("Image content was extracted to %s\n", flagValues.target) + digest, err := img.Digest() + if err != nil { + return fmt.Errorf("failed to retrieve digest from bundle image: %w", err) + } + if flagValues.resultFileImageDigest != "" { - digest, err := (*img).Digest() + if err = ioutil.WriteFile(flagValues.resultFileImageDigest, []byte(digest.String()), 0644); err != nil { + return err + } + } + + if flagValues.prune { + // Some container registry implementations, i.e. library/registry:2 will fail to + // delete the image when there is no image digest given. Use image digest from the + // image pulling to construct an image name including tag and digest. + ref, err = name.NewDigest(fmt.Sprintf("%s@%s", ref.Name(), digest.String())) if err != nil { - return fmt.Errorf("retrieving digest of bundle image: %v", err) + return err } - if err = ioutil.WriteFile(flagValues.resultFileImageDigest, []byte(digest.String()), 0644); err != nil { + log.Printf("Deleting image %q", ref) + if err := Prune(ctx, ref, auth); err != nil { return err } } @@ -108,8 +128,20 @@ func resolveAuthBasedOnTarget(ref name.Reference) (authn.Authenticator, error) { return authn.Anonymous, nil } - // Read the registry credentials from the well-known location - file, err := os.Open(filepath.Join(flagValues.secretPath, ".dockerconfigjson")) + // Read the registry credentials from the well-known location if it exists + var mountedSecretDefaultFileName = filepath.Join(flagValues.secretPath, ".dockerconfigjson") + if _, err := os.Stat(mountedSecretDefaultFileName); err == nil { + return ResolveAuthBasedOnTargetUsingConfigFile(ref, mountedSecretDefaultFileName) + } + + // Otherwise, treat secret path as a file (for none Kubernetes setups) + return ResolveAuthBasedOnTargetUsingConfigFile(ref, flagValues.secretPath) +} + +// ResolveAuthBasedOnTargetUsingConfigFile resolves if possible the respective authenticator to be used for +// given image reference (full registry and image name) +func ResolveAuthBasedOnTargetUsingConfigFile(ref name.Reference, dockerConfigFile string) (authn.Authenticator, error) { + file, err := os.Open(dockerConfigFile) if err != nil { return nil, err } @@ -139,9 +171,16 @@ func resolveAuthBasedOnTarget(ref name.Reference) (authn.Authenticator, error) { servers = append(servers, name) } - return nil, fmt.Errorf("failed to find registry credentials for %s, credentials are available for: %s", + var availableConfigs string + if len(servers) > 0 { + availableConfigs = strings.Join(servers, ", ") + } else { + availableConfigs = "none" + } + + return nil, fmt.Errorf("failed to find registry credentials for %s, available configurations: %s", registryName, - strings.Join(servers, ", "), + availableConfigs, ) } @@ -154,3 +193,157 @@ func resolveAuthBasedOnTarget(ref name.Reference) (authn.Authenticator, error) { RegistryToken: authConfig.RegistryToken, }), nil } + +// Prune removes the image from the container registry +// +// Deleting a tag, or a whole repo is not as straightforward as initially +// planned as DockerHub seems to restrict deleting a single tag for +// standard users. This might be subject to change, but as of September +// 2021 it is limited to the business tier. However, there is an API call +// to delete the whole repository. In case there is only one tag used in +// a repository, the effect is pretty much the same. For convenience, there +// is a provider switch to deal with images on DockerHub differently. +// +// DockerHub images: +// - In case the repository only has one tag, the repository is deleted. +// - If there are multiple tags, the tag to be deleted is overwritten +// with an empty image (to remove the content, and save quota). +// - Edge case would be no tags in the repository, which is ignored. +// +// Other registries: +// Use standard spec delete API request to delete the provided tag. +// +func Prune(ctx context.Context, ref name.Reference, auth authn.Authenticator) error { + switch ref.Context().RegistryStr() { + case "index.docker.io": + list, err := remote.List(ref.Context(), remote.WithContext(ctx), remote.WithAuth(auth)) + if err != nil { + return err + } + + switch len(list) { + case 0: + return nil + + case 1: + var authr *authn.AuthConfig + authr, err = auth.Authorization() + if err != nil { + return err + } + + var token string + token, err = dockerHubLogin(authr.Username, authr.Password) + if err != nil { + return err + } + + return dockerHubRepoDelete(token, ref) + + default: + log.Printf("Removing a specific image tag is not supported on %q, the respective image tag will be overwritten with an empty image.\n", ref.Context().RegistryStr()) + + // In case the input argument included a digest, the reference + // needs to be updated to exclude the digest for the empty image + // override to succeed. + switch ref.(type) { + case name.Digest: + ref, err = name.NewTag(ref.Context().Name()) + if err != nil { + return err + } + } + + return remote.Write( + ref, + empty.Image, + remote.WithContext(ctx), + remote.WithAuth(auth), + ) + } + + default: + return remote.Delete( + ref, + remote.WithContext(ctx), + remote.WithAuth(auth), + ) + } +} + +func dockerHubLogin(username string, password string) (string, error) { + type LoginData struct { + Username string `json:"username"` + Password string `json:"password"` + } + + loginData, err := json.Marshal(LoginData{Username: username, Password: password}) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", "https://hub.docker.com/v2/users/login/", bytes.NewReader(loginData)) + if err != nil { + return "", err + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + bodyData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + switch resp.StatusCode { + case http.StatusOK: + type LoginToken struct { + Token string `json:"token"` + } + + var loginToken LoginToken + if err := json.Unmarshal(bodyData, &loginToken); err != nil { + return "", err + } + + return loginToken.Token, nil + + default: + return "", fmt.Errorf(string(bodyData)) + } +} + +func dockerHubRepoDelete(token string, ref name.Reference) error { + req, err := http.NewRequest("DELETE", fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/", ref.Context().RepositoryStr()), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", "JWT "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + switch resp.StatusCode { + case http.StatusAccepted: + return nil + + default: + return fmt.Errorf("failed with HTTP status code %d: %s", resp.StatusCode, string(respData)) + } +} diff --git a/cmd/bundle/main_test.go b/cmd/bundle/main_test.go index 6d7c353d35..7163ff58f4 100644 --- a/cmd/bundle/main_test.go +++ b/cmd/bundle/main_test.go @@ -6,6 +6,7 @@ package main_test import ( "context" + "fmt" "io/ioutil" "log" "os" @@ -19,9 +20,12 @@ import ( "github.com/google/go-containerregistry/pkg/name" containerreg "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "k8s.io/apimachinery/pkg/util/rand" ) var _ = Describe("Bundle Loader", func() { + const exampleImage = "ghcr.io/shipwright-io/sample-go/source-bundle:latest" + var run = func(args ...string) error { // discard log output log.SetOutput(ioutil.Discard) @@ -79,7 +83,7 @@ var _ = Describe("Bundle Loader", func() { Context("validations and error cases", func() { It("should succeed in case the help is requested", func() { - Expect(run("--help")).ToNot(HaveOccurred()) + Expect(run("--help")).To(Succeed()) }) It("should fail in case the image is not specified", func() { @@ -87,17 +91,31 @@ var _ = Describe("Bundle Loader", func() { "--image", "", )).To(HaveOccurred()) }) + + It("should fail in case the provided credentials do not match the required registry", func() { + withTempFile("config.json", func(filename string) { + Expect(ioutil.WriteFile(filename, []byte(`{}`), 0644)).To(BeNil()) + Expect(run( + "--image", "secret.typo.registry.com/foo:bar", + "--secret-path", filename, + )).To(MatchError("failed to find registry credentials for secret.typo.registry.com, available configurations: none")) + + Expect(ioutil.WriteFile(filename, []byte(`{"auths":{"secret.private.registry.com":{"auth":"Zm9vQGJhci5jb206RGlkWW91UmVhbGx5RGVjb2RlVGhpcz8K"}}}`), 0644)).To(BeNil()) + Expect(run( + "--image", "secret.typo.registry.com/foo:bar", + "--secret-path", filename, + )).To(MatchError("failed to find registry credentials for secret.typo.registry.com, available configurations: secret.private.registry.com")) + }) + }) }) Context("Pulling image anonymously", func() { - const exampleImage = "ghcr.io/shipwright-io/sample-go/source-bundle:latest" - It("should pull and unbundle an image from a public registry", func() { withTempDir(func(target string) { Expect(run( "--image", exampleImage, "--target", target, - )).ToNot(HaveOccurred()) + )).To(Succeed()) Expect(filepath.Join(target, "LICENSE")).To(BeAnExistingFile()) }) @@ -109,16 +127,107 @@ var _ = Describe("Bundle Loader", func() { Expect(run( "--image", exampleImage, "--target", target, - "--result-file-image-digest", - filename, - )).ToNot(HaveOccurred()) + "--result-file-image-digest", filename, + )).To(Succeed()) tag, err := name.NewTag(exampleImage) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(filecontent(filename)).To(Equal(getImageDigest(tag).String())) }) }) }) }) + + Context("Pulling image from private location", func() { + var testImage string + var dockerConfigFile string + + var copyImage = func(src, dst name.Reference) { + srcAuth, err := ResolveAuthBasedOnTargetUsingConfigFile(src, dockerConfigFile) + Expect(err).ToNot(HaveOccurred()) + + srcDesc, err := remote.Get(src, remote.WithAuth(srcAuth)) + Expect(err).ToNot(HaveOccurred()) + + srcImage, err := srcDesc.Image() + Expect(err).ToNot(HaveOccurred()) + + dstAuth, err := ResolveAuthBasedOnTargetUsingConfigFile(dst, dockerConfigFile) + Expect(err).ToNot(HaveOccurred()) + + err = remote.Write(dst, srcImage, remote.WithAuth(dstAuth)) + Expect(err).ToNot(HaveOccurred()) + } + + BeforeEach(func() { + registryLocation, ok := os.LookupEnv("TEST_BUNDLE_REGISTRY_TARGET") + if !ok { + Skip("skipping test case with private registry location, because TEST_BUNDLE_REGISTRY_TARGET environment variable is not set, i.e. 'docker.io/some-namespace'") + } + + dockerConfigFile, ok = os.LookupEnv("TEST_BUNDLE_DOCKERCONFIGFILE") + if !ok { + Skip("skipping test case with private registry, because TEST_BUNDLE_DOCKERCONFIGFILE environment variable is not set, i.e. '$HOME/.docker/config.json'") + } + + testImage = fmt.Sprintf("%s/%s:%s", + registryLocation, + rand.String(5), + "source", + ) + + src, err := name.ParseReference(exampleImage) + Expect(err).ToNot(HaveOccurred()) + + dst, err := name.ParseReference(testImage) + Expect(err).ToNot(HaveOccurred()) + + copyImage(src, dst) + }) + + AfterEach(func() { + ref, err := name.ParseReference(testImage) + Expect(err).ToNot(HaveOccurred()) + + auth, err := ResolveAuthBasedOnTargetUsingConfigFile(ref, dockerConfigFile) + Expect(err).ToNot(HaveOccurred()) + + // Delete test image (best effort) + _ = Prune(context.TODO(), ref, auth) + }) + + It("should pull and unpack an image from a private registry", func() { + withTempDir(func(target string) { + Expect(run( + "--image", testImage, + "--target", target, + )).To(Succeed()) + + Expect(filepath.Join(target, "LICENSE")).To(BeAnExistingFile()) + }) + }) + + It("should delete the image after it was pulled", func() { + withTempDir(func(target string) { + Expect(run( + "--image", testImage, + "--prune", + "--secret-path", dockerConfigFile, + "--target", target, + )).To(Succeed()) + + Expect(filepath.Join(target, "LICENSE")).To(BeAnExistingFile()) + + ref, err := name.ParseReference(testImage) + Expect(err).ToNot(HaveOccurred()) + + auth, err := ResolveAuthBasedOnTargetUsingConfigFile(ref, dockerConfigFile) + Expect(err).ToNot(HaveOccurred()) + + _, err = remote.Head(ref, remote.WithAuth(auth)) + Expect(err).To(HaveOccurred()) + }) + }) + }) }) diff --git a/pkg/bundle/bundle.go b/pkg/bundle/bundle.go index 1b59a5817f..c3b3e64918 100644 --- a/pkg/bundle/bundle.go +++ b/pkg/bundle/bundle.go @@ -69,7 +69,7 @@ func PackAndPush(ref name.Reference, directory string, options ...remote.Option) // PullAndUnpack a container image layer content into a local directory. Analog // to the bundle.PackAndPush function, optional remote.Option can be used to // configure settings for the image pull, i.e. access credentials. -func PullAndUnpack(ref name.Reference, targetPath string, options ...remote.Option) (*containerreg.Image, error) { +func PullAndUnpack(ref name.Reference, targetPath string, options ...remote.Option) (containerreg.Image, error) { desc, err := remote.Get(ref, options...) if err != nil { return nil, err @@ -87,7 +87,7 @@ func PullAndUnpack(ref name.Reference, targetPath string, options ...remote.Opti return nil, err } - return &image, nil + return image, nil } // Pack reads a directory and creates a tar stream with its content by: From 2df86f3eeef656d14f8fcc860430540ad26c48eb Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Thu, 24 Mar 2022 00:27:10 +0100 Subject: [PATCH 2/3] Introduce `Prune` to Build spec Add `Prune` field into the Build specification. Update docs to reflect new prune option. Generate CRDs with new changes. Update `Build` prototype setup to have source credentials and prune option. Introduce `--prune` flag in source step setup. Add test case to verify flow end to end. --- deploy/crds/shipwright.io_buildruns.yaml | 7 + deploy/crds/shipwright.io_builds.yaml | 7 + docs/build.md | 11 +- pkg/apis/build/v1alpha1/source.go | 20 +++ .../build/v1alpha1/zz_generated.deepcopy.go | 7 +- .../buildrun/resources/sources/bundle.go | 5 + test/data/registry.yaml | 3 + test/e2e/common_suite_test.go | 16 +++ test/e2e/e2e_bundle_test.go | 122 ++++++++++++++++++ 9 files changed, 193 insertions(+), 5 deletions(-) diff --git a/deploy/crds/shipwright.io_buildruns.yaml b/deploy/crds/shipwright.io_buildruns.yaml index 538c7219bf..af5d60b4fc 100644 --- a/deploy/crds/shipwright.io_buildruns.yaml +++ b/deploy/crds/shipwright.io_buildruns.yaml @@ -650,6 +650,13 @@ spec: image: description: Image reference, i.e. quay.io/org/image:tag type: string + prune: + description: "Prune specifies whether the image is suppose + to be deleted. Allowed values are 'Never' (no deletion) + and `AfterPull` (removal after the image was successfully + pulled from the registry). \n If not defined, it defaults + to 'Never'." + type: string required: - image type: object diff --git a/deploy/crds/shipwright.io_builds.yaml b/deploy/crds/shipwright.io_builds.yaml index 238d96f45b..c33ad8b6d8 100644 --- a/deploy/crds/shipwright.io_builds.yaml +++ b/deploy/crds/shipwright.io_builds.yaml @@ -339,6 +339,13 @@ spec: image: description: Image reference, i.e. quay.io/org/image:tag type: string + prune: + description: "Prune specifies whether the image is suppose + to be deleted. Allowed values are 'Never' (no deletion) + and `AfterPull` (removal after the image was successfully + pulled from the registry). \n If not defined, it defaults + to 'Never'." + type: string required: - image type: object diff --git a/docs/build.md b/docs/build.md index 9f013927eb..433f049e35 100644 --- a/docs/build.md +++ b/docs/build.md @@ -73,7 +73,7 @@ The `Build` definition supports the following fields: - [`apiVersion`](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields) - Specifies the API version, for example `shipwright.io/v1alpha1`. - [`kind`](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields) - Specifies the Kind type, for example `Build`. - [`metadata`](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields) - Metadata that identify the CRD instance, for example the name of the `Build`. - - `spec.source.URL` - Refers to the Git repository containing the source code. + - `spec.source` - Refers to the location of the source code, for example a Git repository or source bundle image. - `spec.strategy` - Refers to the `BuildStrategy` to be used, see the [examples](../samples/buildstrategy) - `spec.builder.image` - Refers to the image containing the build tools to build the source code. (_Use this path for Dockerless strategies, this is just required for `source-to-image` buildStrategy_) - `spec.output`- Refers to the location where the generated image would be pushed. @@ -91,10 +91,13 @@ The `Build` definition supports the following fields: ### Defining the Source -A `Build` resource can specify a Git source, together with other parameters like: +A `Build` resource can specify a Git repository or bundle image source, together with other parameters like: -- `source.credentials.name` - For private repositories, the name is a reference to an existing secret on the same namespace containing the `ssh` data. -- `source.revision` - An specific revision to select from the source repository, this can be a commit, tag or branch name. If not defined, it will fallback to the git repository default branch. +- `source.url` - Specify the source location using a Git repository. +- `source.bundleContainer.image` - Specify a source bundle container image to be used as the source. +- `source.bundleContainer.prune` - Configure whether the source bundle image should be deleted after the source was obtained (defaults to `Never`, other option is `AfterPull` to delete the image after a successful image pull). +- `source.credentials.name` - For private repositories/registries, the name is a reference to an existing secret on the same namespace containing the SSH private key, or Docker access credentials, respectively. +- `source.revision` - An specific revision to select from the source repository, this can be a commit, tag or branch name. If not defined, it will fallback to the Git repository default branch. - `source.contextDir` - For repositories where the source code is not located at the root folder, you can specify this path here. By default, the Build controller won't validate that the Git repository exists. If the validation is desired, users can define the `build.shipwright.io/verify.repository` annotation with `true` explicitly. For example: diff --git a/pkg/apis/build/v1alpha1/source.go b/pkg/apis/build/v1alpha1/source.go index 40d0dd84ab..089a95cc8b 100644 --- a/pkg/apis/build/v1alpha1/source.go +++ b/pkg/apis/build/v1alpha1/source.go @@ -8,10 +8,30 @@ import ( corev1 "k8s.io/api/core/v1" ) +// PruneOption defines the supported options for image pruning +type PruneOption string + +const ( + // Do not delete image after it was pulled + PruneNever PruneOption = "Never" + + // Delete image after it was successfully pulled + PruneAfterPull PruneOption = "AfterPull" +) + // BundleContainer describes the source code bundle container to pull type BundleContainer struct { // Image reference, i.e. quay.io/org/image:tag Image string `json:"image"` + + // Prune specifies whether the image is suppose to be deleted. Allowed + // values are 'Never' (no deletion) and `AfterPull` (removal after the + // image was successfully pulled from the registry). + // + // If not defined, it defaults to 'Never'. + // + // +optional + Prune *PruneOption `json:"prune,omitempty"` } // Source describes the Git source repository to fetch. diff --git a/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go index ffaf7a0a48..d919ef3d25 100644 --- a/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go @@ -513,6 +513,11 @@ func (in *BuildStrategyStatus) DeepCopy() *BuildStrategyStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BundleContainer) DeepCopyInto(out *BundleContainer) { *out = *in + if in.Prune != nil { + in, out := &in.Prune, &out.Prune + *out = new(PruneOption) + **out = **in + } return } @@ -893,7 +898,7 @@ func (in *Source) DeepCopyInto(out *Source) { if in.BundleContainer != nil { in, out := &in.BundleContainer, &out.BundleContainer *out = new(BundleContainer) - **out = **in + (*in).DeepCopyInto(*out) } if in.Revision != nil { in, out := &in.Revision, &out.Revision diff --git a/pkg/reconciler/buildrun/resources/sources/bundle.go b/pkg/reconciler/buildrun/resources/sources/bundle.go index 169b7a0bbe..aeb1464ec6 100644 --- a/pkg/reconciler/buildrun/resources/sources/bundle.go +++ b/pkg/reconciler/buildrun/resources/sources/bundle.go @@ -61,6 +61,11 @@ func AppendBundleStep( ) } + // add prune flag in when prune after pull is configured + if source.BundleContainer.Prune != nil && *source.BundleContainer.Prune == build.PruneAfterPull { + bundleStep.Container.Args = append(bundleStep.Container.Args, "--prune") + } + taskSpec.Steps = append(taskSpec.Steps, bundleStep) } diff --git a/test/data/registry.yaml b/test/data/registry.yaml index 81ec5cb447..a4ad9a23db 100644 --- a/test/data/registry.yaml +++ b/test/data/registry.yaml @@ -24,6 +24,9 @@ spec: - image: registry:2 name: registry imagePullPolicy: IfNotPresent + env: + - name: REGISTRY_STORAGE_DELETE_ENABLED + value: "true" ports: - containerPort: 5000 hostPort: 32222 diff --git a/test/e2e/common_suite_test.go b/test/e2e/common_suite_test.go index eb60fec9b2..d9a19c29ea 100644 --- a/test/e2e/common_suite_test.go +++ b/test/e2e/common_suite_test.go @@ -65,6 +65,14 @@ func (b *buildPrototype) ClusterBuildStrategy(name string) *buildPrototype { return b } +func (b *buildPrototype) SourceCredentials(name string) *buildPrototype { + if name != "" { + b.build.Spec.Source.Credentials = &v1.LocalObjectReference{Name: name} + } + + return b +} + func (b *buildPrototype) SourceGit(repository string) *buildPrototype { b.build.Spec.Source.URL = pointer.String(repository) b.build.Spec.Source.BundleContainer = nil @@ -79,6 +87,14 @@ func (b *buildPrototype) SourceBundle(image string) *buildPrototype { return b } +func (b *buildPrototype) SourceBundlePrune(prune buildv1alpha1.PruneOption) *buildPrototype { + if b.build.Spec.Source.BundleContainer == nil { + b.build.Spec.Source.BundleContainer = &buildv1alpha1.BundleContainer{} + } + b.build.Spec.Source.BundleContainer.Prune = &prune + return b +} + func (b *buildPrototype) SourceContextDir(contextDir string) *buildPrototype { b.build.Spec.Source.ContextDir = pointer.String(contextDir) return b diff --git a/test/e2e/e2e_bundle_test.go b/test/e2e/e2e_bundle_test.go index 35a417ae6b..bd5ef176b5 100644 --- a/test/e2e/e2e_bundle_test.go +++ b/test/e2e/e2e_bundle_test.go @@ -5,6 +5,7 @@ package e2e_test import ( + "bytes" "fmt" "os" "strings" @@ -12,6 +13,12 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/docker/cli/cli/config" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" ) @@ -133,5 +140,120 @@ var _ = Describe("For a Kubernetes cluster with Tekton and build installed", fun validateBuildRunToSucceed(testBuild, buildRun) validateBuildRunResultsFromBundleSource(buildRun) }) + + It("should prune the source image after pulling it", func() { + var secretName = os.Getenv(EnvVarImageRepoSecret) + var registryName string + var auth authn.Authenticator + var tmpImage = fmt.Sprintf("%s/source-%s:%s", + os.Getenv(EnvVarImageRepo), + testID, + "latest", + ) + + By("looking up the registry name", func() { + ref, err := name.ParseReference(outputImage) + Expect(err).ToNot(HaveOccurred()) + + registryName = ref.Context().RegistryStr() + }) + + By("setting up the respective authenticator", func() { + switch { + case secretName != "": + secret, err := testBuild.Clientset. + CoreV1(). + Secrets(testBuild.Namespace). + Get(testBuild.Context, secretName, v1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + dockerConfigJSON, ok := secret.Data[".dockerconfigjson"] + Expect(ok).To(BeTrue()) + + configFile, err := config.LoadFromReader(bytes.NewReader(dockerConfigJSON)) + Expect(err).ToNot(HaveOccurred()) + + authConfig, err := configFile.GetAuthConfig(registryName) + Expect(err).ToNot(HaveOccurred()) + + auth = authn.FromConfig(authn.AuthConfig{ + Username: authConfig.Username, + Password: authConfig.Password, + Auth: authConfig.Auth, + IdentityToken: authConfig.IdentityToken, + RegistryToken: authConfig.RegistryToken, + }) + + default: + auth = authn.Anonymous + } + }) + + By("creating a temporary new input image based on the default input image", func() { + src, err := name.ParseReference(inputImage) + Expect(err).ToNot(HaveOccurred()) + + // Special case for a local registry in the cluster: + // Since the test client is not running in the cluster, it relies on being able to + // reach the same registry via a local port. Therefore, the image name needs to be + // different for the image copy preparation step. + var dstImage = tmpImage + if strings.Contains(dstImage, "cluster.local") { + dstImage = strings.ReplaceAll( + dstImage, + "registry.registry.svc.cluster.local", + "localhost", + ) + } + + dst, err := name.ParseReference(dstImage) + Expect(err).ToNot(HaveOccurred()) + + srcDesc, err := remote.Get(src) + Expect(err).ToNot(HaveOccurred()) + + image, err := srcDesc.Image() + Expect(err).ToNot(HaveOccurred()) + + Expect(remote.Write( + dst, + image, + remote.WithContext(testBuild.Context), + remote.WithAuth(auth), + )).ToNot(HaveOccurred()) + }) + + By("eventually running the actual build with prune option", func() { + build, err = NewBuildPrototype(). + ClusterBuildStrategy("kaniko"). + Name(testID). + Namespace(testBuild.Namespace). + SourceBundle(tmpImage). + SourceBundlePrune(buildv1alpha1.PruneAfterPull). + SourceCredentials(secretName). + SourceContextDir("docker-build"). + Dockerfile("Dockerfile"). + OutputImage(outputImage). + OutputImageCredentials(secretName). + Create() + Expect(err).ToNot(HaveOccurred()) + + buildRun, err = NewBuildRunPrototype(). + Name(testID). + ForBuild(build). + GenerateServiceAccount(). + Create() + Expect(err).ToNot(HaveOccurred()) + validateBuildRunToSucceed(testBuild, buildRun) + }) + + By("checking the temporary input image was removed", func() { + tmp, err := name.ParseReference(tmpImage) + Expect(err).ToNot(HaveOccurred()) + + _, err = remote.Head(tmp, remote.WithContext(testBuild.Context), remote.WithAuth(auth)) + Expect(err).To(HaveOccurred()) + }) + }) }) }) From 7b11e9d0c74f55f5a5c4ba870b6d195783da80cd Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Thu, 24 Mar 2022 10:37:48 +0100 Subject: [PATCH 3/3] Add accept header Add correct accept header to login call. Co-authored-by: Sascha Schwarze --- cmd/bundle/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index c6d39e3b90..b4dcbc948e 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -287,6 +287,7 @@ func dockerHubLogin(username string, password string) (string, error) { return "", err } + req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req)