From d15dd7abb61f30cbb462ca47ba3a0bdc853a9550 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 29 Nov 2023 22:59:19 +0000 Subject: [PATCH] feat: support importing sboms along with images When we import an image via the from: directive, also pull in the sbom if there is one. Else we need to rebuild the sbom and there may not be enough state/information to do so. Signed-off-by: Ramkumar Chinchani --- pkg/stacker/base.go | 18 +- pkg/stacker/build.go | 2 + pkg/stacker/publisher.go | 266 +---------------------- pkg/stacker/referrer.go | 452 +++++++++++++++++++++++++++++++++++++++ test/bom.bats | 96 +++++++++ 5 files changed, 571 insertions(+), 263 deletions(-) create mode 100644 pkg/stacker/referrer.go diff --git a/pkg/stacker/base.go b/pkg/stacker/base.go index 2a13ab7a..ef78e2fc 100644 --- a/pkg/stacker/base.go +++ b/pkg/stacker/base.go @@ -46,7 +46,23 @@ func GetBase(o BaseLayerOpts) error { case types.OCILayer: fallthrough case types.DockerLayer: - return importContainersImage(o.Layer.From, o.Config, o.Progress) + err := importContainersImage(o.Layer.From, o.Config, o.Progress) + if o.Layer.Bom != nil && o.Layer.Bom.Generate { + url, err := types.NewDockerishUrl(o.Layer.From.Url) + if err != nil { + return err + } + + layerTypeTag := types.LayerType{}.LayerName(o.Name) + registry := url.Host + repo := url.Path + bomPath := path.Join(o.Config.StackerDir, "artifacts", o.Name, fmt.Sprintf("%s.json", o.Name)) + err = getArtifact(bomPath, "application/spdx+json", registry, repo, layerTypeTag, "", "", true) + if err != nil { + log.Warnf("sbom for image %s not found", o.Layer.From.Url) + } + } + return err default: return errors.Errorf("unknown layer type: %v", o.Layer.From.Type) } diff --git a/pkg/stacker/build.go b/pkg/stacker/build.go index 75764a4b..d53104ed 100644 --- a/pkg/stacker/build.go +++ b/pkg/stacker/build.go @@ -40,6 +40,8 @@ type BuildArgs struct { SetupOnly bool Progress bool AnnotationsNamespace string + Username string + Password string } // Builder is responsible for building the layers based on stackerfiles diff --git a/pkg/stacker/publisher.go b/pkg/stacker/publisher.go index a837c485..5d57142c 100644 --- a/pkg/stacker/publisher.go +++ b/pkg/stacker/publisher.go @@ -1,23 +1,14 @@ package stacker import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" "fmt" "io" - "net/http" - "net/url" "os" "path" "path/filepath" "regexp" "strings" - godigest "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" - ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/umoci" "github.com/opencontainers/umoci/oci/casext" "github.com/pkg/errors" @@ -75,255 +66,6 @@ func NewPublisher(opts *PublishArgs) *Publisher { } } -func basicAuth(username, password string) string { - auth := username + ":" + password - return base64.StdEncoding.EncodeToString([]byte(auth)) -} - -func clientRequest(method, url, username, password string, headers map[string]string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequestWithContext(context.TODO(), method, url, body) - if err != nil { - log.Errorf("unable to create http request err:%s", err) - return nil, err - } - - // FIXME: handle bearer auth also - if username != "" && password != "" { - req.Header.Add("Authorization", "Basic "+basicAuth(username, password)) - } - - if len(headers) > 0 { - for k, v := range headers { - req.Header.Add(k, v) - } - } - - res, err := http.DefaultClient.Do(req) - if err != nil { - log.Errorf("http request failed url:%s", url) - return nil, err - } - - return res, nil -} - -func fileDigest(path string) (*godigest.Digest, error) { - fh, err := os.Open(path) - if err != nil { - log.Errorf("unable to open file:%s, err:%s", path, err) - return nil, err - } - defer fh.Close() - - dgst, err := godigest.FromReader(fh) - if err != nil { - log.Errorf("unable get digest for file:%s, err:%s", path, err) - return nil, err - } - - return &dgst, nil -} - -// publishArtifact to a registry/repo for this subject -func (p *Publisher) publishArtifact(path, mtype, registry, repo, subjectTag string, skipTLS bool) error { - username := p.opts.Username - password := p.opts.Password - - subject := distspecURL(registry, repo, subjectTag, skipTLS) - - // check subject exists - res, err := clientRequest(http.MethodHead, subject, username, password, nil, nil) - if err != nil { - log.Errorf("unable to check subject:%s, err:%s", subject, err) - return err - } - if res == nil || res.StatusCode != http.StatusOK { - log.Errorf("subject:%s doesn't exist, ignoring and proceeding", subject) - } - - slen := res.ContentLength - smtype := res.Header.Get("Content-Type") - sdgst, err := godigest.Parse(res.Header.Get("Docker-Content-Digest")) - if slen < 0 || smtype == "" || sdgst == "" || err != nil { - log.Errorf("unable to get descriptor details for subject:%s", subject) - return errors.Errorf("unable to get descriptor details for subject:%s", subject) - } - - // upload the artifact - finfo, err := os.Lstat(path) - if err != nil { - log.Errorf("unable to stat file:%s, err:%s", path, err) - return err - } - - dgst, err := fileDigest(path) - if err != nil { - log.Errorf("unable get digest for file:%s, err:%s", path, err) - return err - } - - fh, err := os.Open(path) - if err != nil { - log.Errorf("unable to open file:%s, err:%s", path, err) - return err - } - defer fh.Close() - - if err := uploadBlob(registry, repo, path, username, password, fh, finfo.Size(), dgst, skipTLS); err != nil { - log.Errorf("unable to upload file:%s, err:%s", path, err) - return err - } - - // check and upload emptyJSON blob - erdr := bytes.NewReader(ispec.DescriptorEmptyJSON.Data) - edgst := ispec.DescriptorEmptyJSON.Digest - - if err := uploadBlob(registry, repo, path, username, password, erdr, ispec.DescriptorEmptyJSON.Size, &edgst, skipTLS); err != nil { - log.Errorf("unable to upload file:%s, err:%s", path, err) - return err - } - - // upload the reference manifest - manifest := ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - MediaType: ispec.MediaTypeImageManifest, - ArtifactType: mtype, - Config: ispec.DescriptorEmptyJSON, - Subject: &ispec.Descriptor{ - MediaType: ispec.MediaTypeImageManifest, - Size: slen, - Digest: sdgst, - }, - Layers: []ispec.Descriptor{ - ispec.Descriptor{ - MediaType: mtype, - Size: finfo.Size(), - Digest: *dgst, - }, - }, - } - - //content, err := json.MarshalIndent(&manifest, "", "\t") - content, err := json.Marshal(&manifest) - if err != nil { - log.Errorf("unable to marshal image manifest, err:%s", err) - return err - } - - // artifact manifest - var regUrl string - mdgst := godigest.FromBytes(content) - if skipTLS { - regUrl = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String()) - } else { - regUrl = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String()) - } - hdrs := map[string]string{ - "Content-Type": ispec.MediaTypeImageManifest, - "Content-Length": fmt.Sprintf("%d", len(content)), - } - res, err = clientRequest(http.MethodPut, regUrl, username, password, hdrs, bytes.NewBuffer(content)) - if err != nil { - log.Errorf("unable to check subject:%s, err:%s", subject, err) - return err - } - if res == nil || res.StatusCode != http.StatusCreated { - log.Errorf("unable to upload manifest, url:%s", regUrl) - return errors.Errorf("unable to upload manifest, url:%s", regUrl) - } - - log.Infof("Copying artifact '%s' done", path) - - return nil -} - -func distspecURL(registry, repo, tag string, skipTLS bool) string { - var url string - - if skipTLS { - url = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag) - } else { - url = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag) - } - - return url -} - -func uploadBlob(registry, repo, path, username, password string, reader io.Reader, size int64, dgst *godigest.Digest, skipTLS bool) error { - // upload with POST, PUT sequence - var regUrl string - if skipTLS { - regUrl = fmt.Sprintf("http://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String()) - } else { - regUrl = fmt.Sprintf("https://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String()) - } - - subject := distspecURL(registry, repo, "", skipTLS) - - log.Debugf("check blob before upload (HEAD): %s", regUrl) - res, err := clientRequest(http.MethodHead, regUrl, username, password, nil, nil) - if err != nil { - log.Errorf("unable to check blob:%s, err:%s", subject, err) - return err - } - log.Debugf("http response headers: +%v status:%v", res.Header, res.Status) - hdr := res.Header.Get("Docker-Content-Digest") - if hdr != "" { - log.Infof("Copying blob %s skipped: already exists", dgst.Hex()[:12]) - return nil - } - - if skipTLS { - regUrl = fmt.Sprintf("http://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0]) - } else { - regUrl = fmt.Sprintf("https://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0]) - } - - log.Debugf("new blob upload (POST): %s", regUrl) - res, err = clientRequest(http.MethodPost, regUrl, username, password, nil, nil) - if err != nil { - log.Errorf("post unable to check subject:%s, err:%s", subject, err) - return err - } - log.Debugf("http response headers: +%v status:%v", res.Header, res.Status) - loc, err := res.Location() - if err != nil { - log.Errorf("unable get upload location url:%s, err:%s", regUrl, err) - return err - } - - log.Debugf("finish blob upload (PUT): %s", regUrl) - req, err := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc.String(), reader) - if err != nil { - log.Errorf("unable to create a http request url:%s", subject) - return err - } - if username != "" && password != "" { - req.Header.Add("Authorization", "Basic "+basicAuth(username, password)) - } - req.URL.RawQuery = url.Values{ - "digest": {dgst.String()}, - }.Encode() - - req.ContentLength = size - - res, err = http.DefaultClient.Do(req) - if err != nil { - log.Errorf("http request failed url:%s", subject) - return err - } - if res == nil || res.StatusCode != http.StatusCreated { - log.Errorf("unable to upload artifact:%s to url:%s", path, regUrl) - return errors.Errorf("unable to upload artifact:%s to url:%s", path, regUrl) - } - - log.Infof("Copying blob %s done", dgst.Hex()[:12]) - - return nil -} - // Publish layers in a single stackerfile func (p *Publisher) Publish(file string) error { opts := p.opts @@ -452,14 +194,14 @@ func (p *Publisher) Publish(file string) error { repo := url.Path // publish sbom - if err := p.publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, fmt.Sprintf("%s.json", layerName)), - "application/spdx+json", registry, repo, layerTypeTag, opts.SkipTLS); err != nil { + if err := publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, fmt.Sprintf("%s.json", layerName)), + "application/spdx+json", registry, repo, layerTypeTag, p.opts.Username, p.opts.Password, opts.SkipTLS); err != nil { return err } // publish inventory - if err := p.publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, "inventory.json"), - "application/vnd.stackerbuild.inventory+json", registry, repo, layerTypeTag, opts.SkipTLS); err != nil { + if err := publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, "inventory.json"), + "application/vnd.stackerbuild.inventory+json", registry, repo, layerTypeTag, p.opts.Username, p.opts.Password, opts.SkipTLS); err != nil { return err } diff --git a/pkg/stacker/referrer.go b/pkg/stacker/referrer.go new file mode 100644 index 00000000..38196519 --- /dev/null +++ b/pkg/stacker/referrer.go @@ -0,0 +1,452 @@ +package stacker + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + + godigest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "stackerbuild.io/stacker/pkg/log" +) + +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func clientRequest(method, url, username, password string, headers map[string]string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(context.TODO(), method, url, body) + if err != nil { + log.Errorf("unable to create http request err:%s", err) + return nil, err + } + + // FIXME: handle bearer auth also + if username != "" && password != "" { + req.Header.Add("Authorization", "Basic "+basicAuth(username, password)) + } + + if len(headers) > 0 { + for k, v := range headers { + req.Header.Add(k, v) + } + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + log.Errorf("http request failed url:%s", url) + return nil, err + } + + return res, nil +} + +func fileDigest(path string) (*godigest.Digest, error) { + fh, err := os.Open(path) + if err != nil { + log.Errorf("unable to open file:%s, err:%s", path, err) + return nil, err + } + defer fh.Close() + + dgst, err := godigest.FromReader(fh) + if err != nil { + log.Errorf("unable get digest for file:%s, err:%s", path, err) + return nil, err + } + + return &dgst, nil +} + +// publishArtifact to a registry/repo for this subject +func publishArtifact(path, mtype, registry, repo, subjectTag, username, password string, skipTLS bool) error { + + subject := distspecURL(registry, repo, subjectTag, skipTLS) + + // check subject exists + res, err := clientRequest(http.MethodHead, subject, username, password, nil, nil) + if err != nil { + log.Errorf("unable to check subject:%s, err:%s", subject, err) + return err + } + if res == nil || res.StatusCode != http.StatusOK { + log.Errorf("subject:%s doesn't exist, ignoring and proceeding", subject) + } + + slen := res.ContentLength + smtype := res.Header.Get("Content-Type") + sdgst, err := godigest.Parse(res.Header.Get("Docker-Content-Digest")) + if slen < 0 || smtype == "" || sdgst == "" || err != nil { + log.Errorf("unable to get descriptor details for subject:%s", subject) + return errors.Errorf("unable to get descriptor details for subject:%s", subject) + } + + // upload the artifact + finfo, err := os.Lstat(path) + if err != nil { + log.Errorf("unable to stat file:%s, err:%s", path, err) + return err + } + + dgst, err := fileDigest(path) + if err != nil { + log.Errorf("unable get digest for file:%s, err:%s", path, err) + return err + } + + fh, err := os.Open(path) + if err != nil { + log.Errorf("unable to open file:%s, err:%s", path, err) + return err + } + defer fh.Close() + + if err := uploadBlob(registry, repo, path, username, password, fh, finfo.Size(), dgst, skipTLS); err != nil { + log.Errorf("unable to upload file:%s, err:%s", path, err) + return err + } + + // check and upload emptyJSON blob + erdr := bytes.NewReader(ispec.DescriptorEmptyJSON.Data) + edgst := ispec.DescriptorEmptyJSON.Digest + + if err := uploadBlob(registry, repo, path, username, password, erdr, ispec.DescriptorEmptyJSON.Size, &edgst, skipTLS); err != nil { + log.Errorf("unable to upload file:%s, err:%s", path, err) + return err + } + + // upload the reference manifest + manifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: mtype, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Size: slen, + Digest: sdgst, + }, + Layers: []ispec.Descriptor{ + ispec.Descriptor{ + MediaType: mtype, + Size: finfo.Size(), + Digest: *dgst, + }, + }, + } + + //content, err := json.MarshalIndent(&manifest, "", "\t") + content, err := json.Marshal(&manifest) + if err != nil { + log.Errorf("unable to marshal image manifest, err:%s", err) + return err + } + + // artifact manifest + var regUrl string + mdgst := godigest.FromBytes(content) + if skipTLS { + regUrl = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String()) + } else { + regUrl = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String()) + } + hdrs := map[string]string{ + "Content-Type": ispec.MediaTypeImageManifest, + "Content-Length": fmt.Sprintf("%d", len(content)), + } + res, err = clientRequest(http.MethodPut, regUrl, username, password, hdrs, bytes.NewBuffer(content)) + if err != nil { + log.Errorf("unable to check subject:%s, err:%s", subject, err) + return err + } + if res == nil || res.StatusCode != http.StatusCreated { + log.Errorf("unable to upload manifest, url:%s", regUrl) + return errors.Errorf("unable to upload manifest, url:%s", regUrl) + } + + log.Infof("Copying artifact '%s' done", path) + + return nil +} + +func distspecURL(registry, repo, tag string, skipTLS bool) string { + var url string + + if skipTLS { + url = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag) + } else { + url = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag) + } + + return url +} + +func uploadBlob(registry, repo, path, username, password string, reader io.Reader, size int64, dgst *godigest.Digest, skipTLS bool) error { + // upload with POST, PUT sequence + var regUrl string + if skipTLS { + regUrl = fmt.Sprintf("http://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String()) + } else { + regUrl = fmt.Sprintf("https://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String()) + } + + subject := distspecURL(registry, repo, "", skipTLS) + + log.Debugf("check blob before upload (HEAD): %s", regUrl) + res, err := clientRequest(http.MethodHead, regUrl, username, password, nil, nil) + if err != nil { + log.Errorf("unable to check blob:%s, err:%s", subject, err) + return err + } + log.Debugf("http response headers: +%v status:%v", res.Header, res.Status) + hdr := res.Header.Get("Docker-Content-Digest") + if hdr != "" { + log.Infof("Copying blob %s skipped: already exists", dgst.Hex()[:12]) + return nil + } + + if skipTLS { + regUrl = fmt.Sprintf("http://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0]) + } else { + regUrl = fmt.Sprintf("https://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0]) + } + + log.Debugf("new blob upload (POST): %s", regUrl) + res, err = clientRequest(http.MethodPost, regUrl, username, password, nil, nil) + if err != nil { + log.Errorf("post unable to check subject:%s, err:%s", subject, err) + return err + } + log.Debugf("http response headers: +%v status:%v", res.Header, res.Status) + loc, err := res.Location() + if err != nil { + log.Errorf("unable get upload location url:%s, err:%s", regUrl, err) + return err + } + + log.Debugf("finish blob upload (PUT): %s", regUrl) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc.String(), reader) + if err != nil { + log.Errorf("unable to create a http request url:%s", subject) + return err + } + if username != "" && password != "" { + req.Header.Add("Authorization", "Basic "+basicAuth(username, password)) + } + req.URL.RawQuery = url.Values{ + "digest": {dgst.String()}, + }.Encode() + + req.ContentLength = size + + res, err = http.DefaultClient.Do(req) + if err != nil { + log.Errorf("http request failed url:%s", subject) + return err + } + if res == nil || res.StatusCode != http.StatusCreated { + log.Errorf("unable to upload artifact:%s to url:%s", path, regUrl) + return errors.Errorf("unable to upload artifact:%s to url:%s", path, regUrl) + } + + log.Infof("Copying blob %s done", dgst.Hex()[:12]) + + return nil +} + +// getArtifact to a registry/repo for this subject +func getArtifact(path, mtype, registry, repo, subjectTag, username, password string, skipTLS bool) error { + subject := distspecURL(registry, repo, subjectTag, skipTLS) + + // check subject exists + res, err := clientRequest(http.MethodHead, subject, username, password, nil, nil) + if err != nil { + log.Errorf("unable to check subject:%s, err:%s", subject, err) + return err + } + if res == nil || res.StatusCode != http.StatusOK { + log.Errorf("subject:%s doesn't exist, ignoring and proceeding", subject) + } + + slen := res.ContentLength + smtype := res.Header.Get("Content-Type") + sdgst, err := godigest.Parse(res.Header.Get("Docker-Content-Digest")) + if slen < 0 || smtype == "" || sdgst == "" || err != nil { + log.Errorf("unable to get descriptor details for subject:%s", subject) + return errors.Errorf("unable to get descriptor details for subject:%s", subject) + } + + // upload the artifact + finfo, err := os.Lstat(path) + if err != nil { + log.Errorf("unable to stat file:%s, err:%s", path, err) + return err + } + + dgst, err := fileDigest(path) + if err != nil { + log.Errorf("unable get digest for file:%s, err:%s", path, err) + return err + } + + fh, err := os.Open(path) + if err != nil { + log.Errorf("unable to open file:%s, err:%s", path, err) + return err + } + defer fh.Close() + + if err := downloadBlob(registry, repo, path, username, password, fh, finfo.Size(), dgst, skipTLS); err != nil { + log.Errorf("unable to upload file:%s, err:%s", path, err) + return err + } + + // check and upload emptyJSON blob + erdr := bytes.NewReader(ispec.DescriptorEmptyJSON.Data) + edgst := ispec.DescriptorEmptyJSON.Digest + + if err := downloadBlob(registry, repo, path, username, password, erdr, ispec.DescriptorEmptyJSON.Size, &edgst, skipTLS); err != nil { + log.Errorf("unable to upload file:%s, err:%s", path, err) + return err + } + + // upload the reference manifest + manifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: mtype, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Size: slen, + Digest: sdgst, + }, + Layers: []ispec.Descriptor{ + ispec.Descriptor{ + MediaType: mtype, + Size: finfo.Size(), + Digest: *dgst, + }, + }, + } + + //content, err := json.MarshalIndent(&manifest, "", "\t") + content, err := json.Marshal(&manifest) + if err != nil { + log.Errorf("unable to marshal image manifest, err:%s", err) + return err + } + + // artifact manifest + var regUrl string + mdgst := godigest.FromBytes(content) + if skipTLS { + regUrl = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String()) + } else { + regUrl = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String()) + } + hdrs := map[string]string{ + "Content-Type": ispec.MediaTypeImageManifest, + "Content-Length": fmt.Sprintf("%d", len(content)), + } + res, err = clientRequest(http.MethodPut, regUrl, username, password, hdrs, bytes.NewBuffer(content)) + if err != nil { + log.Errorf("unable to check subject:%s, err:%s", subject, err) + return err + } + if res == nil || res.StatusCode != http.StatusCreated { + log.Errorf("unable to upload manifest, url:%s", regUrl) + return errors.Errorf("unable to upload manifest, url:%s", regUrl) + } + + log.Infof("Copying artifact '%s' done", path) + + return nil +} + +func downloadBlob(registry, repo, path, username, password string, reader io.Reader, size int64, dgst *godigest.Digest, skipTLS bool) error { + // upload with POST, PUT sequence + var regUrl string + if skipTLS { + regUrl = fmt.Sprintf("http://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String()) + } else { + regUrl = fmt.Sprintf("https://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String()) + } + + subject := distspecURL(registry, repo, "", skipTLS) + + log.Debugf("check blob before upload (HEAD): %s", regUrl) + res, err := clientRequest(http.MethodHead, regUrl, username, password, nil, nil) + if err != nil { + log.Errorf("unable to check blob:%s, err:%s", subject, err) + return err + } + log.Debugf("http response headers: +%v status:%v", res.Header, res.Status) + hdr := res.Header.Get("Docker-Content-Digest") + if hdr != "" { + log.Infof("Copying blob %s skipped: already exists", dgst.Hex()[:12]) + return nil + } + + if skipTLS { + regUrl = fmt.Sprintf("http://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0]) + } else { + regUrl = fmt.Sprintf("https://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0]) + } + + log.Debugf("new blob upload (POST): %s", regUrl) + res, err = clientRequest(http.MethodPost, regUrl, username, password, nil, nil) + if err != nil { + log.Errorf("post unable to check subject:%s, err:%s", subject, err) + return err + } + log.Debugf("http response headers: +%v status:%v", res.Header, res.Status) + loc, err := res.Location() + if err != nil { + log.Errorf("unable get upload location url:%s, err:%s", regUrl, err) + return err + } + + log.Debugf("finish blob upload (PUT): %s", regUrl) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc.String(), reader) + if err != nil { + log.Errorf("unable to create a http request url:%s", subject) + return err + } + if username != "" && password != "" { + req.Header.Add("Authorization", "Basic "+basicAuth(username, password)) + } + req.URL.RawQuery = url.Values{ + "digest": {dgst.String()}, + }.Encode() + + req.ContentLength = size + + res, err = http.DefaultClient.Do(req) + if err != nil { + log.Errorf("http request failed url:%s", subject) + return err + } + if res == nil || res.StatusCode != http.StatusCreated { + log.Errorf("unable to upload artifact:%s to url:%s", path, regUrl) + return errors.Errorf("unable to upload artifact:%s to url:%s", path, regUrl) + } + + log.Infof("Copying blob %s done", dgst.Hex()[:12]) + + return nil +} diff --git a/test/bom.bats b/test/bom.bats index b5358de4..fc35eb0c 100644 --- a/test/bom.bats +++ b/test/bom.bats @@ -199,3 +199,99 @@ EOF fi stacker clean } + +@test "pull boms if published" { + skip_slow_test + cat > stacker.yaml < stacker.yaml <