Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support importing sboms along with images #567

Merged
merged 1 commit into from
Mar 18, 2024
Merged

feat: support importing sboms along with images #567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ BATS = $(TOOLS_D)/bin/bats
BATS_VERSION := v1.10.0
# OCI registry
ZOT := $(TOOLS_D)/bin/zot
ZOT_VERSION := v2.0.0
ZOT_VERSION := v2.0.2

export PATH := $(TOOLS_D)/bin:$(PATH)

Expand Down
10 changes: 9 additions & 1 deletion pkg/stacker/base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,15 @@
case types.OCILayer:
fallthrough
case types.DockerLayer:
return importContainersImage(o.Layer.From, o.Config, o.Progress)
err := importContainersImage(o.Layer.From, o.Config, o.Progress)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a comment w.r.t when we take this path (similar to the comment for this PR)?

if o.Layer.Bom != nil && o.Layer.Bom.Generate && (o.Layer.From.Type == types.DockerLayer) {
bomPath := path.Join(o.Config.StackerDir, "artifacts", o.Name)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there a common function off of o.Config which would return the "artifacts" dir?

err = getArtifact(bomPath, "application/spdx+json", o.Layer.From.Url, "", "", o.Layer.From.Insecure)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we have a const for the mimetype ?

if err != nil {
log.Errorf("sbom for image %s not found", o.Layer.From.Url)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
log.Errorf("sbom for image %s not found", o.Layer.From.Url)
log.Errorf("sbom artifact for image %s not found", o.Layer.From.Url)

}

Check warning on line 55 in pkg/stacker/base.go

View check run for this annotation

Codecov / codecov/patch

pkg/stacker/base.go#L54-L55 

Added lines #L54 - L55 were not covered by tests
}
return err
default:
return errors.Errorf("unknown layer type: %v", o.Layer.From.Type)
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/stacker/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ type BuildArgs struct {
SetupOnly bool
Progress bool
AnnotationsNamespace string
Username string
Password string
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this need any sort of special handling so that the value is not leaked in logs?

}

// Builder is responsible for building the layers based on stackerfiles
Expand Down
266 changes: 4 additions & 262 deletions pkg/stacker/publisher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,14 @@
package stacker

import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"strings"

godigest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/umoci"
"github.com/opencontainers/umoci/oci/casext"
"github.com/pkg/errors"
Expand Down Expand Up @@ -75,255 +66,6 @@ func NewPublisher(opts *PublishArgs) *Publisher {
}
}

func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}

func clientRequest(method, url, username, password string, headers map[string]string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequestWithContext(context.TODO(), method, url, body)
if err != nil {
log.Errorf("unable to create http request err:%s", err)
return nil, err
}

// FIXME: handle bearer auth also
if username != "" && password != "" {
req.Header.Add("Authorization", "Basic "+basicAuth(username, password))
}

if len(headers) > 0 {
for k, v := range headers {
req.Header.Add(k, v)
}
}

res, err := http.DefaultClient.Do(req)
if err != nil {
log.Errorf("http request failed url:%s", url)
return nil, err
}

return res, nil
}

func fileDigest(path string) (*godigest.Digest, error) {
rchincha marked this conversation as resolved.
Show resolved Hide resolved
fh, err := os.Open(path)
if err != nil {
log.Errorf("unable to open file:%s, err:%s", path, err)
return nil, err
}
defer fh.Close()

dgst, err := godigest.FromReader(fh)
if err != nil {
log.Errorf("unable get digest for file:%s, err:%s", path, err)
return nil, err
}

return &dgst, nil
}

// publishArtifact to a registry/repo for this subject
func (p *Publisher) publishArtifact(path, mtype, registry, repo, subjectTag string, skipTLS bool) error {
username := p.opts.Username
password := p.opts.Password

subject := distspecURL(registry, repo, subjectTag, skipTLS)

// check subject exists
res, err := clientRequest(http.MethodHead, subject, username, password, nil, nil)
if err != nil {
log.Errorf("unable to check subject:%s, err:%s", subject, err)
return err
}
if res == nil || res.StatusCode != http.StatusOK {
log.Errorf("subject:%s doesn't exist, ignoring and proceeding", subject)
}

slen := res.ContentLength
smtype := res.Header.Get("Content-Type")
sdgst, err := godigest.Parse(res.Header.Get("Docker-Content-Digest"))
if slen < 0 || smtype == "" || sdgst == "" || err != nil {
log.Errorf("unable to get descriptor details for subject:%s", subject)
return errors.Errorf("unable to get descriptor details for subject:%s", subject)
}

// upload the artifact
finfo, err := os.Lstat(path)
if err != nil {
log.Errorf("unable to stat file:%s, err:%s", path, err)
return err
}

dgst, err := fileDigest(path)
if err != nil {
log.Errorf("unable get digest for file:%s, err:%s", path, err)
return err
}

fh, err := os.Open(path)
if err != nil {
log.Errorf("unable to open file:%s, err:%s", path, err)
return err
}
defer fh.Close()

if err := uploadBlob(registry, repo, path, username, password, fh, finfo.Size(), dgst, skipTLS); err != nil {
log.Errorf("unable to upload file:%s, err:%s", path, err)
return err
}

// check and upload emptyJSON blob
erdr := bytes.NewReader(ispec.DescriptorEmptyJSON.Data)
edgst := ispec.DescriptorEmptyJSON.Digest

if err := uploadBlob(registry, repo, path, username, password, erdr, ispec.DescriptorEmptyJSON.Size, &edgst, skipTLS); err != nil {
log.Errorf("unable to upload file:%s, err:%s", path, err)
return err
}

// upload the reference manifest
manifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: mtype,
Config: ispec.DescriptorEmptyJSON,
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Size: slen,
Digest: sdgst,
},
Layers: []ispec.Descriptor{
ispec.Descriptor{
MediaType: mtype,
Size: finfo.Size(),
Digest: *dgst,
},
},
}

//content, err := json.MarshalIndent(&manifest, "", "\t")
content, err := json.Marshal(&manifest)
if err != nil {
log.Errorf("unable to marshal image manifest, err:%s", err)
return err
}

// artifact manifest
var regUrl string
mdgst := godigest.FromBytes(content)
if skipTLS {
regUrl = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String())
} else {
regUrl = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String())
}
hdrs := map[string]string{
"Content-Type": ispec.MediaTypeImageManifest,
"Content-Length": fmt.Sprintf("%d", len(content)),
}
res, err = clientRequest(http.MethodPut, regUrl, username, password, hdrs, bytes.NewBuffer(content))
if err != nil {
log.Errorf("unable to check subject:%s, err:%s", subject, err)
return err
}
if res == nil || res.StatusCode != http.StatusCreated {
log.Errorf("unable to upload manifest, url:%s", regUrl)
return errors.Errorf("unable to upload manifest, url:%s", regUrl)
}

log.Infof("Copying artifact '%s' done", path)

return nil
}

func distspecURL(registry, repo, tag string, skipTLS bool) string {
var url string

if skipTLS {
url = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag)
} else {
url = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag)
}

return url
}

func uploadBlob(registry, repo, path, username, password string, reader io.Reader, size int64, dgst *godigest.Digest, skipTLS bool) error {
// upload with POST, PUT sequence
var regUrl string
if skipTLS {
regUrl = fmt.Sprintf("http://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String())
} else {
regUrl = fmt.Sprintf("https://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String())
}

subject := distspecURL(registry, repo, "", skipTLS)

log.Debugf("check blob before upload (HEAD): %s", regUrl)
res, err := clientRequest(http.MethodHead, regUrl, username, password, nil, nil)
if err != nil {
log.Errorf("unable to check blob:%s, err:%s", subject, err)
return err
}
log.Debugf("http response headers: +%v status:%v", res.Header, res.Status)
hdr := res.Header.Get("Docker-Content-Digest")
if hdr != "" {
log.Infof("Copying blob %s skipped: already exists", dgst.Hex()[:12])
return nil
}

if skipTLS {
regUrl = fmt.Sprintf("http://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0])
} else {
regUrl = fmt.Sprintf("https://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0])
}

log.Debugf("new blob upload (POST): %s", regUrl)
res, err = clientRequest(http.MethodPost, regUrl, username, password, nil, nil)
if err != nil {
log.Errorf("post unable to check subject:%s, err:%s", subject, err)
return err
}
log.Debugf("http response headers: +%v status:%v", res.Header, res.Status)
loc, err := res.Location()
if err != nil {
log.Errorf("unable get upload location url:%s, err:%s", regUrl, err)
return err
}

log.Debugf("finish blob upload (PUT): %s", regUrl)
req, err := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc.String(), reader)
if err != nil {
log.Errorf("unable to create a http request url:%s", subject)
return err
}
if username != "" && password != "" {
req.Header.Add("Authorization", "Basic "+basicAuth(username, password))
}
req.URL.RawQuery = url.Values{
"digest": {dgst.String()},
}.Encode()

req.ContentLength = size

res, err = http.DefaultClient.Do(req)
if err != nil {
log.Errorf("http request failed url:%s", subject)
return err
}
if res == nil || res.StatusCode != http.StatusCreated {
log.Errorf("unable to upload artifact:%s to url:%s", path, regUrl)
return errors.Errorf("unable to upload artifact:%s to url:%s", path, regUrl)
}

log.Infof("Copying blob %s done", dgst.Hex()[:12])

return nil
}

// Publish layers in a single stackerfile
func (p *Publisher) Publish(file string) error {
opts := p.opts
Expand Down Expand Up @@ -452,14 +194,14 @@ func (p *Publisher) Publish(file string) error {
repo := url.Path

// publish sbom
if err := p.publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, fmt.Sprintf("%s.json", layerName)),
"application/spdx+json", registry, repo, layerTypeTag, opts.SkipTLS); err != nil {
if err := publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, fmt.Sprintf("%s.json", layerName)),
"application/spdx+json", registry, repo, layerTypeTag, p.opts.Username, p.opts.Password, opts.SkipTLS); err != nil {
return err
}

// publish inventory
if err := p.publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, "inventory.json"),
"application/vnd.stackerbuild.inventory+json", registry, repo, layerTypeTag, opts.SkipTLS); err != nil {
if err := publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, "inventory.json"),
"application/vnd.stackerbuild.inventory+json", registry, repo, layerTypeTag, p.opts.Username, p.opts.Password, opts.SkipTLS); err != nil {
return err
}

Expand Down
Loading
Loading