From 6d069c760c0dec14a716d67ce090f81c0fabd7e5 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani <45800463+rchincha@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:40:14 -0700 Subject: [PATCH] fix: allow bom build and verification for `build_only` layers (#609) * fix: allow bom build and verification for `build_only` layers From our experience, package information may be removed in such layers making it much harder to discover and auto-construct BOMs. So allow this for `build_only` layers also. Signed-off-by: Ramkumar Chinchani * fix: import a bom only if available from built layers It is possible that a build_only layer doesn't generate a BOM, so it cannot be imported in a derived layer. Signed-off-by: Ramkumar Chinchani --------- Signed-off-by: Ramkumar Chinchani --- pkg/stacker/bom.go | 7 +++ pkg/stacker/build.go | 47 +++++++------- test/bom.bats | 146 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 23 deletions(-) diff --git a/pkg/stacker/bom.go b/pkg/stacker/bom.go index f1633cb4..e048db8a 100644 --- a/pkg/stacker/bom.go +++ b/pkg/stacker/bom.go @@ -3,10 +3,12 @@ package stacker import ( "fmt" "io" + "io/fs" "os" "path" "path/filepath" + "github.com/pkg/errors" "stackerbuild.io/stacker/pkg/container" "stackerbuild.io/stacker/pkg/log" "stackerbuild.io/stacker/pkg/types" @@ -109,6 +111,11 @@ func ImportArtifacts(sc types.StackerConfig, src types.ImageSource, name string) // if a bom is available, add it here so it can be merged srcpath := path.Join(sc.StackerDir, "artifacts", src.Tag, fmt.Sprintf("%s.json", src.Tag)) + _, err := os.Lstat(srcpath) + if err != nil && errors.Is(err, fs.ErrNotExist) { + return nil + } + dstfp, err := os.CreateTemp(path.Join(sc.StackerDir, "artifacts", name), fmt.Sprintf("%s-*.json", name)) if err != nil { return err diff --git a/pkg/stacker/build.go b/pkg/stacker/build.go index 1a082b0a..c1d30f76 100644 --- a/pkg/stacker/build.go +++ b/pkg/stacker/build.go @@ -509,6 +509,30 @@ func (b *Builder) build(s types.Storage, file string) error { } } + // build artifacts such as BOMs, etc + if l.Bom != nil && l.Bom.Generate { + log.Debugf("generating layer artifacts for %s", name) + + if err := ImportArtifacts(opts.Config, l.From, name); err != nil { + log.Errorf("unable to import previously built artifacts, err:%v", err) + return err + } + + for _, pkg := range l.Bom.Packages { + if err := BuildLayerArtifacts(opts.Config, s, l, name, pkg); err != nil { + log.Errorf("failed to generate layer artifacts for %s - %v", name, err) + return err + } + } + + if err := VerifyLayerArtifacts(opts.Config, s, l, name); err != nil { + log.Errorf("failed to validate layer artifacts for %s - %v", name, err) + // the generated bom is invalid, remove it so we don't publish it and also get a cache miss for rebuilds + _ = os.Remove(path.Join(opts.Config.StackerDir, "artifacts", name, fmt.Sprintf("%s.json", name))) + return err + } + } + // This is a build only layer, meaning we don't need to include // it in the final image, as outputs from it are going to be // imported into future images. Let's just snapshot it and add @@ -554,29 +578,6 @@ func (b *Builder) build(s types.Storage, file string) error { log.Infof("filesystem %s built successfully", name) - // build artifacts such as BOMs, etc - if l.Bom != nil && l.Bom.Generate { - log.Debugf("generating layer artifacts for %s", name) - - if err := ImportArtifacts(opts.Config, l.From, name); err != nil { - log.Errorf("unable to import previously built artifacts, err:%v", err) - return err - } - - for _, pkg := range l.Bom.Packages { - if err := BuildLayerArtifacts(opts.Config, s, l, name, pkg); err != nil { - log.Errorf("failed to generate layer artifacts for %s - %v", name, err) - return err - } - } - - if err := VerifyLayerArtifacts(opts.Config, s, l, name); err != nil { - log.Errorf("failed to validate layer artifacts for %s - %v", name, err) - // the generated bom is invalid, remove it so we don't publish it and also get a cache miss for rebuilds - _ = os.Remove(path.Join(opts.Config.StackerDir, "artifacts", name, fmt.Sprintf("%s.json", name))) - return err - } - } } return oci.GC(context.Background()) diff --git a/test/bom.bats b/test/bom.bats index ae1fca48..57307b58 100644 --- a/test/bom.bats +++ b/test/bom.bats @@ -305,3 +305,149 @@ EOF fi stacker clean } + +@test "all container contents must be accounted for even for build_only layers" { + skip_slow_test + cat > stacker.yaml <<"EOF" +bom-parent: + build_only: true + from: + type: oci + url: ${{CENTOS_OCI}} + bom: + generate: true + namespace: "https://test.io/artifacts" + packages: + - name: pkg1 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg1] + - name: pkg2 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg2] + run: | + # discover installed pkgs + /stacker/bin/stacker bom discover + # our own custom packages + mkdir -p /pkg1 + touch /pkg1/file + mkdir -p /pkg2 + touch /pkg2/file + # should cause build to fail! + mkdir -p /orphan-without-a-package + touch /orphan-without-a-package/file + # cleanup + rm -rf /var/lib/alternatives /tmp/* \ + /etc/passwd- /etc/group- /etc/shadow- /etc/gshadow- \ + /etc/sysconfig/network /etc/nsswitch.conf.bak \ + /etc/rpm/macros.image-language-conf /var/lib/rpm/.dbenv.lock \ + /var/lib/rpm/Enhancename /var/lib/rpm/Filetriggername \ + /var/lib/rpm/Recommendname /var/lib/rpm/Suggestname \ + /var/lib/rpm/Supplementname /var/lib/rpm/Transfiletriggername \ + /var/log/anaconda \ + /etc/sysconfig/anaconda /etc/sysconfig/network-scripts/ifcfg-* \ + /etc/sysconfig/sshd-permitrootlogin /root/anaconda-* /root/original-* /run/nologin \ + /var/lib/rpm/.rpm.lock /etc/.pwd.lock /etc/BUILDTIME + annotations: + org.opencontainers.image.authors: bom-test + org.opencontainers.image.vendor: bom-test + org.opencontainers.image.licenses: MIT +EOF + run stacker build --substitute CENTOS_OCI=${CENTOS_OCI} + [ "$status" -ne 0 ] + # a full inventory for this image + [ -f .stacker/artifacts/bom-parent/inventory.json ] + # sbom for this image shouldn't be generated + [ ! -a .stacker/artifacts/bom-parent/first.json ] + # building a second time also fails due to missed cache + run stacker build + [ "$status" -ne 0 ] + stacker clean +} + +@test "skip bom generation for built layer" { + skip_slow_test + cat > stacker.yaml <<"EOF" +first: + from: + type: oci + url: ${{CENTOS_OCI}} + bom: + generate: true + namespace: "https://test.io/artifacts" + packages: + - name: pkg1 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg1] + - name: pkg2 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg2] + run: | + # our own custom packages + mkdir -p /pkg1 + touch /pkg1/file + mkdir -p /pkg2 + touch /pkg2/file + +second: + from: + type: built + tag: first + bom: + generate: true + namespace: "https://test.io/artifacts" + packages: + - name: pkg1 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg1] + - name: pkg2 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg2] + - name: pkg3 + version: 1.0.0 + license: Apache-2.0 + paths: [/pkg3] + run: | + # discover installed pkgs + /stacker/bin/stacker bom discover + # our own custom packages + mkdir -p /pkg3 + touch /pkg3/file + # cleanup + rm -rf /var/lib/alternatives /tmp/* \ + /etc/passwd- /etc/group- /etc/shadow- /etc/gshadow- \ + /etc/sysconfig/network /etc/nsswitch.conf.bak \ + /etc/rpm/macros.image-language-conf /var/lib/rpm/.dbenv.lock \ + /var/lib/rpm/Enhancename /var/lib/rpm/Filetriggername \ + /var/lib/rpm/Recommendname /var/lib/rpm/Suggestname \ + /var/lib/rpm/Supplementname /var/lib/rpm/Transfiletriggername \ + /var/log/anaconda \ + /etc/sysconfig/anaconda /etc/sysconfig/network-scripts/ifcfg-* \ + /etc/sysconfig/sshd-permitrootlogin /root/anaconda-* /root/original-* /run/nologin \ + /var/lib/rpm/.rpm.lock /etc/.pwd.lock /etc/BUILDTIME + annotations: + org.opencontainers.image.authors: bom-test + org.opencontainers.image.vendor: bom-test + org.opencontainers.image.licenses: MIT +EOF + stacker build --substitute CENTOS_OCI=${CENTOS_OCI} + # a full inventory for this image + [ -f .stacker/artifacts/second/inventory.json ] + # sbom for this image + [ -f .stacker/artifacts/second/second.json ] + if [ -n "${ZOT_HOST}:${ZOT_PORT}" ]; then + zot_setup + stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest --substitute CENTOS_OCI=${CENTOS_OCI} + refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/second:latest --format "{{json .}}" | jq '.referrer | length') + [ $refs -eq 2 ] + refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/second:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID') + [ $refs == \"SPDXRef-DOCUMENT\" ] + zot_teardown + fi + stacker clean +}