diff --git a/pkg/stacker/bom.go b/pkg/stacker/bom.go
index f1633cb4..e048db8a 100644
--- a/pkg/stacker/bom.go
+++ b/pkg/stacker/bom.go
@@ -3,10 +3,12 @@ package stacker
 import (
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/pkg/errors"
 	"stackerbuild.io/stacker/pkg/container"
 	"stackerbuild.io/stacker/pkg/log"
 	"stackerbuild.io/stacker/pkg/types"
@@ -109,6 +111,11 @@ func ImportArtifacts(sc types.StackerConfig, src types.ImageSource, name string)
 		// if a bom is available, add it here so it can be merged
 		srcpath := path.Join(sc.StackerDir, "artifacts", src.Tag, fmt.Sprintf("%s.json", src.Tag))
 
+		_, err := os.Lstat(srcpath)
+		if err != nil && errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+
 		dstfp, err := os.CreateTemp(path.Join(sc.StackerDir, "artifacts", name), fmt.Sprintf("%s-*.json", name))
 		if err != nil {
 			return err
diff --git a/pkg/stacker/build.go b/pkg/stacker/build.go
index 1a082b0a..c1d30f76 100644
--- a/pkg/stacker/build.go
+++ b/pkg/stacker/build.go
@@ -509,6 +509,30 @@ func (b *Builder) build(s types.Storage, file string) error {
 			}
 		}
 
+		// build artifacts such as BOMs, etc
+		if l.Bom != nil && l.Bom.Generate {
+			log.Debugf("generating layer artifacts for %s", name)
+
+			if err := ImportArtifacts(opts.Config, l.From, name); err != nil {
+				log.Errorf("unable to import previously built artifacts, err:%v", err)
+				return err
+			}
+
+			for _, pkg := range l.Bom.Packages {
+				if err := BuildLayerArtifacts(opts.Config, s, l, name, pkg); err != nil {
+					log.Errorf("failed to generate layer artifacts for %s - %v", name, err)
+					return err
+				}
+			}
+
+			if err := VerifyLayerArtifacts(opts.Config, s, l, name); err != nil {
+				log.Errorf("failed to validate layer artifacts for %s - %v", name, err)
+				// the generated bom is invalid, remove it so we don't publish it and also get a cache miss for rebuilds
+				_ = os.Remove(path.Join(opts.Config.StackerDir, "artifacts", name, fmt.Sprintf("%s.json", name)))
+				return err
+			}
+		}
+
 		// This is a build only layer, meaning we don't need to include
 		// it in the final image, as outputs from it are going to be
 		// imported into future images. Let's just snapshot it and add
@@ -554,29 +578,6 @@ func (b *Builder) build(s types.Storage, file string) error {
 
 		log.Infof("filesystem %s built successfully", name)
 
-		// build artifacts such as BOMs, etc
-		if l.Bom != nil && l.Bom.Generate {
-			log.Debugf("generating layer artifacts for %s", name)
-
-			if err := ImportArtifacts(opts.Config, l.From, name); err != nil {
-				log.Errorf("unable to import previously built artifacts, err:%v", err)
-				return err
-			}
-
-			for _, pkg := range l.Bom.Packages {
-				if err := BuildLayerArtifacts(opts.Config, s, l, name, pkg); err != nil {
-					log.Errorf("failed to generate layer artifacts for %s - %v", name, err)
-					return err
-				}
-			}
-
-			if err := VerifyLayerArtifacts(opts.Config, s, l, name); err != nil {
-				log.Errorf("failed to validate layer artifacts for %s - %v", name, err)
-				// the generated bom is invalid, remove it so we don't publish it and also get a cache miss for rebuilds
-				_ = os.Remove(path.Join(opts.Config.StackerDir, "artifacts", name, fmt.Sprintf("%s.json", name)))
-				return err
-			}
-		}
 	}
 
 	return oci.GC(context.Background())
diff --git a/test/bom.bats b/test/bom.bats
index ae1fca48..57307b58 100644
--- a/test/bom.bats
+++ b/test/bom.bats
@@ -305,3 +305,149 @@ EOF
     fi
     stacker clean
 }
+
+@test "all container contents must be accounted for even for build_only layers" {
+  skip_slow_test
+  cat > stacker.yaml <<"EOF"
+bom-parent:
+    build_only: true
+    from:
+        type: oci
+        url: ${{CENTOS_OCI}}
+    bom:
+      generate: true
+      namespace: "https://test.io/artifacts"
+      packages:
+      - name: pkg1
+        version: 1.0.0
+        license: Apache-2.0
+        paths: [/pkg1]
+      - name: pkg2
+        version: 1.0.0
+        license: Apache-2.0
+        paths: [/pkg2]
+    run: |
+      # discover installed pkgs
+      /stacker/bin/stacker bom discover
+      # our own custom packages
+      mkdir -p /pkg1
+      touch /pkg1/file
+      mkdir -p /pkg2
+      touch /pkg2/file
+      # should cause build to fail!
+      mkdir -p /orphan-without-a-package
+      touch /orphan-without-a-package/file
+      # cleanup
+      rm -rf /var/lib/alternatives /tmp/* \
+        /etc/passwd- /etc/group- /etc/shadow- /etc/gshadow- \
+        /etc/sysconfig/network /etc/nsswitch.conf.bak \
+        /etc/rpm/macros.image-language-conf /var/lib/rpm/.dbenv.lock \
+        /var/lib/rpm/Enhancename /var/lib/rpm/Filetriggername \
+        /var/lib/rpm/Recommendname /var/lib/rpm/Suggestname \
+        /var/lib/rpm/Supplementname /var/lib/rpm/Transfiletriggername \
+        /var/log/anaconda \
+        /etc/sysconfig/anaconda /etc/sysconfig/network-scripts/ifcfg-* \
+        /etc/sysconfig/sshd-permitrootlogin /root/anaconda-* /root/original-* /run/nologin \
+        /var/lib/rpm/.rpm.lock /etc/.pwd.lock /etc/BUILDTIME
+    annotations:
+      org.opencontainers.image.authors: bom-test
+      org.opencontainers.image.vendor: bom-test
+      org.opencontainers.image.licenses: MIT
+EOF
+    run stacker build --substitute CENTOS_OCI=${CENTOS_OCI}
+    [ "$status" -ne 0 ]
+    # a full inventory for this image
+    [ -f .stacker/artifacts/bom-parent/inventory.json ]
+    # sbom for this image shouldn't be generated
+    [ ! -a .stacker/artifacts/bom-parent/first.json ]
+    # building a second time also fails due to missed cache
+    run stacker build
+    [ "$status" -ne 0 ]
+    stacker clean
+}
+
+@test "skip bom generation for built layer" {
+  skip_slow_test
+  cat > stacker.yaml <<"EOF"
+first:
+    from:
+        type: oci
+        url: ${{CENTOS_OCI}}
+    bom:
+      generate: true
+      namespace: "https://test.io/artifacts"
+      packages:
+      - name: pkg1
+        version: 1.0.0
+        license: Apache-2.0
+        paths: [/pkg1]
+      - name: pkg2
+        version: 1.0.0
+        license: Apache-2.0
+        paths: [/pkg2]
+    run: |
+      # our own custom packages
+      mkdir -p /pkg1
+      touch /pkg1/file
+      mkdir -p /pkg2
+      touch /pkg2/file
+
+second:
+  from:
+    type: built
+    tag: first
+  bom:
+    generate: true
+    namespace: "https://test.io/artifacts"
+    packages:
+    - name: pkg1
+      version: 1.0.0
+      license: Apache-2.0
+      paths: [/pkg1]
+    - name: pkg2
+      version: 1.0.0
+      license: Apache-2.0
+      paths: [/pkg2]
+    - name: pkg3
+      version: 1.0.0
+      license: Apache-2.0
+      paths: [/pkg3]
+  run: |
+    # discover installed pkgs
+    /stacker/bin/stacker bom discover
+    # our own custom packages
+    mkdir -p /pkg3
+    touch /pkg3/file
+    # cleanup
+    rm -rf /var/lib/alternatives /tmp/* \
+      /etc/passwd- /etc/group- /etc/shadow- /etc/gshadow- \
+      /etc/sysconfig/network /etc/nsswitch.conf.bak \
+      /etc/rpm/macros.image-language-conf /var/lib/rpm/.dbenv.lock \
+      /var/lib/rpm/Enhancename /var/lib/rpm/Filetriggername \
+      /var/lib/rpm/Recommendname /var/lib/rpm/Suggestname \
+      /var/lib/rpm/Supplementname /var/lib/rpm/Transfiletriggername \
+      /var/log/anaconda \
+      /etc/sysconfig/anaconda /etc/sysconfig/network-scripts/ifcfg-* \
+      /etc/sysconfig/sshd-permitrootlogin /root/anaconda-* /root/original-* /run/nologin \
+      /var/lib/rpm/.rpm.lock /etc/.pwd.lock /etc/BUILDTIME
+  annotations:
+      org.opencontainers.image.authors: bom-test
+      org.opencontainers.image.vendor: bom-test
+      org.opencontainers.image.licenses: MIT
+EOF
+    stacker build --substitute CENTOS_OCI=${CENTOS_OCI}
+    # a full inventory for this image
+    [ -f .stacker/artifacts/second/inventory.json ]
+    # sbom for this image
+    [ -f .stacker/artifacts/second/second.json ]
+    if [ -n "${ZOT_HOST}:${ZOT_PORT}" ]; then
+      zot_setup
+      stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest --substitute CENTOS_OCI=${CENTOS_OCI}
+      refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/second:latest --format "{{json .}}" | jq '.referrer | length')
+      [ $refs -eq 2 ]
+      refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/second:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID')
+      [ $refs == \"SPDXRef-DOCUMENT\" ]
+      zot_teardown
+    fi
+    stacker clean
+}