fix: remove superfluous userxattr whiteouts

overlayfs creates whiteout entries whenever a dir is created which is
not ideal. Instead, we inspect all the lower layers and only if a dir
exists, we emit a whiteout entry.

Note that we link against the 'stacker' branch of project-stacker/umoci

Signed-off-by: Ramkumar Chinchani <[email protected]>

Makefile

@@ -46,7 +46,7 @@ BATS = $(TOOLS_D)/bin/bats
 BATS_VERSION := v1.10.0
 # OCI registry
 ZOT := $(TOOLS_D)/bin/zot
-ZOT_VERSION := v2.0.2
+ZOT_VERSION := v2.1.0
 
 export PATH := $(TOOLS_D)/bin:$(PATH)
 

go.mod

@@ -284,6 +284,6 @@ require (
 )
 
 replace (
-	github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20240417195808-16c510104378
+	github.com/opencontainers/umoci => github.com/rchincha/umoci v0.0.0-20240725220840-84c229617bbc
 	stackerbuild.io/stacker-bom => github.com/project-stacker/stacker-bom v0.0.0-20240509203427-4d685e046780
 )

go.sum

@@ -795,8 +795,6 @@ github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU
 github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
 github.com/project-stacker/stacker-bom v0.0.0-20240509203427-4d685e046780 h1:VJQ/G6xlNQqEvdzTdtXJ/XNvxv9LQTDJORik1wuxXJU=
 github.com/project-stacker/stacker-bom v0.0.0-20240509203427-4d685e046780/go.mod h1:S7hlUdKwPKIMNx2ceiqmLKyXq+FOVaVnAuk77AzdhtI=
-github.com/project-stacker/umoci v0.0.0-20240417195808-16c510104378 h1:NHGVwIe0Icrbn8b7WW55FQjETHm5B0i6Gt6UZmzaF9A=
-github.com/project-stacker/umoci v0.0.0-20240417195808-16c510104378/go.mod h1:XUXUpCpA/Y8aJWezK1i8o4WDR0Y/vhMcWg+FUNQkKMQ=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
@@ -818,6 +816,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/rchincha/umoci v0.0.0-20240725220840-84c229617bbc h1:sVLSTbOcpkvJumNzsBR9ePJPEzKF024iDUxGgu2LKJ8=
+github.com/rchincha/umoci v0.0.0-20240725220840-84c229617bbc/go.mod h1:XUXUpCpA/Y8aJWezK1i8o4WDR0Y/vhMcWg+FUNQkKMQ=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=

install-build-deps.sh

@@ -18,6 +18,7 @@ installdeps_fedora() {
         sudo dnf install golang
         go version
     fi
+    sudo dnf install bsdtar
 }
 
 installdeps_ubuntu() {
@@ -40,7 +41,8 @@ installdeps_ubuntu() {
             parallel \
             pkg-config \
             squashfs-tools \
-            squashfuse
+            squashfuse \
+            libarchive-tools
     # skopeo deps
     sudo apt -yy install \
        libgpgme-dev \

pkg/overlay/overlay-dirs.go

@@ -48,7 +48,7 @@ func generateOverlayDirLayer(name string, layerType types.LayerType, overlayDir
 	defer oci.Close()
 
 	contents := path.Join(config.RootFSDir, name, "overlay_dirs", path.Base(overlayDir.Source))
-	blob, mediaType, rootHash, err := generateBlob(layerType, contents, config.OCIDir)
+	blob, mediaType, rootHash, err := generateBlob(layerType, contents, config.OCIDir, nil)
 	if err != nil {
 		return ispec.Descriptor{}, err
 	}

pkg/overlay/pack.go

@@ -135,7 +135,7 @@ func ConvertAndOutput(config types.StackerConfig, tag, name string, layerType ty
 		bundlePath := overlayPath(config.RootFSDir, theLayer.Digest)
 		overlayDir := path.Join(bundlePath, "overlay")
 		// generate blob
-		blob, mediaType, rootHash, err := generateBlob(layerType, overlayDir, config.OCIDir)
+		blob, mediaType, rootHash, err := generateBlob(layerType, overlayDir, config.OCIDir, nil)
 		if err != nil {
 			return err
 		}
@@ -283,23 +283,23 @@ func (o *overlay) initializeBasesInOutput(name string, layerTypes []types.LayerT
 	return nil
 }
 
-func (o *overlay) Repack(name string, layerTypes []types.LayerType, sfm types.StackerFiles) error {
+func (o *overlay) Repack(name string, layer types.Layer, layerTypes []types.LayerType, sfm types.StackerFiles) error {
 	err := o.initializeBasesInOutput(name, layerTypes, sfm)
 	if err != nil {
 		return err
 	}
 
-	return repackOverlay(o.config, name, layerTypes)
+	return repackOverlay(o.config, name, layer, layerTypes)
 }
 
 // generateBlob generates either a tar blob or a squashfs blob based on layerType
-func generateBlob(layerType types.LayerType, contents string, ociDir string) (io.ReadCloser, string, string, error) {
+func generateBlob(layerType types.LayerType, contents string, ociDir string, lowerDirs []string) (io.ReadCloser, string, string, error) {
 	var blob io.ReadCloser
 	var err error
 	var mediaType string
 	var rootHash string
 	if layerType.Type == "tar" {
-		packOptions := layer.RepackOptions{TranslateOverlayWhiteouts: true}
+		packOptions := layer.RepackOptions{TranslateOverlayWhiteouts: true, OverlayLowerDirs: lowerDirs}
 		blob = layer.GenerateInsertLayer(contents, "/", false, &packOptions)
 		mediaType = ispec.MediaTypeImageLayer
 	} else {
@@ -382,7 +382,9 @@ func stripOverlayAttrsUnder(dirPath string) error {
 		})
 }
 
-func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mutate.Mutator, name string, layerTypes []types.LayerType) (bool, error) {
+func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mutate.Mutator,
+	name string, layer types.Layer, layerTypes []types.LayerType,
+) (bool, error) {
 	dir := path.Join(config.RootFSDir, name, "overlay")
 	ents, err := os.ReadDir(dir)
 	if err != nil {
@@ -430,12 +432,27 @@ func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mu
 		return false, err
 	}
 
+	var ovl overlayMetadata
+	if layer.From.Type != types.BuiltLayer {
+		ovl, err = readOverlayMetadata(config.RootFSDir, name)
+	} else {
+		ovl, err = readOverlayMetadata(config.RootFSDir, layer.From.Tag)
+	}
+	if err != nil {
+		return false, err
+	}
+
 	descs := []ispec.Descriptor{}
 	for i, layerType := range layerTypes {
 		mutator := mutators[i]
 		var desc ispec.Descriptor
 
-		blob, mediaType, rootHash, err := generateBlob(layerType, dir, config.OCIDir)
+		lowerDirs := []string{}
+		for i := len(ovl.Manifests[layerType].Layers) - 1; i >= 0; i-- {
+			lowerDirs = append(lowerDirs, overlayPath(config.RootFSDir, ovl.Manifests[layerType].Layers[i].Digest))
+		}
+
+		blob, mediaType, rootHash, err := generateBlob(layerType, dir, config.OCIDir, lowerDirs)
 		if err != nil {
 			return false, err
 		}
@@ -547,7 +564,7 @@ func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mu
 	return true, nil
 }
 
-func repackOverlay(config types.StackerConfig, name string, layerTypes []types.LayerType) error {
+func repackOverlay(config types.StackerConfig, name string, layer types.Layer, layerTypes []types.LayerType) error {
 	oci, err := umoci.OpenLayout(config.OCIDir)
 	if err != nil {
 		return err
@@ -602,7 +619,7 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L
 	// generate blobs for each build layer
 	for _, buildLayer := range ovl.BuiltLayers {
 
-		didMutate, err := generateLayer(config, oci, mutators, buildLayer, layerTypes)
+		didMutate, err := generateLayer(config, oci, mutators, buildLayer, layer, layerTypes)
 		if err != nil {
 			return err
 		}
@@ -637,7 +654,7 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L
 		return err
 	}
 
-	didMutate, err := generateLayer(config, oci, mutators, name, layerTypes)
+	didMutate, err := generateLayer(config, oci, mutators, name, layer, layerTypes)
 	if err != nil {
 		return err
 	}

pkg/stacker/build.go

@@ -551,7 +551,7 @@ func (b *Builder) build(s types.Storage, file string) error {
 			continue
 		}
 
-		err = s.Repack(name, opts.LayerTypes, b.builtStackerfiles)
+		err = s.Repack(name, l, opts.LayerTypes, b.builtStackerfiles)
 		if err != nil {
 			return err
 		}

pkg/types/storage.go

@@ -47,7 +47,7 @@ type Storage interface {
 	Unpack(tag, name string) error
 
 	// Repack repacks the specified working dir into the specified OCI dir.
-	Repack(name string, layerTypes []LayerType, sfm StackerFiles) error
+	Repack(name string, layer Layer, layerTypes []LayerType, sfm StackerFiles) error
 
 	// GetLXCRootfsConfig returns the string that should be set as
 	// lxc.rootfs.path in the LXC container's config.

test/publish.bats

@@ -260,8 +260,12 @@ parent:
   run: |
     rm -rf /etc/apk/repositories
 EOF
+  stacker check
   stacker build
-  stacker publish --skip-tls --url docker://${REGISTRY_URL} --tag latest --substitute BUSYBOX_OCI=${BUSYBOX_OCI}
+  manifest0=$(cat oci/index.json | jq -r .manifests[0].digest | cut -f2 -d:)
+  layers0=$(cat oci/blobs/sha256/$manifest0 | jq -r .layers[1].digest | cut -f2 -d:)
+  bsdtar -tvf oci/blobs/sha256/$layers0
+  stacker publish --skip-tls --url docker://${REGISTRY_URL} --tag latest
   stacker clean
 
   cat > stacker.yaml <<"EOF"
@@ -271,6 +275,10 @@ child:
     url: docker://${{REGISTRY_URL}}/parent:latest
     insecure: true
   run: |
+    ps
+    ls -l /
+    ls -l /etc
+    ls -l /etc/apk
     [ ! -f /etc/apk/repositories ]
 EOF
   stacker build --substitute REGISTRY_URL=${REGISTRY_URL}

test/whiteout.bats

@@ -0,0 +1,75 @@
+load helpers
+
+function setup() {
+    stacker_setup
+}
+
+function teardown() {
+    cleanup
+}
+
+@test "test not adding extraneous whiteouts" {
+    cat > stacker.yaml <<EOF
+image:
+  from:
+    type: docker
+    url: docker://public.ecr.aws/ubuntu/ubuntu:latest
+  run: |
+    apt-get update
+    apt-get -y install libsensors-config
+EOF
+
+  stacker build
+  echo "checking"
+  for f in $(ls oci/blobs/sha256/); do
+    file oci/blobs/sha256/$f | grep "gzip" || {
+      echo "skipping $f"
+        continue
+    }
+    bsdtar -tvf oci/blobs/sha256/$f
+    run "bsdtar -tvf oci/blobs/sha256/$f | grep '.wh.sensors.d'"
+    if [ "$status" -eq 0 ]; then
+      echo "should not have a sensors.d whiteout!";
+      exit 1;
+    fi
+  done
+}
+
+@test "dont emit whiteout for new dir creates" {
+  cat > stacker.yaml <<EOF
+  # a1.tar has /a1/file
+bb:
+  from:
+    type: docker
+    url: docker://busybox
+  run: |
+    mkdir /a1
+    touch /a1/file
+
+nodir:
+  from:
+    type: built
+    tag: bb
+  run: |
+    rm -rf /a1
+
+emptydir:
+  from:
+    type: built
+    tag: bb
+  run: |
+    rm -rf /a1
+    mkdir /a1
+
+fulldir:
+  from:
+    type: built
+    tag: bb
+  run: |
+    rm -rf /a1
+    mkdir /a1
+    touch /a1/newfile
+EOF
+
+  stacker build
+}