From 4f4977a9668359bd6703b928a1e6a8245b8c8757 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani <45800463+rchincha@users.noreply.github.com> Date: Fri, 20 Oct 2023 12:50:51 -0700 Subject: [PATCH] fix(gzip)!: change the default block size (#529) BREAKING CHANGE: the default gzip block size is changed to 256<<12, was previously 256<<10. A tar layer with the same content but compressed with different gzip blocksize will result in different sha256sums in the final OCI Image. Ecosystem tools have one current size in use and stacker's current size differ. Interactions between a stacker-built OCI image and ecosystem tools which recompress lower layers results in bloated registries which will have identical tar content but different compressed sha256 blobs. Unfortunately, the OCI image spec doesn't standardize/encode this in the specification document. Hence, we change to the current common block size used in the ecosystem here in the stacker implementation. We now link against our own fork: github.com/project-stacker/umoci which may change depending on the PR getting merged to upstream. Signed-off-by: Ramkumar Chinchani (cherry picked from commit 0cf2d706d0505efa743e1b42c8ad248e774df0ab) Signed-off-by: Ramkumar Chinchani --- pkg/overlay/pack.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/overlay/pack.go b/pkg/overlay/pack.go index dad09720..641b6ec9 100644 --- a/pkg/overlay/pack.go +++ b/pkg/overlay/pack.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" "github.com/klauspost/pgzip" @@ -29,6 +30,14 @@ import ( "stackerbuild.io/stacker/pkg/types" ) +var tarEx sync.Mutex + +// Container image layers are often tar.gz, however there is nothing in the +// spec or documentation which standardizes compression params which can cause +// different layer hashes even for the same tar. So picking compression params +// that most tooling appears to be using. +const gzipBlockSize = mutate.GzipBlockSize(256 << 12) + func safeOverlayName(d digest.Digest) string { // dirs used in overlay lowerdir args can't have : in them, so lets // sanitize it @@ -418,7 +427,7 @@ func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mu defer blob.Close() if layerType.Type == "tar" { - desc, err = mutator.Add(context.Background(), mediaType, blob, history, mutate.GzipCompressor, nil) + desc, err = mutator.Add(context.Background(), mediaType, blob, history, mutate.GzipCompressor.WithOpt(gzipBlockSize), nil) if err != nil { return false, err }