From d89af756a7080e1a2d35014b71dea76686f53613 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 19 Oct 2023 20:11:45 +0000 Subject: [PATCH] fix(gzip)!: change the default block size BREAKING CHANGE: the default gzip block size is changed to 256<<12, was previously 256<<10. A tar layer with the same content but compressed with different gzip blocksize will result in different sha256sums in the final OCI Image. Ecosystem tools have one current size in use and stacker's current size differ. Interactions between a stacker-built OCI image and ecosystem tools which recompress lower layers results in bloated registries which will have identical tar content but different compressed sha256 blobs. Unfortunately, the OCI image spec doesn't standardize/encode this in the specification document. Hence, we change to the current common block size used in the ecosystem here in the stacker implementation. We now link against our own fork: github.com/project-stacker/umoci which may change depending on the PR getting merged to upstream. Signed-off-by: Ramkumar Chinchani --- go.mod | 9 ++++++--- go.sum | 8 ++++---- pkg/overlay/pack.go | 8 +++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index d27da772..24c5b521 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/moby/buildkit v0.11.4 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc4 - github.com/opencontainers/umoci v0.4.8-0.20230920134428-7dc114a520bc + github.com/opencontainers/umoci v0.0.0-00000000000000-000000000000 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.9 github.com/sirupsen/logrus v1.9.0 @@ -30,7 +30,7 @@ require ( github.com/udhos/equalfile v0.3.0 github.com/urfave/cli/v2 v2.25.0 github.com/vbatts/go-mtree v0.5.3 - golang.org/x/sys v0.10.0 + golang.org/x/sys v0.13.0 golang.org/x/term v0.8.0 gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/bom v0.5.2-0.20230512052447-fef7b03b207d @@ -233,4 +233,7 @@ require ( sigs.k8s.io/release-utils v0.7.4 // indirect ) -replace stackerbuild.io/stacker-bom => github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250 +replace ( + github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20231019200834-3f97387412c4 + stackerbuild.io/stacker-bom => github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250 +) diff --git a/go.sum b/go.sum index 56dbd8ba..3dddfa08 100644 --- a/go.sum +++ b/go.sum @@ -690,8 +690,6 @@ github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0 github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/opencontainers/umoci v0.4.8-0.20230920134428-7dc114a520bc h1:zgr8RvoUdIXVdayfl7tR5VM9eawzfVss4foXcIawouM= -github.com/opencontainers/umoci v0.4.8-0.20230920134428-7dc114a520bc/go.mod h1:m/PjYk1TA9ja9k1M2PYkrLpSYH+80O4pABbZcln/IiU= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170 h1:DiLBVp4DAcZlBVBEtJpNWZpZVq0AEeCY7Hqk8URVs4o= github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= @@ -725,6 +723,8 @@ github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250 h1:5gSyDxGXisvvu+aMUq7WRxgq3phvdy9/1CM/TqUHLVQ= github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250/go.mod h1:P0o0hINRm/kcAB0CRf/W9RMLBWWb2EzzhPysXipj3Cg= +github.com/project-stacker/umoci v0.0.0-20231019200834-3f97387412c4 h1:mtCuBc3xMRcZQCPHbDsfKXkr3TJL3N4OPg+2tQnH55w= +github.com/project-stacker/umoci v0.0.0-20231019200834-3f97387412c4/go.mod h1:XUXUpCpA/Y8aJWezK1i8o4WDR0Y/vhMcWg+FUNQkKMQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= @@ -1193,8 +1193,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/pkg/overlay/pack.go b/pkg/overlay/pack.go index 0d342a42..0e893242 100644 --- a/pkg/overlay/pack.go +++ b/pkg/overlay/pack.go @@ -32,6 +32,12 @@ import ( var tarEx sync.Mutex +// Container image layers are often tar.gz, however there is nothing in the +// spec or documentation which standardizes compression params which can cause +// different layer hashes even for the same tar. So picking compression params +// that most tooling appears to be using. +const gzipBlockSize = mutate.GzipBlockSize(256 << 12) + func safeOverlayName(d digest.Digest) string { // dirs used in overlay lowerdir args can't have : in them, so lets // sanitize it @@ -408,7 +414,7 @@ func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mu defer blob.Close() if layerType.Type == "tar" { - desc, err = mutator.Add(context.Background(), mediaType, blob, history, mutate.GzipCompressor, nil) + desc, err = mutator.Add(context.Background(), mediaType, blob, history, mutate.GzipCompressor.WithOpt(gzipBlockSize), nil) if err != nil { return false, err }