From 9d8f3a29685788ee69a8db077c51c8237590f5d3 Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Fri, 12 Jan 2024 15:37:17 +0100 Subject: [PATCH] Add source timestamp for Bundle step Add flag to write source timestamp into result file. Fix `PackAndPush` function to set timestamp of the base image and not for all files in the main layer of the image to keep the timestamps of the files in the bundle layer. --- cmd/bundle/main.go | 46 +++++++++++++++----- cmd/bundle/main_test.go | 88 +++++++++++++++++++++++++++++++++++++-- pkg/bundle/bundle.go | 43 ++++++++++++------- pkg/bundle/bundle_test.go | 5 ++- 4 files changed, 154 insertions(+), 28 deletions(-) diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 21a5c09399..1db85beafc 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -9,8 +9,11 @@ import ( "fmt" "log" "os" + "strconv" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/pflag" "github.com/shipwright-io/build/pkg/bundle" @@ -18,12 +21,13 @@ import ( ) type settings struct { - help bool - image string - prune bool - target string - secretPath string - resultFileImageDigest string + help bool + image string + prune bool + target string + secretPath string + resultFileImageDigest string + resultFileSourceTimestamp string } var flagValues settings @@ -36,6 +40,7 @@ func init() { pflag.StringVar(&flagValues.image, "image", "", "Location of the bundle image (mandatory)") pflag.StringVar(&flagValues.target, "target", "/workspace/source", "The target directory to place the code") pflag.StringVar(&flagValues.resultFileImageDigest, "result-file-image-digest", "", "A file to write the image digest") + pflag.StringVar(&flagValues.resultFileSourceTimestamp, "result-file-source-timestamp", "", "A file to write the source timestamp") pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains access credentials (optional)") pflag.BoolVar(&flagValues.prune, "prune", false, "Delete bundle image from registry after it was pulled") @@ -72,10 +77,20 @@ func Do(ctx context.Context) error { } log.Printf("Pulling image %q", ref) - img, err := bundle.PullAndUnpack( - ref, - flagValues.target, - options...) + desc, err := remote.Get(ref, options...) + if err != nil { + return err + } + + img, err := desc.Image() + if err != nil { + return err + } + + rc := mutate.Extract(img) + defer rc.Close() + + unpackDetails, err := bundle.Unpack(rc, flagValues.target) if err != nil { return err } @@ -93,6 +108,17 @@ func Do(ctx context.Context) error { } } + if flagValues.resultFileSourceTimestamp != "" { + if unpackDetails.MostRecentFileTimestamp != nil { + if err = os.WriteFile(flagValues.resultFileSourceTimestamp, []byte(strconv.FormatInt(unpackDetails.MostRecentFileTimestamp.Unix(), 10)), 0644); err != nil { + return err + } + + } else { + log.Printf("Unable to determine source timestamp of content in %s\n", flagValues.target) + } + } + if flagValues.prune { // Some container registry implementations, i.e. library/registry:2 will fail to // delete the image when there is no image digest given. Use image digest from the diff --git a/cmd/bundle/main_test.go b/cmd/bundle/main_test.go index a7f8c366dc..d1cc01ae83 100644 --- a/cmd/bundle/main_test.go +++ b/cmd/bundle/main_test.go @@ -9,25 +9,31 @@ import ( "fmt" "io" "log" + "net/http/httptest" + "net/url" "os" "path/filepath" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/shipwright-io/build/cmd/bundle" - "github.com/shipwright-io/build/pkg/image" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" containerreg "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "k8s.io/apimachinery/pkg/util/rand" + + "github.com/shipwright-io/build/pkg/bundle" + "github.com/shipwright-io/build/pkg/image" ) var _ = Describe("Bundle Loader", func() { const exampleImage = "ghcr.io/shipwright-io/sample-go/source-bundle:latest" - var run = func(args ...string) error { + run := func(args ...string) error { // discard log output log.SetOutput(io.Discard) @@ -40,7 +46,7 @@ var _ = Describe("Bundle Loader", func() { return Do(context.Background()) } - var withTempDir = func(f func(target string)) { + withTempDir := func(f func(target string)) { path, err := os.MkdirTemp(os.TempDir(), "bundle") Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(path) @@ -56,6 +62,24 @@ var _ = Describe("Bundle Loader", func() { f(file.Name()) } + withTempRegistry := func(f func(endpoint string)) { + logLogger := log.Logger{} + logLogger.SetOutput(GinkgoWriter) + + s := httptest.NewServer( + registry.New( + registry.Logger(&logLogger), + registry.WithReferrersSupport(true), + ), + ) + defer s.Close() + + u, err := url.Parse(s.URL) + Expect(err).ToNot(HaveOccurred()) + + f(u.Host) + } + filecontent := func(path string) string { data, err := os.ReadFile(path) Expect(err).ToNot(HaveOccurred()) @@ -234,4 +258,62 @@ var _ = Describe("Bundle Loader", func() { }) }) }) + + Context("Result file checks", func() { + tmpFile := func(dir string, name string, data []byte, timestamp time.Time) { + var path = filepath.Join(dir, name) + + Expect(os.WriteFile( + path, + data, + os.FileMode(0644), + )).To(Succeed()) + + Expect(os.Chtimes( + path, + timestamp, + timestamp, + )).To(Succeed()) + } + + // Creates a controlled reference image with one file called "file" with modification + // timestamp of Friday, February 13, 2009 11:31:30 PM (unix timestamp 1234567890) + withReferenceImage := func(f func(dig name.Digest)) { + withTempRegistry(func(endpoint string) { + withTempDir(func(target string) { + timestamp := time.Unix(1234567890, 0) + + ref, err := name.ParseReference(fmt.Sprintf("%s/namespace/image:tag", endpoint)) + Expect(err).ToNot(HaveOccurred()) + Expect(ref).ToNot(BeNil()) + + tmpFile(target, "file", []byte("foobar"), timestamp) + + dig, err := bundle.PackAndPush(ref, target) + Expect(err).ToNot(HaveOccurred()) + Expect(dig).ToNot(BeNil()) + + f(dig) + }) + }) + } + + It("should store source timestamp in result file", func() { + withTempDir(func(target string) { + withTempDir(func(result string) { + withReferenceImage(func(dig name.Digest) { + resultSourceTimestamp := filepath.Join(result, "source-timestamp") + + Expect(run( + "--image", dig.String(), + "--target", target, + "--result-file-source-timestamp", resultSourceTimestamp, + )).To(Succeed()) + + Expect(filecontent(resultSourceTimestamp)).To(Equal("1234567890")) + }) + }) + }) + }) + }) }) diff --git a/pkg/bundle/bundle.go b/pkg/bundle/bundle.go index 2a32ea78ba..384ba37225 100644 --- a/pkg/bundle/bundle.go +++ b/pkg/bundle/bundle.go @@ -26,6 +26,11 @@ import ( const shpIgnoreFilename = ".shpignore" +// UnpackDetails contains details about the files that were unpacked +type UnpackDetails struct { + MostRecentFileTimestamp *time.Time +} + // PackAndPush a local directory as-is into a container image. See // remote.Option for optional options to the image push to the registry, for // example to provide the appropriate access credentials. @@ -35,12 +40,12 @@ func PackAndPush(ref name.Reference, directory string, options ...remote.Option) return name.Digest{}, err } - image, err := mutate.AppendLayers(empty.Image, bundleLayer) + image, err := mutate.Time(empty.Image, time.Unix(0, 0)) if err != nil { return name.Digest{}, err } - image, err = mutate.Time(image, time.Unix(0, 0)) + image, err = mutate.AppendLayers(image, bundleLayer) if err != nil { return name.Digest{}, err } @@ -77,7 +82,7 @@ func PullAndUnpack(ref name.Reference, targetPath string, options ...remote.Opti rc := mutate.Extract(image) defer rc.Close() - if err = Unpack(rc, targetPath); err != nil { + if _, err = Unpack(rc, targetPath); err != nil { return nil, err } @@ -224,16 +229,17 @@ func Pack(directory string) (io.ReadCloser, error) { // Unpack reads a tar stream and writes the content into the local file system // with all files and directories. -func Unpack(in io.Reader, targetPath string) error { +func Unpack(in io.Reader, targetPath string) (*UnpackDetails, error) { + var details = UnpackDetails{} var tr = tar.NewReader(in) for { header, err := tr.Next() switch { case err == io.EOF: - return nil + return &details, nil case err != nil: - return err + return nil, err case header == nil: continue @@ -241,37 +247,46 @@ func Unpack(in io.Reader, targetPath string) error { var target = filepath.Join(targetPath, header.Name) if strings.Contains(target, "/../") { - return fmt.Errorf("targetPath validation failed, path contains unexpected special elements") + return nil, fmt.Errorf("targetPath validation failed, path contains unexpected special elements") } switch header.Typeflag { case tar.TypeDir: if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { - return err + return nil, err } case tar.TypeReg: // Edge case in which that tarball did not have a directory entry dir, _ := filepath.Split(target) if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { - return err + return nil, err } file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) if err != nil { - return err + return nil, err } if _, err := io.Copy(file, tr); err != nil { file.Close() - return err + return nil, err + } + + if err := file.Close(); err != nil { + return nil, err } - file.Close() - os.Chtimes(target, header.AccessTime, header.ModTime) + if err := os.Chtimes(target, header.AccessTime, header.ModTime); err != nil { + return nil, err + } + + if details.MostRecentFileTimestamp == nil || details.MostRecentFileTimestamp.Before(header.ModTime) { + details.MostRecentFileTimestamp = &header.ModTime + } default: - return fmt.Errorf("provided tarball contains unsupported file type, only directories and regular files are supported") + return nil, fmt.Errorf("provided tarball contains unsupported file type, only directories and regular files are supported") } } } diff --git a/pkg/bundle/bundle_test.go b/pkg/bundle/bundle_test.go index 870a3a9d4e..cd980ae5cb 100644 --- a/pkg/bundle/bundle_test.go +++ b/pkg/bundle/bundle_test.go @@ -54,7 +54,10 @@ var _ = Describe("Bundle", func() { Expect(err).ToNot(HaveOccurred()) Expect(r).ToNot(BeNil()) - Expect(Unpack(r, tempDir)).To(Succeed()) + details, err := Unpack(r, tempDir) + Expect(details).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(filepath.Join(tempDir, "README.md")).To(BeAnExistingFile()) Expect(filepath.Join(tempDir, ".someToolDir", "config.json")).ToNot(BeAnExistingFile()) Expect(filepath.Join(tempDir, "somefile")).To(BeAnExistingFile())