From 005988caf59caa5354148c6c9c049558ba31b48d Mon Sep 17 00:00:00 2001 From: Michael McCracken Date: Fri, 10 May 2024 09:22:37 -0700 Subject: [PATCH] feat: print hashes for imports in yaml format (#608) For import types that will be checked, save the calculated hash of the downloaded file and print them together in the format of a yaml imports section for easy copy and pasting to add hashes to a file. Because we only the parse file contents after the substitution placeholders are replaced, this isn't a direct copy and paste replacement, but it is an improvement, and it should be easy enough to tell which expanded paths come from what input. Signed-off-by: Michael McCracken --- pkg/stacker/base.go | 2 +- pkg/stacker/import.go | 63 ++++++++++++++++++++++++++---------------- pkg/stacker/network.go | 16 +++++++---- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/pkg/stacker/base.go b/pkg/stacker/base.go index 2698559f..99a715a9 100644 --- a/pkg/stacker/base.go +++ b/pkg/stacker/base.go @@ -40,7 +40,7 @@ func GetBase(o BaseLayerOpts) error { return err } - _, err := acquireUrl(o.Config, o.Storage, o.Layer.From.Url, cacheDir, "", "", nil, -1, -1, o.Progress) + _, _, err := acquireUrl(o.Config, o.Storage, o.Layer.From.Url, cacheDir, "", "", nil, -1, -1, o.Progress) return err /* now we can do all the containers/image types */ case types.OCILayer: diff --git a/pkg/stacker/import.go b/pkg/stacker/import.go index 965a17a7..b00f95f4 100644 --- a/pkg/stacker/import.go +++ b/pkg/stacker/import.go @@ -72,22 +72,26 @@ func filesDiffer(p1 string, info1 os.FileInfo, p2 string, info2 os.FileInfo) (bo return !eq, nil } -func verifyImportFileHash(imp string, hash string) error { - if len(hash) == 0 { - return nil - } +// check that the file's hash matches the given hash. +// If the given hash is "", that is treated as a match. +// always return the actual hash. +func verifyImportFileHash(imp string, hash string) (string, error) { actualHash, err := lib.HashFile(imp, false) if err != nil { - return err + return actualHash, err } actualHash = strings.TrimPrefix(actualHash, "sha256:") + if len(hash) == 0 { + return actualHash, nil + } + if actualHash != strings.ToLower(hash) { - return errors.Errorf("The requested hash of %s import is different than the actual hash: %s != %s", + return actualHash, errors.Errorf("The requested hash of %s import is different than the actual hash: %s != %s", imp, hash, actualHash) } - return nil + return actualHash, nil } func importFile(imp string, cacheDir string, hash string, idest string, mode *fs.FileMode, uid, gid int) (string, error) { @@ -97,7 +101,7 @@ func importFile(imp string, cacheDir string, hash string, idest string, mode *fs } if !e1.IsDir() { - err := verifyImportFileHash(imp, hash) + _, err := verifyImportFileHash(imp, hash) if err != nil { return "", err } @@ -245,22 +249,25 @@ func validateHash(hash string) error { return nil } +// downloads or copies import url depending on scheme, and returns the path and +// hash of the downloaded file func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache string, expectedHash string, idest string, mode *fs.FileMode, uid, gid int, progress bool, -) (string, error) { +) (string, string, error) { url, err := types.NewDockerishUrl(i) if err != nil { - return "", err + return "", "", err } // validate the given hash if err = validateHash(expectedHash); err != nil { - return "", err + return "", "", err } // It's just a path, let's copy it to .stacker. if url.Scheme == "" { - return importFile(i, cache, expectedHash, idest, mode, uid, gid) + path, err := importFile(i, cache, expectedHash, idest, mode, uid, gid) + return path, "", err } else if url.Scheme == "http" || url.Scheme == "https" { // otherwise, we need to download it // first verify the hashes @@ -273,10 +280,11 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st log.Debugf("Remote file: hash: %s length: %s", remoteHash, remoteSize) // verify if the given hash from stackerfile matches the remote one. if len(expectedHash) > 0 && len(remoteHash) > 0 && strings.ToLower(expectedHash) != remoteHash { - return "", errors.Errorf("The requested hash of %s import is different than the actual hash: %s != %s", + return "", "", errors.Errorf("The requested hash of %s import is different than the actual hash: %s != %s", i, expectedHash, remoteHash) } - return Download(cache, i, progress, expectedHash, remoteHash, remoteSize, idest, mode, uid, gid) + path, err := Download(cache, i, progress, expectedHash, remoteHash, remoteSize, idest, mode, uid, gid) + return path, remoteHash, err } else if url.Scheme == "stacker" { // we always Grab() things from stacker://, because we need to // mount the container's rootfs to get them and don't @@ -285,23 +293,19 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st p := path.Join(cache, path.Base(url.Path)) snap, cleanup, err := storage.TemporaryWritableSnapshot(url.Host) if err != nil { - return "", err + return "", "", err } defer cleanup() err = Grab(c, storage, snap, url.Path, cache, idest, mode, uid, gid) if err != nil { - return "", err + return "", "", err } - err = verifyImportFileHash(p, expectedHash) - if err != nil { - return "", err - } - - return p, nil + // return "" as the hash, it is not checked + return p, "", nil } - return "", errors.Errorf("unsupported url scheme %s", i) + return "", "", errors.Errorf("unsupported url scheme %s", i) } func CleanImportsDir(c types.StackerConfig, name string, imports types.Imports, cache *BuildCache) error { @@ -365,6 +369,7 @@ func Import(c types.StackerConfig, storage types.Storage, name string, imports t return errors.Wrapf(err, "couldn't read existing directory") } + importHashes := map[string]string{} for _, i := range imports { cache := dir @@ -387,11 +392,16 @@ func Import(c types.StackerConfig, storage types.Storage, name string, imports t cache = tmpdir } - name, err := acquireUrl(c, storage, i.Path, cache, i.Hash, i.Dest, i.Mode, i.Uid, i.Gid, progress) + name, downloadedFileHash, err := acquireUrl(c, storage, i.Path, cache, i.Hash, i.Dest, i.Mode, i.Uid, i.Gid, progress) if err != nil { return err } + // "" is returned for local files, ignore they won't be checked anyway + if downloadedFileHash != "" { + importHashes[i.Path] = downloadedFileHash + } + for i, ext := range existing { if ext.Name() == path.Base(name) { existing = append(existing[:i], existing[i+1:]...) @@ -408,5 +418,10 @@ func Import(c types.StackerConfig, storage types.Storage, name string, imports t } } + log.Infof("imported file hashes (after substitutions):") + for path, hash := range importHashes { + log.Infof(" - path: %q\n hash: %q", path, hash) + } + return nil } diff --git a/pkg/stacker/network.go b/pkg/stacker/network.go index 3924d378..c25f1207 100644 --- a/pkg/stacker/network.go +++ b/pkg/stacker/network.go @@ -98,17 +98,21 @@ func Download(cacheDir string, url string, progress bool, expectedHash, remoteHa if err != nil { return "", err } - if expectedHash != "" { - log.Infof("Checking shasum of downloaded file") - downloadHash, err := lib.HashFile(name, false) - if err != nil { - return "", err - } + downloadHash, err := lib.HashFile(name, false) + if err != nil { + return "", err + } + if expectedHash != "" { + log.Infof("Checking shasum of downloaded file") downloadHash = strings.TrimPrefix(downloadHash, "sha256:") log.Debugf("Downloaded file hash: %s", downloadHash) + if downloadHash != remoteHash { + log.Warnf("Downloaded file hash %q does not match hash from HTTP header %q", downloadHash, remoteHash) + } + if expectedHash != downloadHash { os.RemoveAll(name) return "", errors.Errorf("Downloaded file hash does not match. Expected: %s Actual: %s", expectedHash, downloadHash)