diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index d615ffd186f..06b387f9f54 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -205,24 +205,35 @@ const ( ActionBuildDisk = "build-disk" ) -// GetDefaultSystemEcludes returns a list of transient paths +// GetDefaultSystemExcludes returns a list of paths // that are commonly present in an Elemental based running system. -// Those paths are not needed or wanted in order to replicate the -// root-tree as they are generated at runtime. -func GetDefaultSystemExcludes(rootDir string) []string { +// Those paths are not needed or wanted in order to replicate the root-tree. +func GetDefaultSystemExcludes() []string { return []string{ - filepath.Join(rootDir, "/.snapshots"), - filepath.Join(rootDir, "/mnt"), - filepath.Join(rootDir, "/proc"), - filepath.Join(rootDir, "/sys"), - filepath.Join(rootDir, "/dev"), - filepath.Join(rootDir, "/tmp"), - filepath.Join(rootDir, "/run"), - filepath.Join(rootDir, "/host"), - filepath.Join(rootDir, "/etc/resolv.conf"), + ".snapshots", + "mnt/*", + "proc/*", + "sys/*", + "dev/*", + "tmp/*", + "run/*", + "host", + "etc/resolv.conf", } } +// GetDefaultSystemExcludes returns a list of transient paths +// that are commonly present in an Elemental based running system. +// Paths are rooted to the given rootDir. Those paths are not +// needed or wanted in order to replicate the root-tree. +func GetDefaultSystemRootedExcludes(rootDir string) []string { + var list []string + for _, path := range GetDefaultSystemExcludes() { + list = append(list, filepath.Join(rootDir, path)) + } + return list +} + func GetKernelPatterns() []string { return []string{ "/boot/uImage*", diff --git a/pkg/elemental/elemental.go b/pkg/elemental/elemental.go index cef5628fe78..d229d7826ff 100644 --- a/pkg/elemental/elemental.go +++ b/pkg/elemental/elemental.go @@ -369,13 +369,14 @@ func CreateImageFromTree(c types.Config, img *types.Image, rootDir string, prelo c.Logger.Warnf("failed SELinux labelling at %s: %v", rootDir, err) } - err = utils.CreateSquashFS(c.Runner, c.Logger, rootDir, img.File, c.SquashFsCompressionConfig) + excludes := cnst.GetDefaultSystemExcludes() + err = utils.CreateSquashFS(c.Runner, c.Logger, rootDir, img.File, c.SquashFsCompressionConfig, excludes...) if err != nil { c.Logger.Errorf("failed creating squashfs image for %s: %v", img.File, err) return err } } else { - excludes := cnst.GetDefaultSystemExcludes(rootDir) + excludes := cnst.GetDefaultSystemRootedExcludes(rootDir) err = CreateFileSystemImage(c, img, rootDir, preload, excludes...) if err != nil { c.Logger.Errorf("failed creating filesystem image: %v", err) @@ -585,7 +586,7 @@ func DumpSource( } imgSrc.SetDigest(digest) } else if imgSrc.IsDir() { - excludes := cnst.GetDefaultSystemExcludes(imgSrc.Value()) + excludes := cnst.GetDefaultSystemRootedExcludes(imgSrc.Value()) err = syncFunc(c.Logger, c.Runner, c.Fs, imgSrc.Value(), target, excludes...) if err != nil { return err diff --git a/pkg/utils/common.go b/pkg/utils/common.go index b42c58bced9..b597b04a233 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -279,17 +279,25 @@ func CosignVerify(fs types.FS, runner types.Runner, image string, publicKey stri } // CreateSquashFS creates a squash file at destination from a source, with options -// TODO: Check validity of source maybe? -func CreateSquashFS(runner types.Runner, logger types.Logger, source string, destination string, options []string) error { +func CreateSquashFS(runner types.Runner, logger types.Logger, source string, destination string, options []string, excludes ...string) error { // create args args := []string{source, destination} // append options passed to args in order to have the correct order // protect against options passed together in the same string , i.e. "-x add" instead of "-x", "add" var optionsExpanded []string for _, op := range options { - optionsExpanded = append(optionsExpanded, strings.Split(op, " ")...) + opExpanded := strings.Split(op, " ") + if opExpanded[0] == "-e" { + logger.Warnf("Ignoring option '%s', exclude directories must be passed as excludes argument", op) + continue + } + optionsExpanded = append(optionsExpanded, opExpanded...) } args = append(args, optionsExpanded...) + if len(excludes) >= 0 { + excludesOpt := append([]string{"-wildcards", "-e"}, excludes...) + args = append(args, excludesOpt...) + } out, err := runner.Run("mksquashfs", args...) if err != nil { logger.Debugf("Error running squashfs creation, stdout: %s", out) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index c64f497349a..5af5a604fcb 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -452,13 +452,18 @@ var _ = Describe("Utils", Label("utils"), func() { Expect(utils.CreateDirStructure(fs, "/my/root")).NotTo(BeNil()) }) }) - Describe("SyncData", Label("SyncData"), func() { - It("Copies all files from source to target", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") + Describe("Rsync tests", Label("rsync"), func() { + var sourceDir, destDir string + var err error + + BeforeEach(func() { + sourceDir, err = utils.TempDir(fs, "", "elementalsource") Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") + destDir, err = utils.TempDir(fs, "", "elementaltarget") Expect(err).ShouldNot(HaveOccurred()) + }) + It("Copies all files from source to target", func() { for i := 0; i < 5; i++ { _, _ = utils.TempFile(fs, sourceDir, "file*") } @@ -479,11 +484,6 @@ var _ = Describe("Utils", Label("utils"), func() { }) It("Copies all files from source to target respecting excludes", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") - Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") - Expect(err).ShouldNot(HaveOccurred()) - utils.MkdirAll(fs, filepath.Join(sourceDir, "host"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) @@ -522,11 +522,6 @@ var _ = Describe("Utils", Label("utils"), func() { }) It("Copies all files from source to target respecting excludes with '/' prefix", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") - Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") - Expect(err).ShouldNot(HaveOccurred()) - utils.MkdirAll(fs, filepath.Join(sourceDir, "host"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) @@ -536,16 +531,14 @@ var _ = Describe("Utils", Label("utils"), func() { filesDest, err := fs.ReadDir(destDir) Expect(err).To(BeNil()) - destNames := getNamesFromListFiles(filesDest) filesSource, err := fs.ReadDir(sourceDir) Expect(err).To(BeNil()) - - SourceNames := getNamesFromListFiles(filesSource) + sourceNames := getNamesFromListFiles(filesSource) // Shouldn't be the same - Expect(destNames).ToNot(Equal(SourceNames)) + Expect(destNames).ToNot(Equal(sourceNames)) Expect(utils.Exists(fs, filepath.Join(destDir, "var", "run"))).To(BeTrue()) Expect(utils.Exists(fs, filepath.Join(destDir, "tmp", "host"))).To(BeTrue()) @@ -553,23 +546,50 @@ var _ = Describe("Utils", Label("utils"), func() { Expect(utils.Exists(fs, filepath.Join(destDir, "run"))).To(BeFalse()) }) + It("Copies all files from source to target respecting excludes with wildcards", func() { + utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) + utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) + Expect(fs.WriteFile(filepath.Join(sourceDir, "run", "testfile"), []byte{}, constants.DirPerm)).To(Succeed()) + + Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "/run/*")).To(BeNil()) + + Expect(utils.Exists(fs, filepath.Join(destDir, "var", "run"))).To(BeTrue()) + Expect(utils.Exists(fs, filepath.Join(destDir, "run"))).To(BeTrue()) + Expect(utils.Exists(fs, filepath.Join(destDir, "run", "testfile"))).To(BeFalse()) + }) + + It("Mirrors all files from source to destination deleting pre-existing files in destination if needed", func() { + utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) + utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) + Expect(fs.WriteFile(filepath.Join(sourceDir, "run", "testfile"), []byte{}, constants.DirPerm)).To(Succeed()) + Expect(fs.WriteFile(filepath.Join(destDir, "testfile"), []byte{}, constants.DirPerm)).To(Succeed()) + + Expect(utils.MirrorData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil()) + + filesDest, err := fs.ReadDir(destDir) + Expect(err).To(BeNil()) + destNames := getNamesFromListFiles(filesDest) + + filesSource, err := fs.ReadDir(sourceDir) + Expect(err).To(BeNil()) + sourceNames := getNamesFromListFiles(filesSource) + + // Should be the same + Expect(destNames).To(Equal(sourceNames)) + + // pre-exising file in destination deleted if this is not part of source + Expect(utils.Exists(fs, filepath.Join(destDir, "testfile"))).To(BeFalse()) + }) + It("should not fail if dirs are empty", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") - Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") - Expect(err).ShouldNot(HaveOccurred()) Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil()) }) It("should fail if destination does not exist", func() { - sourceDir, err := os.MkdirTemp("", "elemental") - Expect(err).To(BeNil()) - defer os.RemoveAll(sourceDir) + fs.RemoveAll(destDir) Expect(utils.SyncData(logger, realRunner, nil, sourceDir, "/welp")).NotTo(BeNil()) }) It("should fail if source does not exist", func() { - destDir, err := os.MkdirTemp("", "elemental") - Expect(err).To(BeNil()) - defer os.RemoveAll(destDir) + fs.RemoveAll(sourceDir) Expect(utils.SyncData(logger, realRunner, nil, "/welp", destDir)).NotTo(BeNil()) }) }) @@ -948,6 +968,30 @@ var _ = Describe("Utils", Label("utils"), func() { })).To(BeNil()) Expect(err).ToNot(HaveOccurred()) }) + It("ignores any '-e' option", func() { + args := append(constants.GetDefaultSquashfsCompressionOptions(), "-e /some/path") + err := utils.CreateSquashFS(runner, logger, "source", "dest", args) + cmd := []string{"mksquashfs", "source", "dest"} + cmd = append(cmd, constants.GetDefaultSquashfsCompressionOptions()...) + Expect(runner.IncludesCmds([][]string{ + cmd, + })).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + }) + It("excludes given paths", func() { + + err := utils.CreateSquashFS( + runner, logger, "source", "dest", constants.GetDefaultSquashfsCompressionOptions(), + "some/path", "another/path", + ) + cmd := []string{"mksquashfs", "source", "dest"} + cmd = append(cmd, constants.GetDefaultSquashfsCompressionOptions()...) + cmd = append(cmd, "-wildcards", "-e", "some/path", "another/path") + Expect(runner.IncludesCmds([][]string{ + cmd, + })).To(Succeed()) + Expect(err).ToNot(HaveOccurred()) + }) It("returns an error if it fails", func() { runner.ReturnError = errors.New("error") err := utils.CreateSquashFS(runner, logger, "source", "dest", []string{})