diff --git a/examples/overlapping-dir/some-content/some-file.txt b/examples/overlapping-dir/some-content/some-file.txt new file mode 100644 index 00000000..e69de29b diff --git a/examples/overlapping-dir/vendir.yml b/examples/overlapping-dir/vendir-content-path-overlap.yml similarity index 100% rename from examples/overlapping-dir/vendir.yml rename to examples/overlapping-dir/vendir-content-path-overlap.yml diff --git a/examples/overlapping-dir/vendir-directory-path-overlap-1.yml b/examples/overlapping-dir/vendir-directory-path-overlap-1.yml new file mode 100644 index 00000000..d9817dfd --- /dev/null +++ b/examples/overlapping-dir/vendir-directory-path-overlap-1.yml @@ -0,0 +1,13 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor + contents: + - path: dir1 + directory: + path: ./some-content + - path: vendor + contents: + - path: dir2 + directory: + path: ./some-content diff --git a/examples/overlapping-dir/vendir-directory-path-overlap-2.yml b/examples/overlapping-dir/vendir-directory-path-overlap-2.yml new file mode 100644 index 00000000..6cf29f51 --- /dev/null +++ b/examples/overlapping-dir/vendir-directory-path-overlap-2.yml @@ -0,0 +1,15 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: vendor + contents: + - path: dir1 + lazy: true + directory: + path: ./some-content + - path: vendor + contents: + - path: dir3 + lazy: true + directory: + path: ./some-content diff --git a/examples/overlapping-dir/vendir.lock.yml b/examples/overlapping-dir/vendir.lock.yml new file mode 100644 index 00000000..91db41d1 --- /dev/null +++ b/examples/overlapping-dir/vendir.lock.yml @@ -0,0 +1,13 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - configDigest: d0aef6c75d30608bf40883a6c52adcea0e8afc62c23513e0462a059390ea9b6b + directory: {} + path: dir1 + path: vendor +- contents: + - configDigest: 147cb489ce06d7fb463ed370dd6c0f41b237af66a1260fecfbd82e5c33bf6c4a + directory: {} + path: dir3 + path: vendor +kind: LockConfig diff --git a/examples/overlapping-dir/vendor/dir3/some-file.txt b/examples/overlapping-dir/vendor/dir3/some-file.txt new file mode 100644 index 00000000..e69de29b diff --git a/pkg/vendir/cmd/sync.go b/pkg/vendir/cmd/sync.go index b2d8d757..2584faa6 100644 --- a/pkg/vendir/cmd/sync.go +++ b/pkg/vendir/cmd/sync.go @@ -140,10 +140,8 @@ func (o *SyncOptions) Run() error { HelmBinary: os.Getenv("VENDIR_HELM_BINARY"), Cache: cache, Lazy: o.Lazy, - Partial: len(dirs) > 0, } newLockConfig := ctlconf.NewLockConfig() - for _, dirConf := range conf.Directories { // error safe to ignore, since lock file might not exist dirExistingLockConf, _ := existingLockConfig.FindDirectory(dirConf.Path) @@ -178,6 +176,40 @@ func (o *SyncOptions) Run() error { } } + // Clean old dirs from final output dir + // Iterate over directory paths + for _, dir := range newLockConfig.Directories { + // Walk through file system dirs and match with lock file config + err := filepath.WalkDir(dir.Path, func(filePath string, info os.DirEntry, err error) error { + if err != nil { + // dirs might be unreadable due to restrictive file permissions set in vendir.yml + // therefore ignore errors on traversing directories + return nil + } + for _, dir := range newLockConfig.Directories { + for _, content := range dir.Contents { + // if file system path found in config file then skip + if filepath.Join(dir.Path, content.Path) == filePath { + return filepath.SkipDir + } + // if file system path is a parent directory of config file then continue iterating + if strings.HasPrefix(filepath.Join(dir.Path, content.Path), filePath+"/") { + return nil + } + } + } + // if file system path not found in config file then delete + err = os.RemoveAll(filePath) + if err != nil { + return err + } + return filepath.SkipDir + }) + if err != nil { + return fmt.Errorf("Error while cleaning old directories: %s", err) + } + } + newLockConfigBs, err := newLockConfig.AsBytes() if err != nil { return err diff --git a/pkg/vendir/directory/directory.go b/pkg/vendir/directory/directory.go index 8f580cf5..35d9442e 100644 --- a/pkg/vendir/directory/directory.go +++ b/pkg/vendir/directory/directory.go @@ -42,7 +42,6 @@ type SyncOpts struct { HelmBinary string Cache ctlcache.Cache Lazy bool - Partial bool } func createConfigDigest(contents ctlconf.DirectoryContents) (string, error) { @@ -245,31 +244,22 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { return lockConfig, fmt.Errorf("Copying existing content to staging '%s': %s", d.opts.Path, err) } - // after everything else is done, ensure the inner dir's access perms are set - // chmod to the content's permission, fall back to the directory's - err = maybeChmod(stagingDstPath, contents.Permissions, d.opts.Permissions) - if err != nil { - return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err) - } - if lazySyncAddConfigDigest { lockDirContents.ConfigDigest = configDigest } lockConfig.Contents = append(lockConfig.Contents, lockDirContents) - if syncOpts.Partial { - err = stagingDir.PartialRepace(contents.Path, filepath.Join(d.opts.Path, contents.Path)) - if err != nil { - return lockConfig, err - } + err = stagingDir.CopyToFinalOutputDir(d.opts.Path, contents.Path) + if err != nil { + return lockConfig, err } - } - if !syncOpts.Partial { - err = stagingDir.Replace(d.opts.Path) + // after everything else is done, ensure the inner dir's access perms are set + // chmod to the content's permission, fall back to the directory's + err = maybeChmod(filepath.Join(d.opts.Path, contents.Path), contents.Permissions, d.opts.Permissions) if err != nil { - return lockConfig, err + return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err) } } diff --git a/pkg/vendir/directory/staging_dir.go b/pkg/vendir/directory/staging_dir.go index 1e82c590..ced57de3 100644 --- a/pkg/vendir/directory/staging_dir.go +++ b/pkg/vendir/directory/staging_dir.go @@ -147,15 +147,17 @@ func (d StagingDir) Replace(path string) error { } // Replaces single directory of final location dir with single directory of staging dir -func (d StagingDir) PartialRepace(contentPath string, directoryPath string) error { - err := d.prepareOutputDirectory(directoryPath) +func (d StagingDir) CopyToFinalOutputDir(directoryPath string, contentPath string) error { + stagingPath := filepath.Join(d.stagingDir, contentPath) + outputPath := filepath.Join(directoryPath, contentPath) + err := d.prepareOutputDirectory(outputPath) if err != nil { return err } - err = os.Rename(filepath.Join(d.stagingDir, contentPath), directoryPath) + err = os.Rename(stagingPath, outputPath) if err != nil { - return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", d.stagingDir, directoryPath, err) + return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", stagingPath, outputPath, err) } return nil diff --git a/test/e2e/overlapping_dir_err_test.go b/test/e2e/overlapping_content_path_err_test.go similarity index 69% rename from test/e2e/overlapping_dir_err_test.go rename to test/e2e/overlapping_content_path_err_test.go index 16a9fb97..b15cb44c 100644 --- a/test/e2e/overlapping_dir_err_test.go +++ b/test/e2e/overlapping_content_path_err_test.go @@ -8,13 +8,13 @@ import ( "testing" ) -func TestOverlappingDirErr(t *testing.T) { +func TestOverlappingContentPathErr(t *testing.T) { env := BuildEnv(t) vendir := Vendir{t, env.BinaryPath, Logger{}} path := "../../examples/overlapping-dir" - _, err := vendir.RunWithOpts([]string{"sync"}, RunOpts{Dir: path, AllowError: true}) + _, err := vendir.RunWithOpts([]string{"sync", "-f", "vendir-content-path-overlap.yml"}, RunOpts{Dir: path, AllowError: true}) if err == nil { t.Fatalf("Expected err") } diff --git a/test/e2e/overlapping_dir_path_test.go b/test/e2e/overlapping_dir_path_test.go new file mode 100644 index 00000000..3b7eff53 --- /dev/null +++ b/test/e2e/overlapping_dir_path_test.go @@ -0,0 +1,50 @@ +// Copyright 2020 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "github.com/stretchr/testify/require" + "os" + "path/filepath" + "testing" +) + +func TestOverlappingDirPath(t *testing.T) { + env := BuildEnv(t) + vendir := Vendir{t, env.BinaryPath, Logger{}} + + path := "../../examples/overlapping-dir" + + _, err := vendir.RunWithOpts([]string{"sync", "-f", "vendir-directory-path-overlap-1.yml"}, RunOpts{Dir: path, AllowError: true}) + require.NoError(t, err) + + expectedOutputDirs := []string{ + "dir1", + "dir2", + } + + filepath.WalkDir(filepath.Join(path, "vendor"), func(path string, d os.DirEntry, err error) error { + if d.Name() == "vendor" { + return nil + } + require.Contains(t, expectedOutputDirs, d.Name()) + return filepath.SkipAll + }) + + _, err = vendir.RunWithOpts([]string{"sync", "-f", "vendir-directory-path-overlap-2.yml"}, RunOpts{Dir: path, AllowError: true}) + require.NoError(t, err) + + expectedOutputDirs = []string{ + "dir1", + "dir3", + } + + filepath.WalkDir(filepath.Join(path, "vendor"), func(path string, d os.DirEntry, err error) error { + if d.Name() == "vendor" { + return nil + } + require.Contains(t, expectedOutputDirs, d.Name()) + return filepath.SkipAll + }) +}