fix: stop deletion of contents if directory paths overlap

examples/overlapping-dir/vendir-directory-path-overlap-1.yml

@@ -0,0 +1,13 @@
+apiVersion: vendir.k14s.io/v1alpha1
+kind: Config
+directories:
+  - path: vendor
+    contents:
+      - path: dir1
+        directory:
+          path: ./some-content
+  - path: vendor
+    contents:
+      - path: dir2
+        directory:
+          path: ./some-content

examples/overlapping-dir/vendir-directory-path-overlap-2.yml

@@ -0,0 +1,15 @@
+apiVersion: vendir.k14s.io/v1alpha1
+kind: Config
+directories:
+  - path: vendor
+    contents:
+      - path: dir1
+        lazy: true
+        directory:
+          path: ./some-content
+  - path: vendor
+    contents:
+      - path: dir3
+        lazy: true
+        directory:
+          path: ./some-content

examples/overlapping-dir/vendir.lock.yml

@@ -0,0 +1,13 @@
+apiVersion: vendir.k14s.io/v1alpha1
+directories:
+- contents:
+  - configDigest: d0aef6c75d30608bf40883a6c52adcea0e8afc62c23513e0462a059390ea9b6b
+    directory: {}
+    path: dir1
+  path: vendor
+- contents:
+  - configDigest: 147cb489ce06d7fb463ed370dd6c0f41b237af66a1260fecfbd82e5c33bf6c4a
+    directory: {}
+    path: dir3
+  path: vendor
+kind: LockConfig

pkg/vendir/cmd/sync.go

@@ -140,10 +140,8 @@ func (o *SyncOptions) Run() error {
 		HelmBinary:     os.Getenv("VENDIR_HELM_BINARY"),
 		Cache:          cache,
 		Lazy:           o.Lazy,
-		Partial:        len(dirs) > 0,
 	}
 	newLockConfig := ctlconf.NewLockConfig()
-
 	for _, dirConf := range conf.Directories {
 		// error safe to ignore, since lock file might not exist
 		dirExistingLockConf, _ := existingLockConfig.FindDirectory(dirConf.Path)
@@ -178,6 +176,40 @@ func (o *SyncOptions) Run() error {
 		}
 	}
 
+	// Clean old dirs from final output dir
+	// Iterate over directory paths
+	for _, dir := range newLockConfig.Directories {
+		// Walk through file system dirs and match with lock file config
+		err := filepath.WalkDir(dir.Path, func(filePath string, info os.DirEntry, err error) error {
+			if err != nil {
+				// dirs might be unreadable due to restrictive file permissions set in vendir.yml
+				// therefore ignore errors on traversing directories
+				return nil
+			}
+			for _, dir := range newLockConfig.Directories {
+				for _, content := range dir.Contents {
+					// if file system path found in config file then skip
+					if filepath.Join(dir.Path, content.Path) == filePath {
+						return filepath.SkipDir
+					}
+					// if file system path is a parent directory of config file then continue iterating
+					if strings.HasPrefix(filepath.Join(dir.Path, content.Path), filePath+"/") {
+						return nil
+					}
+				}
+			}
+			// if file system path not found in config file then delete
+			err = os.RemoveAll(filePath)
+			if err != nil {
+				return err
+			}
+			return filepath.SkipDir
+		})
+		if err != nil {
+			return fmt.Errorf("Error while cleaning old directories: %s", err)
+		}
+	}
+
 	newLockConfigBs, err := newLockConfig.AsBytes()
 	if err != nil {
 		return err

pkg/vendir/directory/directory.go

@@ -42,7 +42,6 @@ type SyncOpts struct {
 	HelmBinary     string
 	Cache          ctlcache.Cache
 	Lazy           bool
-	Partial        bool
 }
 
 func createConfigDigest(contents ctlconf.DirectoryContents) (string, error) {
@@ -245,31 +244,22 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			return lockConfig, fmt.Errorf("Copying existing content to staging '%s': %s", d.opts.Path, err)
 		}
 
-		// after everything else is done, ensure the inner dir's access perms are set
-		// chmod to the content's permission, fall back to the directory's
-		err = maybeChmod(stagingDstPath, contents.Permissions, d.opts.Permissions)
-		if err != nil {
-			return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err)
-		}
-
 		if lazySyncAddConfigDigest {
 			lockDirContents.ConfigDigest = configDigest
 		}
 
 		lockConfig.Contents = append(lockConfig.Contents, lockDirContents)
 
-		if syncOpts.Partial {
-			err = stagingDir.PartialRepace(contents.Path, filepath.Join(d.opts.Path, contents.Path))
-			if err != nil {
-				return lockConfig, err
-			}
+		err = stagingDir.CopyToFinalOutputDir(d.opts.Path, contents.Path)
+		if err != nil {
+			return lockConfig, err
 		}
-	}
 
-	if !syncOpts.Partial {
-		err = stagingDir.Replace(d.opts.Path)
+		// after everything else is done, ensure the inner dir's access perms are set
+		// chmod to the content's permission, fall back to the directory's
+		err = maybeChmod(filepath.Join(d.opts.Path, contents.Path), contents.Permissions, d.opts.Permissions)
 		if err != nil {
-			return lockConfig, err
+			return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err)
 		}
 	}
 

pkg/vendir/directory/staging_dir.go

@@ -147,15 +147,17 @@ func (d StagingDir) Replace(path string) error {
 }
 
 // Replaces single directory of final location dir with single directory of staging dir
-func (d StagingDir) PartialRepace(contentPath string, directoryPath string) error {
-	err := d.prepareOutputDirectory(directoryPath)
+func (d StagingDir) CopyToFinalOutputDir(directoryPath string, contentPath string) error {
+	stagingPath := filepath.Join(d.stagingDir, contentPath)
+	outputPath := filepath.Join(directoryPath, contentPath)
+	err := d.prepareOutputDirectory(outputPath)
 	if err != nil {
 		return err
 	}
 
-	err = os.Rename(filepath.Join(d.stagingDir, contentPath), directoryPath)
+	err = os.Rename(stagingPath, outputPath)
 	if err != nil {
-		return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", d.stagingDir, directoryPath, err)
+		return fmt.Errorf("Moving staging directory '%s' to final location '%s': %s", stagingPath, outputPath, err)
 	}
 
 	return nil

test/e2e/overlapping_dir_err_test.go → test/e2e/overlapping_content_path_err_test.go

@@ -8,13 +8,13 @@ import (
 	"testing"
 )
 
-func TestOverlappingDirErr(t *testing.T) {
+func TestOverlappingContentPathErr(t *testing.T) {
 	env := BuildEnv(t)
 	vendir := Vendir{t, env.BinaryPath, Logger{}}
 
 	path := "../../examples/overlapping-dir"
 
-	_, err := vendir.RunWithOpts([]string{"sync"}, RunOpts{Dir: path, AllowError: true})
+	_, err := vendir.RunWithOpts([]string{"sync", "-f", "vendir-content-path-overlap.yml"}, RunOpts{Dir: path, AllowError: true})
 	if err == nil {
 		t.Fatalf("Expected err")
 	}

test/e2e/overlapping_dir_path_test.go

@@ -0,0 +1,50 @@
+// Copyright 2020 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"github.com/stretchr/testify/require"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestOverlappingDirPath(t *testing.T) {
+	env := BuildEnv(t)
+	vendir := Vendir{t, env.BinaryPath, Logger{}}
+
+	path := "../../examples/overlapping-dir"
+
+	_, err := vendir.RunWithOpts([]string{"sync", "-f", "vendir-directory-path-overlap-1.yml"}, RunOpts{Dir: path, AllowError: true})
+	require.NoError(t, err)
+
+	expectedOutputDirs := []string{
+		"dir1",
+		"dir2",
+	}
+
+	filepath.WalkDir(filepath.Join(path, "vendor"), func(path string, d os.DirEntry, err error) error {
+		if d.Name() == "vendor" {
+			return nil
+		}
+		require.Contains(t, expectedOutputDirs, d.Name())
+		return filepath.SkipAll
+	})
+
+	_, err = vendir.RunWithOpts([]string{"sync", "-f", "vendir-directory-path-overlap-2.yml"}, RunOpts{Dir: path, AllowError: true})
+	require.NoError(t, err)
+
+	expectedOutputDirs = []string{
+		"dir1",
+		"dir3",
+	}
+
+	filepath.WalkDir(filepath.Join(path, "vendor"), func(path string, d os.DirEntry, err error) error {
+		if d.Name() == "vendor" {
+			return nil
+		}
+		require.Contains(t, expectedOutputDirs, d.Name())
+		return filepath.SkipAll
+	})
+}