Added lazy sync functionality

go.mod

@@ -27,6 +27,8 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
+require gopkg.in/yaml.v3 v3.0.1
+
 require (
 	cloud.google.com/go/compute v1.18.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
@@ -106,7 +108,6 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.29.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c // indirect
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.70.1 // indirect

pkg/vendir/cmd/sync.go

@@ -30,6 +30,7 @@ type SyncOptions struct {
 
 	Directories []string
 	Locked      bool
+	Eager       bool
 
 	Chdir                       string
 	AllowAllSymlinkDestinations bool
@@ -50,6 +51,7 @@ func NewSyncCmd(o *SyncOptions) *cobra.Command {
 
 	cmd.Flags().StringSliceVarP(&o.Directories, "directory", "d", nil, "Sync specific directory (format: dir/sub-dir[=local-dir])")
 	cmd.Flags().BoolVarP(&o.Locked, "locked", "l", false, "Consult lock file to pull exact references (e.g. use git sha instead of branch name)")
+	cmd.Flags().BoolVar(&o.Eager, "eager", false, "Ignore lazy setting in vendir yml and eagerly fetch all remote content")
 
 	cmd.Flags().StringVar(&o.Chdir, "chdir", "", "Set current directory for process")
 	cmd.Flags().BoolVar(&o.AllowAllSymlinkDestinations, "dangerous-allow-all-symlink-destinations", false, "Symlinks to all destinations are allowed")
@@ -95,14 +97,18 @@ func (o *SyncOptions) Run() error {
 		o.ui.PrintBlock(configBs)
 	}
 
-	// If syncing against a lock file, apply lock information
-	// on top of existing config
-	if o.Locked {
-		existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
+	var existingLockConfig ctlconf.LockConfig
+
+	if ctlconf.LockFileExists(o.LockFile) {
+		existingLockConfig, err = ctlconf.NewLockConfigFromFile(o.LockFile)
 		if err != nil {
 			return err
 		}
+	}
 
+	// If syncing against a lock file, apply lock information
+	// on top of existing config
+	if o.Locked {
 		err = conf.Lock(existingLockConfig)
 		if err != nil {
 			return err
@@ -130,11 +136,12 @@ func (o *SyncOptions) Run() error {
 		GithubAPIToken: os.Getenv("VENDIR_GITHUB_API_TOKEN"),
 		HelmBinary:     os.Getenv("VENDIR_HELM_BINARY"),
 		Cache:          cache,
+		Eager:          o.Eager,
 	}
 	newLockConfig := ctlconf.NewLockConfig()
 
 	for _, dirConf := range conf.Directories {
-		dirLockConf, err := ctldir.NewDirectory(dirConf, o.ui).Sync(syncOpts)
+		dirLockConf, err := ctldir.NewDirectory(dirConf, existingLockConfig, o.ui).Sync(syncOpts)
 		if err != nil {
 			return fmt.Errorf("Syncing directory '%s': %s", dirConf.Path, err)
 		}
@@ -150,11 +157,6 @@ func (o *SyncOptions) Run() error {
 
 	// Update only selected directories in lock file
 	if len(dirs) > 0 {
-		existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
-		if err != nil {
-			return err
-		}
-
 		err = existingLockConfig.Merge(newLockConfig)
 		if err != nil {
 			return err

pkg/vendir/config/config.go

@@ -183,6 +183,7 @@ func (c Config) UseDirectory(path, dirPath string) error {
 				ExcludePaths: con.ExcludePaths,
 				IgnorePaths:  con.IgnorePaths,
 				LegalPaths:   con.LegalPaths,
+				Lazy:         con.Lazy,
 			}
 			dir.Contents[j] = newCon
 			c.Directories[i] = dir

pkg/vendir/config/directory.go

@@ -34,6 +34,7 @@ type Directory struct {
 
 type DirectoryContents struct {
 	Path string `json:"path"`
+	Lazy bool   `json:"lazy,omitempty"`
 
 	Git           *DirectoryContentsGit           `json:"git,omitempty"`
 	Hg            *DirectoryContentsHg            `json:"hg,omitempty"`

pkg/vendir/config/lock_config.go

@@ -28,6 +28,13 @@ func NewLockConfig() LockConfig {
 	}
 }
 
+func LockFileExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		return false
+	}
+	return true
+}
+
 func NewLockConfigFromFile(path string) (LockConfig, error) {
 	bs, err := os.ReadFile(path)
 	if err != nil {

pkg/vendir/config/lock_directory.go

@@ -10,6 +10,7 @@ type LockDirectory struct {
 
 type LockDirectoryContents struct {
 	Path string `json:"path"`
+	Hash string `json:"hash"`
 
 	Git           *LockDirectoryContentsGit           `json:"git,omitempty"`
 	Hg            *LockDirectoryContentsHg            `json:"hg,omitempty"`

pkg/vendir/directory/directory.go

@@ -4,7 +4,10 @@
 package directory
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"os"
 	"path/filepath"
 
@@ -24,19 +27,43 @@ import (
 )
 
 type Directory struct {
-	opts ctlconf.Directory
-	ui   ui.UI
+	opts       ctlconf.Directory
+	lockConfig ctlconf.LockConfig
+	ui         ui.UI
 }
 
-func NewDirectory(opts ctlconf.Directory, ui ui.UI) *Directory {
-	return &Directory{opts, ui}
+func NewDirectory(opts ctlconf.Directory, lockConfig ctlconf.LockConfig, ui ui.UI) *Directory {
+	return &Directory{opts, lockConfig, ui}
 }
 
 type SyncOpts struct {
 	RefFetcher     ctlfetch.RefFetcher
 	GithubAPIToken string
 	HelmBinary     string
 	Cache          ctlcache.Cache
+	Eager          bool
+}
+
+func createContentHash(contents ctlconf.DirectoryContents) (string, error) {
+	yaml, err := yaml.Marshal(contents)
+	if err != nil {
+		return "", fmt.Errorf("error during hash creation for path '%s': %s", contents.Path, err)
+	}
+	hash := sha256.Sum256(yaml)
+	hashStr := hex.EncodeToString(hash[:])
+	return hashStr, nil
+}
+
+func (d *Directory) configUnchanged(path string, contents ctlconf.DirectoryContents) (bool, error) {
+	hash, err := createContentHash(contents)
+	if err != nil {
+		return false, err
+	}
+	lockContents, _ := d.lockConfig.FindContents(path, contents.Path)
+	if hash == lockContents.Hash {
+		return true, nil
+	}
+	return false, nil
 }
 
 func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
@@ -56,8 +83,29 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 		if err != nil {
 			return lockConfig, err
 		}
+		// adds hash to lockfile of current content
+		hash, err := createContentHash(contents)
+		if err != nil {
+			return lockConfig, err
+		}
 
-		lockDirContents := ctlconf.LockDirectoryContents{Path: contents.Path}
+		lockDirContents := ctlconf.LockDirectoryContents{
+			Path: contents.Path,
+			Hash: hash,
+		}
+		// check if vendir config has changed. If not, skip syncing
+		unchanged := false
+		var oldLock ctlconf.LockDirectoryContents
+		if contents.Lazy && syncOpts.Eager == false {
+			unchanged, err = d.configUnchanged(d.opts.Path, contents)
+			if err != nil {
+				return lockConfig, err
+			}
+			if unchanged {
+				d.ui.PrintLinef("Skipping fetch since config has not changed: %s%s", d.opts.Path, contents.Path)
+				oldLock, _ = d.lockConfig.FindContents(d.opts.Path, contents.Path)
+			}
+		}
 
 		skipFileFilter := false
 		skipNewRootPath := false
@@ -68,13 +116,16 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 
 			d.ui.PrintLinef("Fetching: %s + %s (git from %s)", d.opts.Path, contents.Path, gitSync.Desc())
 
-			lock, err := gitSync.Sync(stagingDstPath, stagingDir.TempArea())
-			if err != nil {
-				return lockConfig, fmt.Errorf("Syncing directory '%s' with git contents: %s", contents.Path, err)
+			if unchanged == false {
+				lock, err := gitSync.Sync(stagingDstPath, stagingDir.TempArea())
+				if err != nil {
+					return lockConfig, fmt.Errorf("Syncing directory '%s' with git contents: %s", contents.Path, err)
+				}
+				lockDirContents.Git = &lock
+			} else {
+				lockDirContents.Git = oldLock.Git
 			}
 
-			lockDirContents.Git = &lock
-
 		case contents.Hg != nil:
 			hgSync := ctlhg.NewSync(*contents.Hg, NewInfoLog(d.ui), syncOpts.RefFetcher)
 
@@ -143,13 +194,16 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			d.ui.PrintLinef("Fetching: %s + %s (helm chart from %s)",
 				d.opts.Path, contents.Path, helmChartSync.Desc())
 
-			lock, err := helmChartSync.Sync(stagingDstPath, stagingDir.TempArea())
-			if err != nil {
-				return lockConfig, fmt.Errorf("Syncing directory '%s' with helm chart contents: %s", contents.Path, err)
+			if unchanged == false {
+				lock, err := helmChartSync.Sync(stagingDstPath, stagingDir.TempArea())
+				if err != nil {
+					return lockConfig, fmt.Errorf("Syncing directory '%s' with helm chart contents: %s", contents.Path, err)
+				}
+				lockDirContents.HelmChart = &lock
+			} else {
+				lockDirContents.HelmChart = oldLock.HelmChart
 			}
 
-			lockDirContents.HelmChart = &lock
-
 		case contents.Manual != nil:
 			d.ui.PrintLinef("Fetching: %s + %s (manual)", d.opts.Path, contents.Path)
 
@@ -188,31 +242,33 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 			return lockConfig, fmt.Errorf("Unknown contents type for directory '%s'", contents.Path)
 		}
 
-		if !skipFileFilter {
-			err = FileFilter{contents}.Apply(stagingDstPath)
-			if err != nil {
-				return lockConfig, fmt.Errorf("Filtering paths in directory '%s': %s", contents.Path, err)
+		if !unchanged {
+			if !skipFileFilter {
+				err = FileFilter{contents}.Apply(stagingDstPath)
+				if err != nil {
+					return lockConfig, fmt.Errorf("Filtering paths in directory '%s': %s", contents.Path, err)
+				}
 			}
-		}
 
-		if !skipNewRootPath && len(contents.NewRootPath) > 0 {
-			err = NewSubPath(contents.NewRootPath).Extract(stagingDstPath, stagingDstPath, stagingDir.TempArea())
-			if err != nil {
-				return lockConfig, fmt.Errorf("Changing to new root path '%s': %s", contents.Path, err)
+			if !skipNewRootPath && len(contents.NewRootPath) > 0 {
+				err = NewSubPath(contents.NewRootPath).Extract(stagingDstPath, stagingDstPath, stagingDir.TempArea())
+				if err != nil {
+					return lockConfig, fmt.Errorf("Changing to new root path '%s': %s", contents.Path, err)
+				}
 			}
-		}
 
-		// Copy files from current source if values are supposed to be ignored
-		err = stagingDir.CopyExistingFiles(d.opts.Path, stagingDstPath, contents.IgnorePaths)
-		if err != nil {
-			return lockConfig, fmt.Errorf("Copying existing content to staging '%s': %s", d.opts.Path, err)
-		}
+			// Copy files from current source if values are supposed to be ignored
+			err = stagingDir.CopyExistingFiles(d.opts.Path, stagingDstPath, contents.IgnorePaths)
+			if err != nil {
+				return lockConfig, fmt.Errorf("Copying existing content to staging '%s': %s", d.opts.Path, err)
+			}
 
-		// after everything else is done, ensure the inner dir's access perms are set
-		// chmod to the content's permission, fall back to the directory's
-		err = maybeChmod(stagingDstPath, contents.Permissions, d.opts.Permissions)
-		if err != nil {
-			return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err)
+			// after everything else is done, ensure the inner dir's access perms are set
+			// chmod to the content's permission, fall back to the directory's
+			err = maybeChmod(stagingDstPath, contents.Permissions, d.opts.Permissions)
+			if err != nil {
+				return lockConfig, fmt.Errorf("chmod on '%s': %s", stagingDstPath, err)
+			}
 		}
 
 		lockConfig.Contents = append(lockConfig.Contents, lockDirContents)
@@ -232,6 +288,8 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 	return lockConfig, nil
 }
 
+//
+
 // maybeChmod will chmod the path with the first non-nil permission provided.
 // If no permission is handed in or all of them are nil, no chmod will be done.
 func maybeChmod(path string, potentialPerms ...*os.FileMode) error {