Allow a pallet to deploy packages defined by that same pallet (#130)

CHANGELOG.md

@@ -7,9 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 0.5.2 - 2024-02-10
+
 ### Added
 
 - (spec) Added a "fileset" resource type for files (which can include directories).
+- (cli) Added support to for a pallet to deploy packages defined in that same pallet, by referring to the package as an absolute path (rooted at the root of the pallet), if the pallet declares itself as a Forklift repo with the same path.
 
 ## 0.5.1 - 2024-02-07
 

cmd/forklift/dev/plt/deployments.go

@@ -9,27 +9,21 @@ import (
 // ls-depl
 
 func lsDeplAction(c *cli.Context) error {
-	pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+	pallet, cache, err := processFullBaseArgs(c, true, true)
 	if err != nil {
 		return err
 	}
-	if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-		return err
-	}
 
 	return fcli.PrintPalletDepls(0, pallet, cache)
 }
 
 // show-depl
 
 func showDeplAction(c *cli.Context) error {
-	pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+	pallet, cache, err := processFullBaseArgs(c, true, true)
 	if err != nil {
 		return err
 	}
-	if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-		return err
-	}
 
 	deplName := c.Args().First()
 	return fcli.PrintDeplInfo(0, pallet, cache, deplName)
@@ -38,13 +32,10 @@ func showDeplAction(c *cli.Context) error {
 // locate-depl-pkg
 
 func locateDeplPkgAction(c *cli.Context) error {
-	pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+	pallet, cache, err := processFullBaseArgs(c, true, true)
 	if err != nil {
 		return err
 	}
-	if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-		return err
-	}
 
 	deplName := c.Args().First()
 	return fcli.PrintDeplPkgPath(0, pallet, cache, deplName, c.Bool("allow-disabled"))

cmd/forklift/dev/plt/images.go

@@ -12,13 +12,10 @@ import (
 
 func cacheImgAction(toolVersion, repoMinVersion, palletMinVersion string) cli.ActionFunc {
 	return func(c *cli.Context) error {
-		pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+		pallet, cache, err := processFullBaseArgs(c, true, true)
 		if err != nil {
 			return err
 		}
-		if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-			return err
-		}
 		if err = fcli.CheckCompatibility(
 			pallet, cache, toolVersion, repoMinVersion, palletMinVersion, c.Bool("ignore-tool-version"),
 		); err != nil {

cmd/forklift/dev/plt/packages.go

@@ -9,27 +9,21 @@ import (
 // ls-pkg
 
 func lsPkgAction(c *cli.Context) error {
-	pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+	pallet, cache, err := processFullBaseArgs(c, true, true)
 	if err != nil {
 		return err
 	}
-	if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-		return err
-	}
 
 	return fcli.PrintPalletPkgs(0, pallet, cache)
 }
 
 // show-pkg
 
 func showPkgAction(c *cli.Context) error {
-	pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+	pallet, cache, err := processFullBaseArgs(c, true, true)
 	if err != nil {
 		return err
 	}
-	if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-		return err
-	}
 
 	pkgPath := c.Args().First()
 	return fcli.PrintPkgInfo(0, pallet, cache, pkgPath)

cmd/forklift/dev/plt/pallets.go

@@ -13,19 +13,22 @@ import (
 	"github.com/PlanktoScope/forklift/pkg/core"
 )
 
-func processFullBaseArgs(c *cli.Context, ensureCache bool) (
-	pallet *forklift.FSPallet, cache *forklift.LayeredRepoCache, override *forklift.RepoOverrideCache,
-	err error,
+func processFullBaseArgs(c *cli.Context, ensureCache, enableOverrides bool) (
+	pallet *forklift.FSPallet, cache *forklift.LayeredRepoCache, err error,
 ) {
 	if pallet, err = getPallet(c.String("cwd")); err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
+	}
+	if cache, _, err = fcli.GetCache(c.String("workspace"), pallet, ensureCache); err != nil {
+		return nil, nil, err
 	}
-	if cache, override, err = getCache(
-		c.String("workspace"), c.StringSlice("repos"), ensureCache,
-	); err != nil {
-		return nil, nil, nil, err
+	if !enableOverrides {
+		return pallet, cache, nil
 	}
-	return pallet, cache, override, nil
+	if cache, err = overlayCacheOverrides(cache, c.StringSlice("repos"), pallet); err != nil {
+		return nil, nil, err
+	}
+	return pallet, cache, nil
 }
 
 func getPallet(cwdPath string) (pallet *forklift.FSPallet, err error) {
@@ -37,37 +40,25 @@ func getPallet(cwdPath string) (pallet *forklift.FSPallet, err error) {
 	return pallet, nil
 }
 
-func getCache(
-	wpath string, repos []string, ensureCache bool,
-) (*forklift.LayeredRepoCache, *forklift.RepoOverrideCache, error) {
-	cache := &forklift.LayeredRepoCache{}
+func overlayCacheOverrides(
+	underlay forklift.PathedRepoCache, repos []string, pallet *forklift.FSPallet,
+) (cache *forklift.LayeredRepoCache, err error) {
+	cache = &forklift.LayeredRepoCache{
+		Underlay: underlay,
+	}
 	replacementRepos, err := loadReplacementRepos(repos)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	override, err := forklift.NewRepoOverrideCache(replacementRepos, nil)
 	if err != nil {
-		return nil, nil, err
-	}
-	cache.Overlay = override
-
-	workspace, err := forklift.LoadWorkspace(wpath)
-	if err != nil {
-		return nil, nil, err
-	}
-	fsCache, err := workspace.GetRepoCache()
-	if err != nil && len(repos) == 0 {
-		return nil, nil, err
+		return nil, err
 	}
-	cache.Underlay = fsCache
-
-	if ensureCache && !fsCache.Exists() {
-		return nil, nil, errors.New(
-			"you first need to cache the repos specified by your pallet with " +
-				"`forklift dev plt cache-repo`",
-		)
+	if err = setOverrideCacheVersions(pallet, override); err != nil {
+		return nil, err
 	}
-	return cache, override, nil
+	cache.Overlay = override
+	return cache, nil
 }
 
 func loadReplacementRepos(fsPaths []string) (replacements []*core.FSRepo, err error) {
@@ -132,13 +123,10 @@ func showAction(c *cli.Context) error {
 
 func checkAction(toolVersion, repoMinVersion, palletMinVersion string) cli.ActionFunc {
 	return func(c *cli.Context) error {
-		pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+		pallet, cache, err := processFullBaseArgs(c, true, true)
 		if err != nil {
 			return err
 		}
-		if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-			return err
-		}
 		if err = fcli.CheckCompatibility(
 			pallet, cache, toolVersion, repoMinVersion, palletMinVersion, c.Bool("ignore-tool-version"),
 		); err != nil {
@@ -156,13 +144,10 @@ func checkAction(toolVersion, repoMinVersion, palletMinVersion string) cli.Actio
 
 func planAction(toolVersion, repoMinVersion, palletMinVersion string) cli.ActionFunc {
 	return func(c *cli.Context) error {
-		pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+		pallet, cache, err := processFullBaseArgs(c, true, true)
 		if err != nil {
 			return err
 		}
-		if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-			return err
-		}
 		if err = fcli.CheckCompatibility(
 			pallet, cache, toolVersion, repoMinVersion, palletMinVersion, c.Bool("ignore-tool-version"),
 		); err != nil {
@@ -180,13 +165,10 @@ func planAction(toolVersion, repoMinVersion, palletMinVersion string) cli.Action
 
 func applyAction(toolVersion, repoMinVersion, palletMinVersion string) cli.ActionFunc {
 	return func(c *cli.Context) error {
-		pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+		pallet, cache, err := processFullBaseArgs(c, true, true)
 		if err != nil {
 			return err
 		}
-		if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-			return err
-		}
 		if err = fcli.CheckCompatibility(
 			pallet, cache, toolVersion, repoMinVersion, palletMinVersion, c.Bool("ignore-tool-version"),
 		); err != nil {

cmd/forklift/dev/plt/repositories.go

@@ -22,7 +22,7 @@ import (
 
 func cacheRepoAction(toolVersion, repoMinVersion, palletMinVersion string) cli.ActionFunc {
 	return func(c *cli.Context) error {
-		pallet, cache, _, err := processFullBaseArgs(c, false)
+		pallet, cache, err := processFullBaseArgs(c, false, false)
 		if err != nil {
 			return err
 		}
@@ -62,13 +62,10 @@ func lsRepoAction(c *cli.Context) error {
 // show-repo
 
 func showRepoAction(c *cli.Context) error {
-	pallet, cache, overrideCache, err := processFullBaseArgs(c, true)
+	pallet, cache, err := processFullBaseArgs(c, true, true)
 	if err != nil {
 		return err
 	}
-	if err = setOverrideCacheVersions(pallet, overrideCache); err != nil {
-		return err
-	}
 
 	repoPath := c.Args().First()
 	return fcli.PrintRepoInfo(0, pallet, cache, repoPath)
@@ -78,7 +75,7 @@ func showRepoAction(c *cli.Context) error {
 
 func addRepoAction(toolVersion, repoMinVersion, palletMinVersion string) cli.ActionFunc {
 	return func(c *cli.Context) error {
-		pallet, cache, _, err := processFullBaseArgs(c, false)
+		pallet, cache, err := processFullBaseArgs(c, false, false)
 		if err != nil {
 			return err
 		}

cmd/forklift/plt/pallets.go

@@ -14,22 +14,13 @@ import (
 
 func processFullBaseArgs(
 	c *cli.Context, ensureCache bool,
-) (pallet *forklift.FSPallet, cache *forklift.FSRepoCache, err error) {
-	workspace, err := forklift.LoadWorkspace(c.String("workspace"))
-	if err != nil {
-		return nil, nil, err
-	}
+) (pallet *forklift.FSPallet, cache forklift.PathedRepoCache, err error) {
 	if pallet, err = getPallet(c.String("workspace")); err != nil {
 		return nil, nil, err
 	}
-	if cache, err = workspace.GetRepoCache(); err != nil {
+	if cache, _, err = fcli.GetCache(c.String("workspace"), pallet, ensureCache); err != nil {
 		return nil, nil, err
 	}
-	if ensureCache && !cache.Exists() {
-		return nil, nil, errors.New(
-			"you first need to cache the repos specified by your pallet with `forklift plt cache-repo`",
-		)
-	}
 	return pallet, cache, nil
 }
 

internal/app/forklift/caching.go

@@ -214,6 +214,7 @@ func (c *LayeredRepoCache) LoadFSPkg(pkgPath string, version string) (*core.FSPk
 		pkg, err := c.Overlay.LoadFSPkg(pkgPath, version)
 		return pkg, errors.Wrap(err, "couldn't load package from overlay")
 	}
+
 	pkg, err := c.Underlay.LoadFSPkg(pkgPath, version)
 	return pkg, errors.Wrap(err, "couldn't load package from underlay")
 }
@@ -305,6 +306,9 @@ func (c *RepoOverrideCache) SetVersions(repoPath string, versions map[string]str
 // IncludesFSRepo reports whether the RepoOverrideCache instance has a repo with the
 // specified path and version.
 func (c *RepoOverrideCache) IncludesFSRepo(repoPath string, version string) bool {
+	if c == nil {
+		return false
+	}
 	if _, ok := c.repos[repoPath]; !ok {
 		return false
 	}
@@ -364,6 +368,9 @@ func (c *RepoOverrideCache) LoadFSRepos(searchPattern string) ([]*core.FSRepo, e
 // IncludesFSPkg reports whether the RepoOverrideCache instance has a repo with the specified
 // version which covers the specified package path.
 func (c *RepoOverrideCache) IncludesFSPkg(pkgPath string, version string) bool {
+	if c == nil {
+		return false
+	}
 	// Beyond a certain number of repos, it's probably faster to just recurse down via the subdirs.
 	// But we probably don't need to worry about this for now.
 	for _, repo := range c.repos {

internal/app/forklift/cli/packages.go

@@ -18,7 +18,7 @@ func PrintPkg(indent int, cache forklift.PathedRepoCache, pkg *core.FSPkg) {
 	if core.CoversPath(cache, pkg.FS.Path()) {
 		IndentedPrintf(indent, "Path in cache: %s\n", core.GetSubdirPath(cache, pkg.FS.Path()))
 	} else {
-		IndentedPrintf(indent, "External path (replacing cached package): %s\n", pkg.FS.Path())
+		IndentedPrintf(indent, "Absolute path (replacing any cached copy): %s\n", pkg.FS.Path())
 	}
 
 	PrintPkgSpec(indent, pkg.Def.Package)
@@ -36,7 +36,7 @@ func printPkgRepo(indent int, cache forklift.PathedRepoCache, pkg *core.FSPkg) {
 		IndentedPrintf(indent, "Version: %s\n", pkg.Repo.Version)
 	} else {
 		IndentedPrintf(
-			indent, "External path (replacing cached repository): %s\n", pkg.Repo.FS.Path(),
+			indent, "Absolute path (replacing any cached copy): %s\n", pkg.Repo.FS.Path(),
 		)
 	}
 

internal/app/forklift/cli/pallets-repositories.go

@@ -14,6 +14,47 @@ import (
 	"github.com/PlanktoScope/forklift/pkg/core"
 )
 
+func GetCache(
+	wpath string, pallet *forklift.FSPallet, ensureCache bool,
+) (*forklift.LayeredRepoCache, *forklift.RepoOverrideCache, error) {
+	cache := &forklift.LayeredRepoCache{}
+	override, err := makeOverrideCacheFromPallet(pallet)
+	if err != nil {
+		return nil, nil, err
+	}
+	cache.Overlay = override
+
+	workspace, err := forklift.LoadWorkspace(wpath)
+	if err != nil {
+		return nil, nil, err
+	}
+	fsCache, err := workspace.GetRepoCache()
+	if err != nil && override == nil {
+		return nil, nil, err
+	}
+	cache.Underlay = fsCache
+
+	if ensureCache && !fsCache.Exists() {
+		return nil, nil, errors.New("you first need to cache the repos specified by your pallet")
+	}
+	return cache, override, nil
+}
+
+func makeOverrideCacheFromPallet(pallet *forklift.FSPallet) (*forklift.RepoOverrideCache, error) {
+	palletAsRepo, err := core.LoadFSRepo(pallet.FS, ".")
+	if err != nil {
+		// The common case is that the pallet is not a repo (and thus can't be loaded as one), so we
+		// mask the error:
+		return nil, nil
+	}
+	return forklift.NewRepoOverrideCache(
+		[]*core.FSRepo{palletAsRepo}, map[string][]string{
+			// In a pallet which is a repo, the implicit repo requirement is for an empty version string
+			palletAsRepo.Path(): {""},
+		},
+	)
+}
+
 // Print
 
 func PrintPalletRepos(indent int, pallet *forklift.FSPallet) error {
@@ -59,7 +100,7 @@ func PrintRepoInfo(
 			indent, "Path in cache: %s\n", core.GetSubdirPath(cache, cachedRepo.FS.Path()),
 		)
 	} else {
-		IndentedPrintf(indent, "External path (replacing cached repo): %s\n", cachedRepo.FS.Path())
+		IndentedPrintf(indent, "Absolute path (replacing any cached copy): %s\n", cachedRepo.FS.Path())
 	}
 	IndentedPrintf(indent, "Description: %s\n", cachedRepo.Def.Repo.Description)
 

internal/app/forklift/pallets-deployments.go

@@ -29,7 +29,7 @@ func ResolveDepl(
 		pkgReqLoader, pkgLoader, pkgPath,
 	); err != nil {
 		return nil, errors.Wrapf(
-			err, "couldn't load package %s to resolved from package deployment %s", pkgPath, depl.Name,
+			err, "couldn't load package %s to resolve from package deployment %s", pkgPath, depl.Name,
 		)
 	}
 	return resolved, nil