pack: support .packignore

In some cases, there are files that may be useful, but should not
be part of artifact. The ability to add files and directories to
the .packignore file has been added, which allows you to ignore
these files and directories when packing.

Closes #812

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - `tt pack `: added TCM file packaging.
+- `tt pack `: support `.packignore` file to specify files that should not be included
+  in package (works the same as `.gitignore`).
 
 ### Changed
 

cli/pack/common.go

@@ -1,6 +1,9 @@
 package pack
 
 import (
+	"bufio"
+	"bytes"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -33,6 +36,8 @@ const (
 	versionLuaFileName = "VERSION.lua"
 
 	rocksManifestPath = ".rocks/share/tarantool/rocks/manifest"
+
+	ignoreFile = ".packignore"
 )
 
 var (
@@ -51,6 +56,8 @@ var (
 	}
 )
 
+type skipFilter func(srcInfo os.FileInfo, src string) bool
+
 type RocksVersions map[string][]string
 
 // packFileInfo contains information to set for files/dirs in rpm/deb packages.
@@ -76,9 +83,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool {
 }
 
 // appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts.
-func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func(
-	srcInfo os.FileInfo, src string) bool {
-	filters := make([]func(srcInfo os.FileInfo, src string) bool, 0)
+func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter {
+	filters := make([]skipFilter, 0)
 	if cliOpts.App == nil {
 		return filters
 	}
@@ -102,9 +108,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func(
 }
 
 // ttEnvironmentFilters prepares a slice of filters for tt environment directories/files.
-func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
-	srcInfo os.FileInfo, src string) bool {
-	filters := make([]func(srcInfo os.FileInfo, src string) bool, 0)
+func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter {
+	filters := make([]skipFilter, 0)
 	if cliOpts == nil {
 		return filters
 	}
@@ -139,10 +144,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
 }
 
 // previousPackageFilters returns filters for the previously built packages.
-func previousPackageFilters(packCtx *PackCtx) []func(
-	srcInfo os.FileInfo, src string) bool {
+func previousPackageFilters(packCtx *PackCtx) []skipFilter {
 	pkgName := packCtx.Name
-	return []func(srcInfo os.FileInfo, src string) bool{
+	return []skipFilter{
 		func(srcInfo os.FileInfo, src string) bool {
 			name := srcInfo.Name()
 			if strings.HasPrefix(name, pkgName) {
@@ -157,6 +161,138 @@ func previousPackageFilters(packCtx *PackCtx) []func(
 	}
 }
 
+func ignorePatternToRegex(pattern string, basepath string) (string, bool, bool) {
+	onlyDirs := strings.HasSuffix(pattern, "/")
+	if onlyDirs {
+		pattern = pattern[:len(pattern)-1]
+	}
+
+	negate := false
+	if strings.HasPrefix(pattern, "!") {
+		negate = true
+		pattern = pattern[1:]
+	} else {
+		if strings.HasPrefix(pattern, "\\!") || strings.HasPrefix(pattern, "\\#") {
+			pattern = pattern[1:]
+		}
+	}
+
+	expr := pattern
+	expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/)*")
+	// expr = strings.ReplaceAll(expr, "**/", "([^/]+/)*")
+	// expr = strings.ReplaceAll(expr, "/**", "[^/]*")
+	expr = strings.ReplaceAll(expr, "*", "[^/]*")
+	expr = strings.ReplaceAll(expr, "?", "[^/]")
+
+	if strings.HasPrefix(pattern, "/") {
+		expr = basepath + expr
+	} else {
+		expr = "/?([^/]+/)*" + expr
+	}
+
+	return expr, negate, onlyDirs
+}
+
+// ignoreFilter returns filter that excludes files based on the patterns.
+func ignoreFilter(fsys fs.FS, ignoreFile string) skipFilter {
+	log.Infof("ignoreFilter: %q", ignoreFile)
+
+	var contents []byte
+	var err error
+	if fsys == nil {
+		contents, err = os.ReadFile(ignoreFile)
+	} else {
+		contents, err = fs.ReadFile(fsys, ignoreFile)
+	}
+
+	if err != nil {
+		if !errors.Is(err, fs.ErrNotExist) {
+			log.Errorf("Failed to read %q: %v", ignoreFile, err)
+		}
+		return nil
+	}
+
+	ignoreFileDir := filepath.Dir(ignoreFile)
+
+	var exprExclude, exprInclude, exprExcludeDirs, exprIncludeDirs []string
+
+	s := bufio.NewScanner(bytes.NewReader(contents))
+	for s.Scan() {
+		pattern := strings.TrimSpace(s.Text())
+		if pattern == "" || strings.HasPrefix(pattern, "#") {
+			continue
+		}
+
+		exprPart, negate, onlyDirs := ignorePatternToRegex(pattern, ignoreFileDir)
+		log.Infof("exprPart: %q %v %v", exprPart, negate, onlyDirs)
+
+		var expr *[]string
+		if onlyDirs {
+			if negate {
+				expr = &exprIncludeDirs
+			} else {
+				expr = &exprExcludeDirs
+			}
+		} else {
+			if negate {
+				expr = &exprInclude
+			} else {
+				expr = &exprExclude
+			}
+		}
+		*expr = append(*expr, "("+exprPart+")")
+	}
+
+	compileRegexp := func(expr []string) *regexp.Regexp {
+		if len(expr) == 0 {
+			return nil
+		}
+		re, err := regexp.Compile("^" + strings.Join(expr, "|") + "$")
+		if err != nil {
+			log.Errorf("  failed to compile expression: %s", err.Error())
+			return nil
+		}
+		return re
+	}
+
+	reInclude := compileRegexp(exprInclude)
+	reExclude := compileRegexp(exprExclude)
+	reIncludeDirs := compileRegexp(exprIncludeDirs)
+	reExcludeDirs := compileRegexp(exprExcludeDirs)
+
+	return func(srcInfo os.FileInfo, src string) bool {
+		log.Infof("ignoreFilter(): %q", src)
+
+		// Skip ignore file itself.
+		if src == ignoreFile {
+			log.Infof("  } true (ignore file itself)")
+			return true
+		}
+		// If it's directory first check patterns that only match directories.
+		if srcInfo.IsDir() {
+			log.Infof("  is dir")
+			if reIncludeDirs != nil && reIncludeDirs.MatchString(src) {
+				log.Infof("  } false (include dirs) %q", reIncludeDirs.String())
+				return false
+			}
+			if reExcludeDirs != nil && reExcludeDirs.MatchString(src) {
+				log.Infof("  } true (exclude dirs) %q", reExcludeDirs.String())
+				return true
+			}
+		}
+		if reInclude != nil && reInclude.MatchString(src) {
+			log.Infof("  } false (include) %q", reInclude.String())
+			return false
+		}
+		if reExclude != nil && reExclude.MatchString(src) {
+			log.Infof("  } true (exclude) %q", reExclude.String())
+			return true
+		}
+		log.Infof("  } false")
+		return false
+	}
+}
+
 // appSrcCopySkip returns a filter func to filter out artifacts paths.
 func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
 	srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) {
@@ -166,13 +302,20 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
 	appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool {
 		return skipDefaults(srcInfo, src)
 	})
+	log.Infof("appSrcCopySkip: srcAppPath: %q", srcAppPath)
+	log.Infof("appSrcCopySkip: ignoreFile: %q", ignoreFile)
+	if f := ignoreFilter(nil, filepath.Join(srcAppPath, ignoreFile)); f != nil {
+		appCopyFilters = append(appCopyFilters, f)
+	}
 
 	return func(srcinfo os.FileInfo, src, dest string) (bool, error) {
 		for _, shouldSkip := range appCopyFilters {
 			if shouldSkip(srcinfo, src) {
+				log.Infof("skip: %q > %q isDir=%v... SKIPPED", src, dest, srcinfo.IsDir())
 				return true, nil
 			}
 		}
+		log.Infof("skip: %q > %q isDir=%v... COPIED", src, dest, srcinfo.IsDir())
 		return false, nil
 	}
 }