diff --git a/CHANGELOG.md b/CHANGELOG.md index d750f5975..c9cf901a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - `tt pack `: added TCM file packaging. +- `tt pack `: support `.packignore` file to specify files that should not be included + in package (works the same as `.gitignore`). ### Changed diff --git a/cli/pack/common.go b/cli/pack/common.go index 25c778c8a..786253ab8 100644 --- a/cli/pack/common.go +++ b/cli/pack/common.go @@ -1,6 +1,9 @@ package pack import ( + "bufio" + "bytes" + "errors" "fmt" "io/fs" "os" @@ -33,6 +36,8 @@ const ( versionLuaFileName = "VERSION.lua" rocksManifestPath = ".rocks/share/tarantool/rocks/manifest" + + ignoreFile = ".packignore" ) var ( @@ -51,6 +56,8 @@ var ( } ) +type skipFilter func(srcInfo os.FileInfo, src string) bool + type RocksVersions map[string][]string // packFileInfo contains information to set for files/dirs in rpm/deb packages. @@ -76,9 +83,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool { } // appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts. -func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts.App == nil { return filters } @@ -102,9 +108,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( } // ttEnvironmentFilters prepares a slice of filters for tt environment directories/files. -func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts == nil { return filters } @@ -139,10 +144,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( } // previousPackageFilters returns filters for the previously built packages. -func previousPackageFilters(packCtx *PackCtx) []func( - srcInfo os.FileInfo, src string) bool { +func previousPackageFilters(packCtx *PackCtx) []skipFilter { pkgName := packCtx.Name - return []func(srcInfo os.FileInfo, src string) bool{ + return []skipFilter{ func(srcInfo os.FileInfo, src string) bool { name := srcInfo.Name() if strings.HasPrefix(name, pkgName) { @@ -157,6 +161,138 @@ func previousPackageFilters(packCtx *PackCtx) []func( } } +func ignorePatternToRegex(pattern string, basepath string) (string, bool, bool) { + onlyDirs := strings.HasSuffix(pattern, "/") + if onlyDirs { + pattern = pattern[:len(pattern)-1] + } + + negate := false + if strings.HasPrefix(pattern, "!") { + negate = true + pattern = pattern[1:] + } else { + if strings.HasPrefix(pattern, "\\!") || strings.HasPrefix(pattern, "\\#") { + pattern = pattern[1:] + } + } + + expr := pattern + expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/)*") + // expr = strings.ReplaceAll(expr, "**/", "([^/]+/)*") + // expr = strings.ReplaceAll(expr, "/**", "[^/]*") + expr = strings.ReplaceAll(expr, "*", "[^/]*") + expr = strings.ReplaceAll(expr, "?", "[^/]") + + if strings.HasPrefix(pattern, "/") { + expr = basepath + expr + } else { + expr = "/?([^/]+/)*" + expr + } + + return expr, negate, onlyDirs +} + +// ignoreFilter returns filter that excludes files based on the patterns. +func ignoreFilter(fsys fs.FS, ignoreFile string) skipFilter { + log.Infof("ignoreFilter: %q", ignoreFile) + + var contents []byte + var err error + if fsys == nil { + contents, err = os.ReadFile(ignoreFile) + } else { + contents, err = fs.ReadFile(fsys, ignoreFile) + } + + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + log.Errorf("Failed to read %q: %v", ignoreFile, err) + } + return nil + } + + ignoreFileDir := filepath.Dir(ignoreFile) + + var exprExclude, exprInclude, exprExcludeDirs, exprIncludeDirs []string + + s := bufio.NewScanner(bytes.NewReader(contents)) + for s.Scan() { + pattern := strings.TrimSpace(s.Text()) + if pattern == "" || strings.HasPrefix(pattern, "#") { + continue + } + + exprPart, negate, onlyDirs := ignorePatternToRegex(pattern, ignoreFileDir) + log.Infof("exprPart: %q %v %v", exprPart, negate, onlyDirs) + + var expr *[]string + if onlyDirs { + if negate { + expr = &exprIncludeDirs + } else { + expr = &exprExcludeDirs + } + } else { + if negate { + expr = &exprInclude + } else { + expr = &exprExclude + } + } + *expr = append(*expr, "("+exprPart+")") + } + + compileRegexp := func(expr []string) *regexp.Regexp { + if len(expr) == 0 { + return nil + } + re, err := regexp.Compile("^" + strings.Join(expr, "|") + "$") + if err != nil { + log.Errorf(" failed to compile expression: %s", err.Error()) + return nil + } + return re + } + + reInclude := compileRegexp(exprInclude) + reExclude := compileRegexp(exprExclude) + reIncludeDirs := compileRegexp(exprIncludeDirs) + reExcludeDirs := compileRegexp(exprExcludeDirs) + + return func(srcInfo os.FileInfo, src string) bool { + log.Infof("ignoreFilter(): %q", src) + + // Skip ignore file itself. + if src == ignoreFile { + log.Infof(" } true (ignore file itself)") + return true + } + // If it's directory first check patterns that only match directories. + if srcInfo.IsDir() { + log.Infof(" is dir") + if reIncludeDirs != nil && reIncludeDirs.MatchString(src) { + log.Infof(" } false (include dirs) %q", reIncludeDirs.String()) + return false + } + if reExcludeDirs != nil && reExcludeDirs.MatchString(src) { + log.Infof(" } true (exclude dirs) %q", reExcludeDirs.String()) + return true + } + } + if reInclude != nil && reInclude.MatchString(src) { + log.Infof(" } false (include) %q", reInclude.String()) + return false + } + if reExclude != nil && reExclude.MatchString(src) { + log.Infof(" } true (exclude) %q", reExclude.String()) + return true + } + log.Infof(" } false") + return false + } +} + // appSrcCopySkip returns a filter func to filter out artifacts paths. func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) { @@ -166,13 +302,20 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool { return skipDefaults(srcInfo, src) }) + log.Infof("appSrcCopySkip: srcAppPath: %q", srcAppPath) + log.Infof("appSrcCopySkip: ignoreFile: %q", ignoreFile) + if f := ignoreFilter(nil, filepath.Join(srcAppPath, ignoreFile)); f != nil { + appCopyFilters = append(appCopyFilters, f) + } return func(srcinfo os.FileInfo, src, dest string) (bool, error) { for _, shouldSkip := range appCopyFilters { if shouldSkip(srcinfo, src) { + log.Infof("skip: %q > %q isDir=%v... SKIPPED", src, dest, srcinfo.IsDir()) return true, nil } } + log.Infof("skip: %q > %q isDir=%v... COPIED", src, dest, srcinfo.IsDir()) return false, nil } } diff --git a/cli/pack/common_test.go b/cli/pack/common_test.go index 43db6e626..e393354eb 100644 --- a/cli/pack/common_test.go +++ b/cli/pack/common_test.go @@ -2,12 +2,16 @@ package pack import ( "io" + "io/fs" "os" "os/exec" + "path" "path/filepath" "strings" "testing" + "testing/fstest" + "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tarantool/tt/cli/cmdcontext" @@ -1114,3 +1118,181 @@ func Test_prepareBundle(t *testing.T) { }) } } + +type testFile struct { + path string + expected bool +} + +type testCase struct { + name string + patterns []string + files []testFile +} + +func createTestFS(tc testCase) fstest.MapFS { + testFS := fstest.MapFS{} + if tc.patterns != nil { + testFS[".packignore"] = &fstest.MapFile{ + Data: []byte(strings.Join(tc.patterns, "\n")), + } + } + for _, file := range tc.files { + testFS[file.path] = &fstest.MapFile{} + } + return testFS +} + +func Test_ignoreFilterSinglePattern(t *testing.T) { + testCases := []testCase{ + { + name: "simple", + patterns: []string{ + "test", + }, + files: []testFile{ + {"test", true}, + {"test_blabla", false}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", true}, + {"d/test_blabla", false}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "question_prefix", + patterns: []string{ + "?test", + }, + files: []testFile{ + {"test", false}, + {"2test", true}, + {".test", true}, + {"test_blabla", false}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", false}, + {"d/2test", true}, + {"d/.test", true}, + {"d/test_blabla", false}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "question_suffix", + patterns: []string{ + "test?", + }, + files: []testFile{ + {"test", false}, + {"test2", true}, + {"test.", true}, + {"test_blabla", false}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", false}, + {"d/test2", true}, + {"d/test.", true}, + {"d/test_blabla", false}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "asterisk_prefix", + patterns: []string{ + "*test", + }, + files: []testFile{ + {"test", true}, + {"test_blabla", false}, + {"blabla_test", true}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", true}, + {"d/test_blabla", false}, + {"d/blabla_test", true}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "asterisk_suffix", + patterns: []string{ + "test*", + }, + files: []testFile{ + {"test", true}, + {"test_blabla", true}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", true}, + {"d/test_blabla", true}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + } + + do_test := func(t *testing.T, tc testCase, dst string) { + fsys := fstest.MapFS{} + if tc.patterns != nil { + fsys[ignoreFile] = &fstest.MapFile{ + Data: []byte(strings.Join(tc.patterns, "\n")), + Mode: fs.FileMode(0644), + } + } + for _, file := range tc.files { + fsys[file.path] = &fstest.MapFile{ + Mode: fs.FileMode(0644), + } + } + filter := ignoreFilter(fsys, ignoreFile) + + dst = filepath.Join(dst, tc.name) + err := os.MkdirAll(dst, 0755) + if err != nil { + assert.Nil(t, err) + } + + err = copy.Copy(".", dst, copy.Options{ + FS: fsys, + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return filter(srcinfo, src), nil + }, + PermissionControl: copy.AddPermission(0755), + }) + assert.Nil(t, err) + for _, f := range tc.files { + if f.expected { + assert.NoFileExists(t, path.Join(dst, f.path)) + } else { + assert.FileExists(t, path.Join(dst, f.path)) + } + } + } + + t.Run("no ignore file", func(t *testing.T) { + f := ignoreFilter(fstest.MapFS{}, ignoreFile) + assert.Nil(t, f) + }) + + dst := t.TempDir() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + do_test(t, tc, dst) + }) + } +}