From 3c547567033e3b119a61777c8d7470eebd3dd119 Mon Sep 17 00:00:00 2001 From: Mikhail Elhimov Date: Mon, 3 Feb 2025 21:32:28 +0300 Subject: [PATCH] pack: support .packignore In some cases, there are files that may be useful, but should not be part of artifact. The ability to add files and directories to the .packignore file has been added, which allows you to ignore these files and directories when packing. Closes #812 --- CHANGELOG.md | 2 + cli/pack/common.go | 161 +++++++++++++++++++++++++++++++++-- cli/pack/common_test.go | 182 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 336 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d750f5975..c9cf901a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - `tt pack `: added TCM file packaging. +- `tt pack `: support `.packignore` file to specify files that should not be included + in package (works the same as `.gitignore`). ### Changed diff --git a/cli/pack/common.go b/cli/pack/common.go index 25c778c8a..786253ab8 100644 --- a/cli/pack/common.go +++ b/cli/pack/common.go @@ -1,6 +1,9 @@ package pack import ( + "bufio" + "bytes" + "errors" "fmt" "io/fs" "os" @@ -33,6 +36,8 @@ const ( versionLuaFileName = "VERSION.lua" rocksManifestPath = ".rocks/share/tarantool/rocks/manifest" + + ignoreFile = ".packignore" ) var ( @@ -51,6 +56,8 @@ var ( } ) +type skipFilter func(srcInfo os.FileInfo, src string) bool + type RocksVersions map[string][]string // packFileInfo contains information to set for files/dirs in rpm/deb packages. @@ -76,9 +83,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool { } // appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts. -func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts.App == nil { return filters } @@ -102,9 +108,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( } // ttEnvironmentFilters prepares a slice of filters for tt environment directories/files. -func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts == nil { return filters } @@ -139,10 +144,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( } // previousPackageFilters returns filters for the previously built packages. -func previousPackageFilters(packCtx *PackCtx) []func( - srcInfo os.FileInfo, src string) bool { +func previousPackageFilters(packCtx *PackCtx) []skipFilter { pkgName := packCtx.Name - return []func(srcInfo os.FileInfo, src string) bool{ + return []skipFilter{ func(srcInfo os.FileInfo, src string) bool { name := srcInfo.Name() if strings.HasPrefix(name, pkgName) { @@ -157,6 +161,138 @@ func previousPackageFilters(packCtx *PackCtx) []func( } } +func ignorePatternToRegex(pattern string, basepath string) (string, bool, bool) { + onlyDirs := strings.HasSuffix(pattern, "/") + if onlyDirs { + pattern = pattern[:len(pattern)-1] + } + + negate := false + if strings.HasPrefix(pattern, "!") { + negate = true + pattern = pattern[1:] + } else { + if strings.HasPrefix(pattern, "\\!") || strings.HasPrefix(pattern, "\\#") { + pattern = pattern[1:] + } + } + + expr := pattern + expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/)*") + // expr = strings.ReplaceAll(expr, "**/", "([^/]+/)*") + // expr = strings.ReplaceAll(expr, "/**", "[^/]*") + expr = strings.ReplaceAll(expr, "*", "[^/]*") + expr = strings.ReplaceAll(expr, "?", "[^/]") + + if strings.HasPrefix(pattern, "/") { + expr = basepath + expr + } else { + expr = "/?([^/]+/)*" + expr + } + + return expr, negate, onlyDirs +} + +// ignoreFilter returns filter that excludes files based on the patterns. +func ignoreFilter(fsys fs.FS, ignoreFile string) skipFilter { + log.Infof("ignoreFilter: %q", ignoreFile) + + var contents []byte + var err error + if fsys == nil { + contents, err = os.ReadFile(ignoreFile) + } else { + contents, err = fs.ReadFile(fsys, ignoreFile) + } + + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + log.Errorf("Failed to read %q: %v", ignoreFile, err) + } + return nil + } + + ignoreFileDir := filepath.Dir(ignoreFile) + + var exprExclude, exprInclude, exprExcludeDirs, exprIncludeDirs []string + + s := bufio.NewScanner(bytes.NewReader(contents)) + for s.Scan() { + pattern := strings.TrimSpace(s.Text()) + if pattern == "" || strings.HasPrefix(pattern, "#") { + continue + } + + exprPart, negate, onlyDirs := ignorePatternToRegex(pattern, ignoreFileDir) + log.Infof("exprPart: %q %v %v", exprPart, negate, onlyDirs) + + var expr *[]string + if onlyDirs { + if negate { + expr = &exprIncludeDirs + } else { + expr = &exprExcludeDirs + } + } else { + if negate { + expr = &exprInclude + } else { + expr = &exprExclude + } + } + *expr = append(*expr, "("+exprPart+")") + } + + compileRegexp := func(expr []string) *regexp.Regexp { + if len(expr) == 0 { + return nil + } + re, err := regexp.Compile("^" + strings.Join(expr, "|") + "$") + if err != nil { + log.Errorf(" failed to compile expression: %s", err.Error()) + return nil + } + return re + } + + reInclude := compileRegexp(exprInclude) + reExclude := compileRegexp(exprExclude) + reIncludeDirs := compileRegexp(exprIncludeDirs) + reExcludeDirs := compileRegexp(exprExcludeDirs) + + return func(srcInfo os.FileInfo, src string) bool { + log.Infof("ignoreFilter(): %q", src) + + // Skip ignore file itself. + if src == ignoreFile { + log.Infof(" } true (ignore file itself)") + return true + } + // If it's directory first check patterns that only match directories. + if srcInfo.IsDir() { + log.Infof(" is dir") + if reIncludeDirs != nil && reIncludeDirs.MatchString(src) { + log.Infof(" } false (include dirs) %q", reIncludeDirs.String()) + return false + } + if reExcludeDirs != nil && reExcludeDirs.MatchString(src) { + log.Infof(" } true (exclude dirs) %q", reExcludeDirs.String()) + return true + } + } + if reInclude != nil && reInclude.MatchString(src) { + log.Infof(" } false (include) %q", reInclude.String()) + return false + } + if reExclude != nil && reExclude.MatchString(src) { + log.Infof(" } true (exclude) %q", reExclude.String()) + return true + } + log.Infof(" } false") + return false + } +} + // appSrcCopySkip returns a filter func to filter out artifacts paths. func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) { @@ -166,13 +302,20 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool { return skipDefaults(srcInfo, src) }) + log.Infof("appSrcCopySkip: srcAppPath: %q", srcAppPath) + log.Infof("appSrcCopySkip: ignoreFile: %q", ignoreFile) + if f := ignoreFilter(nil, filepath.Join(srcAppPath, ignoreFile)); f != nil { + appCopyFilters = append(appCopyFilters, f) + } return func(srcinfo os.FileInfo, src, dest string) (bool, error) { for _, shouldSkip := range appCopyFilters { if shouldSkip(srcinfo, src) { + log.Infof("skip: %q > %q isDir=%v... SKIPPED", src, dest, srcinfo.IsDir()) return true, nil } } + log.Infof("skip: %q > %q isDir=%v... COPIED", src, dest, srcinfo.IsDir()) return false, nil } } diff --git a/cli/pack/common_test.go b/cli/pack/common_test.go index 43db6e626..e393354eb 100644 --- a/cli/pack/common_test.go +++ b/cli/pack/common_test.go @@ -2,12 +2,16 @@ package pack import ( "io" + "io/fs" "os" "os/exec" + "path" "path/filepath" "strings" "testing" + "testing/fstest" + "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tarantool/tt/cli/cmdcontext" @@ -1114,3 +1118,181 @@ func Test_prepareBundle(t *testing.T) { }) } } + +type testFile struct { + path string + expected bool +} + +type testCase struct { + name string + patterns []string + files []testFile +} + +func createTestFS(tc testCase) fstest.MapFS { + testFS := fstest.MapFS{} + if tc.patterns != nil { + testFS[".packignore"] = &fstest.MapFile{ + Data: []byte(strings.Join(tc.patterns, "\n")), + } + } + for _, file := range tc.files { + testFS[file.path] = &fstest.MapFile{} + } + return testFS +} + +func Test_ignoreFilterSinglePattern(t *testing.T) { + testCases := []testCase{ + { + name: "simple", + patterns: []string{ + "test", + }, + files: []testFile{ + {"test", true}, + {"test_blabla", false}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", true}, + {"d/test_blabla", false}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "question_prefix", + patterns: []string{ + "?test", + }, + files: []testFile{ + {"test", false}, + {"2test", true}, + {".test", true}, + {"test_blabla", false}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", false}, + {"d/2test", true}, + {"d/.test", true}, + {"d/test_blabla", false}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "question_suffix", + patterns: []string{ + "test?", + }, + files: []testFile{ + {"test", false}, + {"test2", true}, + {"test.", true}, + {"test_blabla", false}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", false}, + {"d/test2", true}, + {"d/test.", true}, + {"d/test_blabla", false}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "asterisk_prefix", + patterns: []string{ + "*test", + }, + files: []testFile{ + {"test", true}, + {"test_blabla", false}, + {"blabla_test", true}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", true}, + {"d/test_blabla", false}, + {"d/blabla_test", true}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + { + name: "asterisk_suffix", + patterns: []string{ + "test*", + }, + files: []testFile{ + {"test", true}, + {"test_blabla", true}, + {"blabla_test", false}, + {"bla_test_bla", false}, + {"foo", false}, + {"d/test", true}, + {"d/test_blabla", true}, + {"d/blabla_test", false}, + {"d/bla_test_bla", false}, + {"d/foo", false}, + }, + }, + } + + do_test := func(t *testing.T, tc testCase, dst string) { + fsys := fstest.MapFS{} + if tc.patterns != nil { + fsys[ignoreFile] = &fstest.MapFile{ + Data: []byte(strings.Join(tc.patterns, "\n")), + Mode: fs.FileMode(0644), + } + } + for _, file := range tc.files { + fsys[file.path] = &fstest.MapFile{ + Mode: fs.FileMode(0644), + } + } + filter := ignoreFilter(fsys, ignoreFile) + + dst = filepath.Join(dst, tc.name) + err := os.MkdirAll(dst, 0755) + if err != nil { + assert.Nil(t, err) + } + + err = copy.Copy(".", dst, copy.Options{ + FS: fsys, + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return filter(srcinfo, src), nil + }, + PermissionControl: copy.AddPermission(0755), + }) + assert.Nil(t, err) + for _, f := range tc.files { + if f.expected { + assert.NoFileExists(t, path.Join(dst, f.path)) + } else { + assert.FileExists(t, path.Join(dst, f.path)) + } + } + } + + t.Run("no ignore file", func(t *testing.T) { + f := ignoreFilter(fstest.MapFS{}, ignoreFile) + assert.Nil(t, f) + }) + + dst := t.TempDir() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + do_test(t, tc, dst) + }) + } +}