From e14d8381f15f723da311bb50c06071ad346f0139 Mon Sep 17 00:00:00 2001 From: Mikhail Elhimov Date: Mon, 3 Feb 2025 21:32:28 +0300 Subject: [PATCH] pack: support .packignore In some cases, there are files that may be useful, but should not be part of artifact. The ability to add files and directories to the .packignore file has been added, which allows you to ignore these files and directories when packing. Closes #812 @TarantoolBot document Title: `tt pack` support `.packignore` Use `.packignore` in the same way as `.gitignore` allows to exclude unnecessary files while preparing application package with `tt pack command. --- CHANGELOG.md | 2 + cli/pack/common.go | 34 +- cli/pack/ignore.go | 132 ++++ cli/pack/ignore_test.go | 933 +++++++++++++++++++++++++++++ cli/util/osfs.go | 25 + test/integration/pack/test_pack.py | 113 ++++ 6 files changed, 1227 insertions(+), 12 deletions(-) create mode 100644 cli/pack/ignore.go create mode 100644 cli/pack/ignore_test.go create mode 100644 cli/util/osfs.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e9500be..23d1a48d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - `tt aeon connect`: add connection from the `app:insance_name`. +- `tt pack `: support `.packignore` file to specify files that should not be included + in package (works the same as `.gitignore`). ### Changed diff --git a/cli/pack/common.go b/cli/pack/common.go index 25c778c8a..c34094993 100644 --- a/cli/pack/common.go +++ b/cli/pack/common.go @@ -1,6 +1,7 @@ package pack import ( + "errors" "fmt" "io/fs" "os" @@ -33,6 +34,8 @@ const ( versionLuaFileName = "VERSION.lua" rocksManifestPath = ".rocks/share/tarantool/rocks/manifest" + + ignoreFile = ".packignore" ) var ( @@ -51,6 +54,8 @@ var ( } ) +type skipFilter func(srcInfo os.FileInfo, src string) bool + type RocksVersions map[string][]string // packFileInfo contains information to set for files/dirs in rpm/deb packages. @@ -76,9 +81,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool { } // appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts. -func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts.App == nil { return filters } @@ -102,9 +106,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( } // ttEnvironmentFilters prepares a slice of filters for tt environment directories/files. -func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts == nil { return filters } @@ -139,10 +142,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( } // previousPackageFilters returns filters for the previously built packages. -func previousPackageFilters(packCtx *PackCtx) []func( - srcInfo os.FileInfo, src string) bool { +func previousPackageFilters(packCtx *PackCtx) []skipFilter { pkgName := packCtx.Name - return []func(srcInfo os.FileInfo, src string) bool{ + return []skipFilter{ func(srcInfo os.FileInfo, src string) bool { name := srcInfo.Name() if strings.HasPrefix(name, pkgName) { @@ -159,13 +161,18 @@ func previousPackageFilters(packCtx *PackCtx) []func( // appSrcCopySkip returns a filter func to filter out artifacts paths. func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, - srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) { + srcAppPath string) (func(srcinfo os.FileInfo, src, dest string) (bool, error), error) { appCopyFilters := appArtifactsFilters(cliOpts, srcAppPath) appCopyFilters = append(appCopyFilters, ttEnvironmentFilters(packCtx, cliOpts)...) appCopyFilters = append(appCopyFilters, previousPackageFilters(packCtx)...) appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool { return skipDefaults(srcInfo, src) }) + if f, err := ignoreFilter(util.GetOsFS(), filepath.Join(srcAppPath, ignoreFile)); err == nil { + appCopyFilters = append(appCopyFilters, f) + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("failed to load %q: %w", ignoreFile, err) + } return func(srcinfo os.FileInfo, src, dest string) (bool, error) { for _, shouldSkip := range appCopyFilters { @@ -174,7 +181,7 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, } } return false, nil - } + }, nil } // getAppNamesToPack generates application names list to pack. @@ -430,7 +437,10 @@ func copyAppSrc(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath, dstAppPat return err } - skipFunc := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath) + skipFunc, err := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath) + if err != nil { + return err + } // Copying application. log.Debugf("Copying application source %q -> %q", resolvedAppPath, dstAppPath) diff --git a/cli/pack/ignore.go b/cli/pack/ignore.go new file mode 100644 index 000000000..fc3dc306f --- /dev/null +++ b/cli/pack/ignore.go @@ -0,0 +1,132 @@ +package pack + +import ( + "bufio" + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" +) + +type ignorePattern struct { + re *regexp.Regexp + dirOnly bool + isNegate bool +} + +func turnEscapedToHexCode(s string, c rune) string { + return strings.ReplaceAll(s, `\`+string(c), fmt.Sprintf(`\x%x`, c)) +} + +func splitIgnorePattern(pattern string) (cleanPattern string, dirOnly bool, isNegate bool) { + // First, get rid of `\\` to simplify further handling of escaped sequences. + // From now on any `\c` always means escaped 'c' (previously it might also + // occur as a part of `\\c` sequence which denotes '\' followed by ). + cleanPattern = turnEscapedToHexCode(pattern, '\\') + + // Remove trailing spaces (unless escaped one). + cleanPattern = turnEscapedToHexCode(cleanPattern, ' ') + cleanPattern = strings.TrimRight(cleanPattern, " ") + + cleanPattern, dirOnly = strings.CutSuffix(cleanPattern, "/") + cleanPattern, isNegate = strings.CutPrefix(cleanPattern, "!") + return +} + +func createIgnorePattern(pattern string, basepath string) (ignorePattern, error) { + cleanPattern, dirOnly, isNegate := splitIgnorePattern(pattern) + + // Translate pattern to regex expression. + expr := cleanPattern + // Turn escaped '*' and '?' to their hex representation to simplify the translation. + expr = turnEscapedToHexCode(expr, '*') + expr = turnEscapedToHexCode(expr, '?') + // Escape symbols that designate themselves in pattern, but have special meaning in regex. + for _, s := range []string{"(", ")", "{", "}", "+"} { + // Do unescape first to avoid double escaping of the ones that are already escaped. + expr = strings.ReplaceAll(expr, "\\"+s, s) + expr = strings.ReplaceAll(expr, s, "\\"+s) + } + // Replace wildcards with the corresponding regex representation. + // Note that '{0,}' (not '*') is used while replacing '**' to avoid confusing + // in the subsequent replacement of a single '*'. + expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/){0,}") + expr, found := strings.CutPrefix(expr, "**/") + if found || !strings.Contains(cleanPattern, "/") { + expr = "([^/]+/){0,}" + expr + } + expr, found = strings.CutSuffix(expr, "/**") + if found { + expr = expr + "/([^/]+/){0,}[^/]+" + } + expr = strings.ReplaceAll(expr, "*", "[^/]*") + expr = strings.ReplaceAll(expr, "?", "[^/]") + + re, err := regexp.Compile("^" + basepath + expr + "$") + if err != nil { + return ignorePattern{}, fmt.Errorf("failed to compile expression: %w", err) + } + + return ignorePattern{ + re: re, + dirOnly: dirOnly, + isNegate: isNegate, + }, nil +} + +// loadIgnorePatterns reads ignore patterns from the patternsFile. +func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error) { + contents, err := fs.ReadFile(fsys, patternsFile) + if err != nil { + return nil, err + } + + basepath, _ := filepath.Split(patternsFile) + + var patterns []ignorePattern + s := bufio.NewScanner(bytes.NewReader(contents)) + for s.Scan() { + pattern := s.Text() + if pattern == "" || strings.HasPrefix(pattern, "#") { + continue + } + + p, err := createIgnorePattern(pattern, basepath) + if err != nil { + return nil, err + } + + patterns = append(patterns, p) + } + return patterns, nil +} + +// ignoreFilter returns filter function that implements .gitignore approach of filtering files. +func ignoreFilter(fsys fs.FS, patternsFile string) (skipFilter, error) { + patterns, err := loadIgnorePatterns(fsys, patternsFile) + if err != nil { + return nil, err + } + + // According to .gitignore documentation "the last matching pattern decides the outcome" + // so we need to iterate in reverse order until the first match. + slices.Reverse(patterns) + + return func(srcInfo os.FileInfo, src string) bool { + // Skip ignore file itself. + if src == patternsFile { + return true + } + for _, p := range patterns { + isApplicable := srcInfo.IsDir() || !p.dirOnly + if isApplicable && p.re.MatchString(src) { + return !p.isNegate + } + } + return false + }, nil +} diff --git a/cli/pack/ignore_test.go b/cli/pack/ignore_test.go new file mode 100644 index 000000000..7d6f56ca4 --- /dev/null +++ b/cli/pack/ignore_test.go @@ -0,0 +1,933 @@ +package pack + +import ( + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" + "testing" + "testing/fstest" + + "github.com/otiai10/copy" + "github.com/stretchr/testify/assert" +) + +// mapSlice is a generic slice mapper function. +func mapSlice[T, V any](ts []T, fn func(T) V) []V { + result := make([]V, len(ts)) + for i, t := range ts { + result[i] = fn(t) + } + return result +} + +// ignoreTestData is used to define the very basic data set +// which then used as a source to generate the actual testcases +// suitable for the corresponding test functions. +type ignoreTestData struct { + name string + pattern string + matches []string + mismatches []string +} + +// The 'pattern' field of any item from this data set refers to name (no path separator). +// The 'matches'/'mismatches' fields must contain only names as well. This constraint allows +// to expand corresponding test cases for the certain function in a more convenient way. +var ignoreTestData_names = []ignoreTestData{ + { + name: "simple_name", + pattern: "foo", + matches: []string{ + "foo", + }, + mismatches: []string{ + "foo2", + ".foo", + "blabla_foo", + "foo_blabla", + "bla_foo_bla", + }, + }, + { + name: "name_with_space", + pattern: "foo with space", + matches: []string{ + "foo with space", + }, + mismatches: []string{ + "foo with space2", + ".foo with space", + "blabla_foo with space", + "foo with space_blabla", + "bla_foo with space_bla", + }, + }, + { + name: "name_ends_with_space", + pattern: "foo_ends_with_space\\ ", + matches: []string{ + "foo_ends_with_space ", + }, + mismatches: []string{ + "foo_ends_with_space", + "foo_ends_with_space ", + ".foo_ends_with_space ", + "blabla_foo_ends_with_space ", + "foo_ends_with_space blabla", + "bla_foo_ends_with_space bla", + }, + }, + { + name: "name_with_brackets", + pattern: "foo(with_brackets)", + matches: []string{ + "foo(with_brackets)", + }, + mismatches: []string{ + "foo(with_brackets)2", + ".foo(with_brackets)", + "blabla_foo(with_brackets)", + "foo(with_brackets)_blabla", + "bla_foo(with_brackets)_bla", + }, + }, + { + name: "name_with_curly_brackets", + pattern: "foo{with_curly_brackets}", + matches: []string{ + "foo{with_curly_brackets}", + }, + mismatches: []string{ + "foo{with_curly_brackets}2", + ".foo{with_curly_brackets}", + "blabla_foo{with_curly_brackets}", + "foo{with_curly_brackets}_blabla", + "bla_foo{with_curly_brackets}_bla", + }, + }, + { + name: "name_with_plus", + pattern: "f+oo", + matches: []string{ + "f+oo", + }, + mismatches: []string{ + "f+oo2", + "ffoo", + ".f+oo", + "blabla_f+oo2", + "f+oo2_blabla", + "bla_f+oo2_bla", + }, + }, + { + name: "name_with_escaped_square_brackets", + pattern: "foo\\[with_escaped_square_brackets\\]", + matches: []string{ + "foo[with_escaped_square_brackets]", + }, + mismatches: []string{ + "foo[with_escaped_square_brackets]2", + ".foo[with_escaped_square_brackets]", + "blabla_foo[with_escaped_square_brackets]2", + "foo[with_escaped_square_brackets]2_blabla", + "bla_foo[with_escaped_square_brackets]2_bla", + }, + }, + { + name: "name_with_escaped_question", + pattern: "foo\\?with_escaped_question", + matches: []string{ + "foo?with_escaped_question", + }, + mismatches: []string{ + "foo?with_escaped_question2", + ".foo?with_escaped_question", + "foo2with_escaped_question", + "blabla_foo?with_escaped_question2", + "foo?with_escaped_question2_blabla", + "bla_foo?with_escaped_question2_bla", + }, + }, + { + name: "name_with_escaped_asterisk", + pattern: "foo\\*with_escaped_asterisk", + matches: []string{ + "foo*with_escaped_asterisk", + }, + mismatches: []string{ + "foo*with_escaped_asterisk2", + ".foo*with_escaped_asterisk", + "blabla_foo*with_escaped_asterisk2", + "foo*with_escaped_asterisk2_blabla", + "bla_foo*with_escaped_asterisk2_bla", + }, + }, + { + name: "name_with_question_prefix", + pattern: "?foo", + matches: []string{ + "2foo", + "?foo", + ".foo", + "*foo", + }, + mismatches: []string{ + "foo", + "foo2", + "blabla_2foo", + "2foo_blabla", + "bla_2foo_bla", + }, + }, + { + name: "name_with_question_suffix", + pattern: "foo?", + matches: []string{ + "foo2", + "foo?", + "foo*", + "foo ", + }, + mismatches: []string{ + "foo", + "blabla_foo2", + "foo2_blabla", + "bla_foo2_bla", + }, + }, + { + name: "name_with_question_between", + pattern: "f?oo", + matches: []string{ + "f2oo", + "fooo", + "f?oo", + "f*oo", + }, + mismatches: []string{ + "foo", + "blabla_f2oo", + "f2oo_blabla", + "bla_f2oo_bla", + }, + }, + { + name: "name_with_asterisk_prefix", + pattern: "*foo", + matches: []string{ + "blabla_foo", + "foo", + ".foo", + "*foo", + "?foo", + }, + mismatches: []string{ + "foo2", + "2foo_blabla", + "bla_2foo_bla", + }, + }, + { + name: "name_with_asterisk_suffix", + pattern: "foo*", + matches: []string{ + "foo_blabla", + "foo", + "foo*", + "foo?", + }, + mismatches: []string{ + "2foo", + "blabla_2foo", + "2foo_blabla", + "bla_2foo_bla", + }, + }, + { + name: "name_with_asterisk_between", + pattern: "f*oo", + matches: []string{ + "f2oo", + "foo", + "f*oo", + "f?oo", + }, + mismatches: []string{ + "foo2", + "blabla_foo2", + "foo2_blabla", + "bla_foo2_bla", + }, + }, + { + name: "name_with_range_basic", + pattern: "f[n-p]o", + matches: []string{ + "fno", + "foo", + "fpo", + }, + mismatches: []string{ + "f2o", + "fmo", + "fqo", + "f?o", + "blabla_foo", + "foo_blabla", + "bla_foo_bla", + }, + }, + { + name: "name_with_range_inverted", + pattern: "f[^n-p]o", + matches: []string{ + "f2o", + "fmo", + "fqo", + "f?o", + }, + mismatches: []string{ + "foo", + "fno", + "fpo", + }, + }, + { + name: "name_with_set_basic", + pattern: "[fgm]oo", + matches: []string{ + "foo", + "goo", + "moo", + }, + mismatches: []string{ + "zoo", + "ooo", + "?oo", + "blabla_foo", + "foo_blabla", + "bla_foo_bla", + }, + }, + { + name: "name_with_set_inverted", + pattern: "[^fgm]oo", + matches: []string{ + "zoo", + "ooo", + "?oo", + }, + mismatches: []string{ + "foo", + "goo", + "moo", + "blabla_zoo", + "zoo_blabla", + "bla_zoo_bla", + }, + }, +} + +var ignoreTestData_paths = []ignoreTestData{ + { + name: "name_at_depth1", + pattern: "*/foo", + matches: []string{ + "in_subdir/foo", + "in_another_subdir/foo", + }, + mismatches: []string{ + "foo", + "in/subdir/of/another/depth/foo", + "foo2", + "similar_in_subdir/foo2", + }, + }, + { + name: "name_at_depth2", + pattern: "*/*/foo", + matches: []string{ + "in_subdir/of_depth2/foo", + "in_another_subdir/of_depth2/foo", + }, + mismatches: []string{ + "foo", + "in/subdir/of/another/depth/foo", + "foo2", + "similar_in_subdir/of_depth2/foo2", + }, + }, + { + name: "depth1_under_name", + pattern: "foo/*", + matches: []string{ + "foo/bar", + "foo/blabla", + }, + mismatches: []string{ + "foo", + "foo2/bar", + "foo2/blabla", + }, + }, + { + name: "depth2_under_name", + pattern: "foo/*/*", + matches: []string{ + "foo/subdir/bar", + "foo/subdir/blabla", + "foo/another_subdir/bar", + }, + mismatches: []string{ + "foo", + "foo/bar", + "foo/blabla", + "foo2/subdir/bar", + }, + }, + { + name: "name_with_double_asterisk_leading", + pattern: "**/foo", + matches: []string{ + "foo", + "in_subdir/foo", + "in/deep/nested/subdir/foo", + }, + mismatches: []string{ + "foo2", + "similar_in_subdir/foo2", + "similar/in/deep/nested/subdir/foo2", + "subdir/foo2/bar", + }, + }, + { + name: "name_with_double_asterisk_trailing", + pattern: "foo/**", + matches: []string{ + "foo/bar", + "foo/with_subdir/bar", + "foo/with/deep/nested/subdir/bar", + }, + mismatches: []string{ + "foo", + "foo_blabla", + "file_in_subdir/foo", + "file/in/deep/nested/subdir/foo", + "similar_subdir/foo2/bar", + }, + }, + { + name: "name_with_inner_double_asterisk", + pattern: "foo/**/bar", + matches: []string{ + "foo/bar", + "foo/subdir/bar", + "foo/deep/nested/subdir/bar", + }, + mismatches: []string{ + "foo/bar2", + "foo/with_subdir/bar2", + "foo/with/deep/nested/subdir/bar2", + "foo2", + "similar_in_subdir/foo2", + "similar/in/deep/nested/subdir/foo2", + "subdir/foo2/bar", + }, + }, +} + +type testCase_ignorePattern struct { + name string + pattern string + expectedMatches []string + expectedMismatches []string + expectedDirOnly bool + expectedIsNegate bool +} + +func runTestSet_ignorePattern(t *testing.T, testCases []testCase_ignorePattern) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + p, err := createIgnorePattern(tc.pattern, "") + assert.Nil(t, err) + assert.NotNil(t, p.re) + assert.Equal(t, tc.expectedDirOnly, p.dirOnly) + assert.Equal(t, tc.expectedIsNegate, p.isNegate) + for _, s := range tc.expectedMatches { + assert.Truef(t, p.re.MatchString(s), + "%q doesn't match %s", s, p.re.String()) + } + for _, s := range tc.expectedMismatches { + assert.False(t, p.re.MatchString(s), + "%q matches %s", s, p.re.String()) + } + }) + } +} + +// Prepare basic test set for createIgnorePattern function +var testCases_ignorePatternBasic = slices.Concat( + mapSlice(ignoreTestData_names, func(td ignoreTestData) testCase_ignorePattern { + return testCase_ignorePattern{ + name: td.name, + pattern: td.pattern, + // Expand with some meaningful paths (note that td.matches itself has no path-item). + expectedMatches: append(td.matches, + "in_subdir/"+td.matches[0], + "in/deep/nested/subdir/"+td.matches[0], + ), + expectedMismatches: td.mismatches, + expectedDirOnly: false, + expectedIsNegate: false, + } + }), + mapSlice(ignoreTestData_paths, func(td ignoreTestData) testCase_ignorePattern { + return testCase_ignorePattern{ + name: td.name, + pattern: td.pattern, + expectedMatches: td.matches, + expectedMismatches: td.mismatches, + expectedDirOnly: false, + expectedIsNegate: false, + } + }), +) + +func Test_createIgnorePattern_basic(t *testing.T) { + runTestSet_ignorePattern(t, testCases_ignorePatternBasic) +} + +func Test_createIgnorePattern_negate(t *testing.T) { + testCases := mapSlice(testCases_ignorePatternBasic, + func(tc testCase_ignorePattern) testCase_ignorePattern { + return testCase_ignorePattern{ + name: tc.name, + pattern: "!" + tc.pattern, + expectedMatches: tc.expectedMatches, + expectedMismatches: tc.expectedMismatches, + expectedDirOnly: tc.expectedDirOnly, + expectedIsNegate: true, + } + }, + ) + runTestSet_ignorePattern(t, testCases) +} + +func Test_createIgnorePattern_dirOnly(t *testing.T) { + testCases := mapSlice(testCases_ignorePatternBasic, + func(tc testCase_ignorePattern) testCase_ignorePattern { + return testCase_ignorePattern{ + name: tc.name, + pattern: tc.pattern + "/", + expectedMatches: tc.expectedMatches, + expectedMismatches: tc.expectedMismatches, + expectedDirOnly: true, + expectedIsNegate: tc.expectedIsNegate, + } + }, + ) + runTestSet_ignorePattern(t, testCases) +} + +func Test_createIgnorePattern_trailingSpace(t *testing.T) { + testCases := mapSlice(testCases_ignorePatternBasic, + func(tc testCase_ignorePattern) testCase_ignorePattern { + return testCase_ignorePattern{ + name: tc.name, + pattern: tc.pattern + strings.Repeat(" ", 1+len(tc.name)%3), + expectedMatches: tc.expectedMatches, + expectedMismatches: tc.expectedMismatches, + expectedDirOnly: tc.expectedDirOnly, + expectedIsNegate: tc.expectedIsNegate, + } + }, + ) + runTestSet_ignorePattern(t, testCases) +} + +// NOTE: For a new test the below snippet can be used +// func Test_createIgnorePattern_someNewTest(t *testing.T) { +// testCases := []testCase_ignorePattern{ +// {...} +// } +// runTestSet_ignorePattern(t, testCases) +// } + +type testCase_ignoreFilter struct { + // Test name. + name string + // Ignore patterns. + patterns []string + // Files that are expected to be ignored during copy. + expectedIgnored []string + // Files that are expected to be copied. + expectedCopied []string +} + +func runTestSet_ignoreFilter(t *testing.T, testCases []testCase_ignoreFilter) { + // Helper function to create mock FS for the testcase + createFS := func(tc testCase_ignoreFilter) fs.FS { + fsys := fstest.MapFS{} + if tc.patterns != nil { + fsys[ignoreFile] = &fstest.MapFile{ + Data: []byte(strings.Join(tc.patterns, "\n")), + Mode: fs.FileMode(0644), + } + } + for _, name := range slices.Concat(tc.expectedCopied, tc.expectedIgnored) { + fsys[name] = &fstest.MapFile{ + Mode: fs.FileMode(0644), + } + } + return fsys + } + + basedst := t.TempDir() + + // Do test + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fsys := createFS(tc) + + filter, err := ignoreFilter(fsys, ignoreFile) + assert.Nil(t, err) + assert.NotNil(t, filter) + + dst := filepath.Join(basedst, tc.name) + err = os.MkdirAll(dst, 0755) + if err != nil { + assert.Nil(t, err) + } + + err = copy.Copy(".", dst, copy.Options{ + FS: fsys, + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return filter(srcinfo, src), nil + }, + PermissionControl: copy.AddPermission(0755), + }) + assert.Nil(t, err) + for _, name := range tc.expectedIgnored { + assert.NoFileExists(t, path.Join(dst, name)) + } + for _, name := range tc.expectedCopied { + assert.FileExists(t, path.Join(dst, name)) + } + }) + } +} + +// Prepare basic test set for ignoreFilter function +var testCases_ignoreFilterBasic = slices.Concat( + mapSlice(ignoreTestData_names, func(td ignoreTestData) testCase_ignoreFilter { + return testCase_ignoreFilter{ + name: td.name, + patterns: []string{ + td.pattern, + }, + // Expand with some meaningful paths (note that td.matches itself has no path-item). + expectedIgnored: append(td.matches, + "in_subdir/"+td.matches[0], + "in/deep/nested/subdir/"+td.matches[0], + "as_subdir/"+td.matches[0]+"/bar", + "as/deep/nested/subdir/"+td.matches[0]+"/bar", + ), + // Expand with some meaningful paths (note that td.mismatches itself has no path-item). + expectedCopied: append(td.mismatches, + "in_subdir/"+td.mismatches[0], + "in/deep/nested/subdir/"+td.mismatches[0], + ), + } + }), +) + +func Test_ignoreFilter_noIgnoreFile(t *testing.T) { + f, err := ignoreFilter(fstest.MapFS{}, ignoreFile) + assert.NotNil(t, err) + assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.Nil(t, f) +} + +func Test_ignoreFilter_singleBasic(t *testing.T) { + runTestSet_ignoreFilter(t, testCases_ignoreFilterBasic) +} + +func Test_ignoreFilter_singleNegate(t *testing.T) { + // Single negate pattern has no effect. + testCases := mapSlice(testCases_ignoreFilterBasic, + func(tc testCase_ignoreFilter) testCase_ignoreFilter { + return testCase_ignoreFilter{ + name: tc.name, + patterns: []string{ + "!" + tc.patterns[0], + }, + expectedIgnored: nil, + expectedCopied: slices.Concat(tc.expectedCopied, tc.expectedIgnored), + } + }) + runTestSet_ignoreFilter(t, testCases) +} + +func Test_ignoreFilter_selfNegate(t *testing.T) { + // An ignore pattern followed by the same but negated (thus it just reinclude all). + testCases := mapSlice(testCases_ignoreFilterBasic, + func(tc testCase_ignoreFilter) testCase_ignoreFilter { + return testCase_ignoreFilter{ + name: tc.name, + patterns: []string{ + tc.patterns[0], + "!" + tc.patterns[0], + }, + expectedIgnored: nil, + expectedCopied: slices.Concat(tc.expectedCopied, tc.expectedIgnored), + } + }) + runTestSet_ignoreFilter(t, testCases) +} + +func Test_ignoreFilter_negateWrongOrder(t *testing.T) { + // An ignore pattern followed by the same but negated (thus it just reinclude all). + testCases := mapSlice(testCases_ignoreFilterBasic, + func(tc testCase_ignoreFilter) testCase_ignoreFilter { + return testCase_ignoreFilter{ + name: tc.name, + patterns: []string{ + "!" + tc.patterns[0], + tc.patterns[0], + }, + expectedIgnored: tc.expectedIgnored, + expectedCopied: tc.expectedCopied, + } + }) + runTestSet_ignoreFilter(t, testCases) +} + +func Test_ignoreFilter_multiNames(t *testing.T) { + testCases := []testCase_ignoreFilter{ + { + name: "any", + patterns: []string{ + "name1", + "name2", + }, + expectedIgnored: []string{ + "name1", + "in_subdir/name1", + "in/deep/nested/subdir/name1", + "as_subdir/name1/foo", + "as/deep/nested/subdir/name1/bar", + "name2", + "in_subdir/name2", + "in/deep/nested/subdir/name2", + "as_subdir/name2/foo", + "as/deep/nested/subdir/name2/bar", + }, + expectedCopied: []string{ + "name3", + "name4", + }, + }, + { + name: "dironly", + patterns: []string{ + "name1/", + "name2/", + }, + expectedIgnored: []string{ + "as_subdir/name1/foo", + "as/deep/nested/subdir/name1/bar", + "as_subdir/name2/foo", + "as/deep/nested/subdir/name2/bar", + }, + expectedCopied: []string{ + "name1", + "in_subdir/name1", + "in/deep/nested/subdir/name1", + "name2", + "in_subdir/name2", + "in/deep/nested/subdir/name2", + "name3", + "name4", + }, + }, + { + name: "mixed", + patterns: []string{ + "name1", + "name2/", + }, + expectedIgnored: []string{ + "name1", + "in_subdir/name1", + "in/deep/nested/subdir/name1", + "as_subdir/name1/foo", + "as/deep/nested/subdir/name1/bar", + "as_subdir/name2/bar", + "as/deep/nested/subdir/name2/bar", + }, + expectedCopied: []string{ + "name2", + "in_subdir/name2", + "in/deep/nested/subdir/name2", + "name3", + "name4", + }, + }, + } + runTestSet_ignoreFilter(t, testCases) +} + +func Test_ignoreFilter_reinclude(t *testing.T) { + testCases := []testCase_ignoreFilter{ + { + name: "by_name", + patterns: []string{ + "*name?", + "!renamed", + }, + expectedIgnored: []string{ + "name1", + "in_subdir/filename2", + "in/deep/nested/subdir/rename3", + "as_subdir/dirname4/bar", + "as_subdir/dirname4/renamed", + "as/deep/nested/subdir/newname5/bar", + "as/deep/nested/subdir/newname5/renamed", + }, + expectedCopied: []string{ + "renamed", + "in_subdir/renamed", + "as_subdir/renamed/bar", + "name13", + "rename14", + }, + }, + { + name: "by_names", + patterns: []string{ + "*name?", + "!renamed", + "!unnamed", + }, + expectedIgnored: []string{ + "name1", + "newname2", + "oldname3", + "in_subdir/filename2", + "in/deep/nested/subdir/rename3", + "as_subdir/dirname4/bar", + "as_subdir/dirname4/renamed", + "as/deep/nested/subdir/newname5/bar", + "as/deep/nested/subdir/newname5/renamed", + }, + expectedCopied: []string{ + "renamed", + "in_subdir/renamed", + "as_subdir/renamed/bar", + "unnamed", + "in_subdir/unnamed", + "as_subdir/unnamed/bar", + "name13", + }, + }, + { + name: "by_pattern", + patterns: []string{ + "*name?", + "!*named", + }, + expectedIgnored: []string{ + "name1", + "newname2", + "oldname3", + "in_subdir/filename2", + "in/deep/nested/subdir/rename3", + "as_subdir/dirname4/bar", + "as_subdir/dirname4/renamed", + "as/deep/nested/subdir/newname5/bar", + "as/deep/nested/subdir/newname5/renamed", + "as/deep/nested/subdir/newname5/unnamed", + }, + expectedCopied: []string{ + "renamed", + "in_subdir/renamed", + "as_subdir/renamed/bar", + "unnamed", + "in_subdir/unnamed", + "as_subdir/unnamed/bar", + "name13", + }, + }, + } + runTestSet_ignoreFilter(t, testCases) +} + +func Test_ignoreFilter_fixedDepth(t *testing.T) { + testCases := []testCase_ignoreFilter{ + { + name: "name_at_depth1", + patterns: []string{ + "*/foo", + }, + expectedIgnored: []string{ + "in_subdir/foo", + "in_another_subdir/foo", + "as_subdir/foo/bar", + "as_another_subdir/foo/bar", + }, + expectedCopied: []string{ + "foo", + "in/subdir/of/another/depth/foo", + "as/subdir/of/another/depth/foo/bar", + "foo2", + "similar_in_subdir/foo2", + "similar_as_subdir/foo2/bar", + }, + }, + { + name: "name_at_depth2", + patterns: []string{ + "*/*/foo", + }, + expectedIgnored: []string{ + "in_subdir/of_depth2/foo", + "in_another_subdir/of_depth2/foo", + "as_subdir/of_depth2/foo/bar", + "as_another_subdir/of_depth2/foo/bar", + }, + expectedCopied: []string{ + "foo", + "in/subdir/of/another/depth/foo", + "as/subdir/of/another/depth/foo/bar", + "foo2", + "similar_in_subdir/of_depth2/foo2", + "similar_as_subdir/of_depth2/foo2/bar", + }, + }, + { + name: "depth1_under_name", + patterns: []string{ + "foo/*", + }, + expectedIgnored: []string{ + "foo/bar", + "foo/blabla", + "foo/with_subdir/bar", + "foo/with_subdir/blabla", + }, + expectedCopied: []string{ + "foo", + "as_subdir/foo/bar", + "as/subdir/of/another/depth/foo/bar", + "foo2/bar", + "foo2/with_subdir/bar", + }, + }, + } + runTestSet_ignoreFilter(t, testCases) +} diff --git a/cli/util/osfs.go b/cli/util/osfs.go new file mode 100644 index 000000000..cc0ea4a31 --- /dev/null +++ b/cli/util/osfs.go @@ -0,0 +1,25 @@ +package util + +import ( + "io/fs" + "os" +) + +type osFS struct{} + +// GetOsFS returns a default implementation of fs.FS interface. In general interface fs.FS +// should be added as an argument to any function where you need to be able to substitute +// non-default FS. The most obvious scenario is using mock FS for testing. In such a case +// while general code uses this default implementation, test code is able to substitute +// some mock FS (like fstest.MapFS). +func GetOsFS() fs.FS { + return osFS{} +} + +func (fs osFS) Open(name string) (fs.File, error) { + return os.Open(name) +} + +func (fs osFS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} diff --git a/test/integration/pack/test_pack.py b/test/integration/pack/test_pack.py index bba754919..e9d4894eb 100644 --- a/test/integration/pack/test_pack.py +++ b/test/integration/pack/test_pack.py @@ -1,11 +1,13 @@ import filecmp import glob +import itertools import os import re import shutil import stat import subprocess import tarfile +from pathlib import Path import pytest import yaml @@ -1875,3 +1877,114 @@ def test_pack_app_local_tarantool(tt_cmd, tmpdir_with_tarantool, tmp_path): build_output = tt_process.stdout.read() assert "Bundle is packed successfully" in build_output + + +@pytest.mark.slow +def test_pack_ignore(tt_cmd, tmp_path): + shutil.copytree(os.path.join(os.path.dirname(__file__), "test_bundles"), + tmp_path, symlinks=True, ignore=None, + copy_function=shutil.copy2, ignore_dangling_symlinks=True, + dirs_exist_ok=True) + + bundle_src = "single_app" + base_dir = tmp_path / bundle_src + + files_to_ignore = [ + " ", + "#", + "#hash_name1", + "name1", + "subdir/name1", + "deep/nested/subdir/name1", + "subdir2/name1/file", + "name2", + "subdir/name3_blabla", + "dir1/file", + "subdir/dir1/file", + "dir2/file", + "subdir/dir3_blabla/file", + "name11", + "subdir/name11", + "deep/nested/subdir/name11", + "dir/name11/file_bla", + "dir/name11/file_blabla", + "dir12/name12", + "subdir/dir12/name12", + "deep/nested/subdir/dir12/name12", + ] + + files_to_pack = [ + "# ", + "#comment", + "#hash_name_reincluded", + "mismatched_name1", + ".mismatched_name2", + "subdir/mismatched_name3", + "deep/nested/subdir/mismatched_name4", + "name2_reincluded", + "subdir/name3_reincluded1", + "dir4/mismatched_dir_name", + "subdir/as_file/dir1", + "subdir/mismatched_dir2/file", + "dir2_reincluded/file", + "subdir/dir3_reincluded1/file" + "name12", + "mismatched_parent_dir/name12", + "deep/nested/mismatched_parent_dir/name12", + ] + + ignore_patterns = [ + " ", + "\\ ", + "#comment", + "\\#", + "\\#hash_name*", + "!#hash_name_reincluded", + "name1", + "name[2-3]*", + "!name2_reincluded", + "!name3_reincluded[1-9]", + "dir1/", + "dir[2-3]*/", + "!dir2_reincluded/", + "!dir3_reincluded[1-9]/", + "**/name11", + "**/dir12/name12", + ] + + # Prepare .packignore layout. + (base_dir / ".packignore").write_text("\n".join(ignore_patterns) + "\n") + for f in itertools.chain(files_to_ignore, files_to_pack): + fpath = Path(base_dir, f) + fpath.parent.mkdir(parents=True, exist_ok=True) + fpath.write_text("") + + packages_wildcard = os.path.join(base_dir, "*.tar.gz") + packages = set(glob.glob(packages_wildcard)) + + rc, _ = run_command_and_get_output( + [tt_cmd, "pack", "tgz"], + cwd=base_dir, + env=dict(os.environ, PWD=base_dir), + ) + assert rc == 0 + + # Find the newly generated package. + new_packages = set(glob.glob(packages_wildcard)) - packages + assert len(new_packages) == 1 + package_file = Path(next(iter(new_packages))) + + extract_path = os.path.join(base_dir, "tmp") + os.mkdir(extract_path) + + tar = tarfile.open(package_file) + tar.extractall(extract_path) + tar.close() + + extract_base_dir = os.path.join(extract_path, bundle_src) + for file_path in [".packignore"] + files_to_ignore: + assert not os.path.exists(os.path.join(extract_base_dir, file_path)), \ + f"'{os.path.join(extract_base_dir, file_path)}' unexpectedly exists" + for file_path in files_to_pack: + assert os.path.exists(os.path.join(extract_base_dir, file_path)), \ + f"'{os.path.join(extract_base_dir, file_path)}' doesn't exist"