Skip to content

Commit

Permalink
pack: support .packignore
Browse files Browse the repository at this point in the history
In some cases, there are files that may be useful, but should not
be part of artifact. The ability to add files and directories to
the .packignore file has been added, which allows you to ignore
these files and directories when packing.

Closes #812

@TarantoolBot document
Title: `tt pack` support `.packignore`

Use `.packignore` in the same way as `.gitignore` allows to exclude
unnecessary files while preparing application package with `tt pack
command.
  • Loading branch information
elhimov committed Mar 6, 2025
1 parent c06c390 commit e14d838
Show file tree
Hide file tree
Showing 6 changed files with 1,227 additions and 12 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
### Added

- `tt aeon connect`: add connection from the `app:insance_name`.
- `tt pack `: support `.packignore` file to specify files that should not be included
in package (works the same as `.gitignore`).

### Changed

Expand Down
34 changes: 22 additions & 12 deletions cli/pack/common.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package pack

import (
"errors"
"fmt"
"io/fs"
"os"
Expand Down Expand Up @@ -33,6 +34,8 @@ const (
versionLuaFileName = "VERSION.lua"

rocksManifestPath = ".rocks/share/tarantool/rocks/manifest"

ignoreFile = ".packignore"
)

var (
Expand All @@ -51,6 +54,8 @@ var (
}
)

type skipFilter func(srcInfo os.FileInfo, src string) bool

type RocksVersions map[string][]string

// packFileInfo contains information to set for files/dirs in rpm/deb packages.
Expand All @@ -76,9 +81,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool {
}

// appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts.
func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func(
srcInfo os.FileInfo, src string) bool {
filters := make([]func(srcInfo os.FileInfo, src string) bool, 0)
func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter {
filters := make([]skipFilter, 0)
if cliOpts.App == nil {
return filters
}
Expand All @@ -102,9 +106,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func(
}

// ttEnvironmentFilters prepares a slice of filters for tt environment directories/files.
func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
srcInfo os.FileInfo, src string) bool {
filters := make([]func(srcInfo os.FileInfo, src string) bool, 0)
func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter {
filters := make([]skipFilter, 0)
if cliOpts == nil {
return filters
}
Expand Down Expand Up @@ -139,10 +142,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
}

// previousPackageFilters returns filters for the previously built packages.
func previousPackageFilters(packCtx *PackCtx) []func(
srcInfo os.FileInfo, src string) bool {
func previousPackageFilters(packCtx *PackCtx) []skipFilter {
pkgName := packCtx.Name
return []func(srcInfo os.FileInfo, src string) bool{
return []skipFilter{
func(srcInfo os.FileInfo, src string) bool {
name := srcInfo.Name()
if strings.HasPrefix(name, pkgName) {
Expand All @@ -159,13 +161,18 @@ func previousPackageFilters(packCtx *PackCtx) []func(

// appSrcCopySkip returns a filter func to filter out artifacts paths.
func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) {
srcAppPath string) (func(srcinfo os.FileInfo, src, dest string) (bool, error), error) {
appCopyFilters := appArtifactsFilters(cliOpts, srcAppPath)
appCopyFilters = append(appCopyFilters, ttEnvironmentFilters(packCtx, cliOpts)...)
appCopyFilters = append(appCopyFilters, previousPackageFilters(packCtx)...)
appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool {
return skipDefaults(srcInfo, src)
})
if f, err := ignoreFilter(util.GetOsFS(), filepath.Join(srcAppPath, ignoreFile)); err == nil {
appCopyFilters = append(appCopyFilters, f)
} else if !errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("failed to load %q: %w", ignoreFile, err)
}

return func(srcinfo os.FileInfo, src, dest string) (bool, error) {
for _, shouldSkip := range appCopyFilters {
Expand All @@ -174,7 +181,7 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
}
}
return false, nil
}
}, nil
}

// getAppNamesToPack generates application names list to pack.
Expand Down Expand Up @@ -430,7 +437,10 @@ func copyAppSrc(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath, dstAppPat
return err
}

skipFunc := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath)
skipFunc, err := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath)
if err != nil {
return err
}

// Copying application.
log.Debugf("Copying application source %q -> %q", resolvedAppPath, dstAppPath)
Expand Down
132 changes: 132 additions & 0 deletions cli/pack/ignore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package pack

import (
"bufio"
"bytes"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
)

type ignorePattern struct {
re *regexp.Regexp
dirOnly bool
isNegate bool
}

func turnEscapedToHexCode(s string, c rune) string {
return strings.ReplaceAll(s, `\`+string(c), fmt.Sprintf(`\x%x`, c))
}

func splitIgnorePattern(pattern string) (cleanPattern string, dirOnly bool, isNegate bool) {
// First, get rid of `\\` to simplify further handling of escaped sequences.
// From now on any `\c` always means escaped 'c' (previously it might also
// occur as a part of `\\c` sequence which denotes '\' followed by <c>).
cleanPattern = turnEscapedToHexCode(pattern, '\\')

// Remove trailing spaces (unless escaped one).
cleanPattern = turnEscapedToHexCode(cleanPattern, ' ')
cleanPattern = strings.TrimRight(cleanPattern, " ")

cleanPattern, dirOnly = strings.CutSuffix(cleanPattern, "/")
cleanPattern, isNegate = strings.CutPrefix(cleanPattern, "!")
return
}

func createIgnorePattern(pattern string, basepath string) (ignorePattern, error) {
cleanPattern, dirOnly, isNegate := splitIgnorePattern(pattern)

// Translate pattern to regex expression.
expr := cleanPattern
// Turn escaped '*' and '?' to their hex representation to simplify the translation.
expr = turnEscapedToHexCode(expr, '*')
expr = turnEscapedToHexCode(expr, '?')
// Escape symbols that designate themselves in pattern, but have special meaning in regex.
for _, s := range []string{"(", ")", "{", "}", "+"} {
// Do unescape first to avoid double escaping of the ones that are already escaped.
expr = strings.ReplaceAll(expr, "\\"+s, s)
expr = strings.ReplaceAll(expr, s, "\\"+s)
}
// Replace wildcards with the corresponding regex representation.
// Note that '{0,}' (not '*') is used while replacing '**' to avoid confusing
// in the subsequent replacement of a single '*'.
expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/){0,}")
expr, found := strings.CutPrefix(expr, "**/")
if found || !strings.Contains(cleanPattern, "/") {
expr = "([^/]+/){0,}" + expr
}
expr, found = strings.CutSuffix(expr, "/**")
if found {
expr = expr + "/([^/]+/){0,}[^/]+"
}
expr = strings.ReplaceAll(expr, "*", "[^/]*")
expr = strings.ReplaceAll(expr, "?", "[^/]")

re, err := regexp.Compile("^" + basepath + expr + "$")
if err != nil {
return ignorePattern{}, fmt.Errorf("failed to compile expression: %w", err)
}

return ignorePattern{
re: re,
dirOnly: dirOnly,
isNegate: isNegate,
}, nil
}

// loadIgnorePatterns reads ignore patterns from the patternsFile.
func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error) {
contents, err := fs.ReadFile(fsys, patternsFile)
if err != nil {
return nil, err
}

basepath, _ := filepath.Split(patternsFile)

var patterns []ignorePattern
s := bufio.NewScanner(bytes.NewReader(contents))
for s.Scan() {
pattern := s.Text()
if pattern == "" || strings.HasPrefix(pattern, "#") {
continue
}

p, err := createIgnorePattern(pattern, basepath)
if err != nil {
return nil, err
}

patterns = append(patterns, p)
}
return patterns, nil
}

// ignoreFilter returns filter function that implements .gitignore approach of filtering files.
func ignoreFilter(fsys fs.FS, patternsFile string) (skipFilter, error) {
patterns, err := loadIgnorePatterns(fsys, patternsFile)
if err != nil {
return nil, err
}

// According to .gitignore documentation "the last matching pattern decides the outcome"
// so we need to iterate in reverse order until the first match.
slices.Reverse(patterns)

return func(srcInfo os.FileInfo, src string) bool {
// Skip ignore file itself.
if src == patternsFile {
return true
}
for _, p := range patterns {
isApplicable := srcInfo.IsDir() || !p.dirOnly
if isApplicable && p.re.MatchString(src) {
return !p.isNegate
}
}
return false
}, nil
}
Loading

0 comments on commit e14d838

Please sign in to comment.