pack: support .packignore at ttenv level

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- `tt pack`: support nested `.packignore` at the root of tt environment.
+
 ### Changed
 
 ### Fixed

cli/pack/common.go

@@ -1,7 +1,6 @@
 package pack
 
 import (
-	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -173,19 +172,14 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
 	appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool {
 		return skipDefaults(srcInfo, src)
 	})
-	if f, err := ignoreFilter(util.GetOsFS(), filepath.Join(srcAppPath, ignoreFile)); err == nil {
-		appCopyFilters = append(appCopyFilters, f)
-	} else if !errors.Is(err, fs.ErrNotExist) {
-		return nil, fmt.Errorf("failed to load %q: %w", ignoreFile, err)
-	}
 
 	return func(srcInfo os.FileInfo, src, dest string) (bool, error) {
 		for _, shouldSkip := range appCopyFilters {
 			if shouldSkip(srcInfo, src) {
 				return true, nil
 			}
 		}
-		return false, nil
+		return packCtx.skipFunc(srcInfo, src, dest)
 	}, nil
 }
 
@@ -242,8 +236,9 @@ func copyEnvModules(bundleEnvPath string, packCtx *PackCtx, cliOpts, newOpts *co
 			if files, _ := dir.Readdir(1); len(files) == 0 {
 				return // No modules.
 			}
-			if err := copy.Copy(directory,
-				util.JoinPaths(bundleEnvPath, newOpts.Modules.Directories[0])); err != nil {
+			dest := util.JoinPaths(bundleEnvPath, newOpts.Modules.Directories[0])
+			err = copy.Copy(directory, dest, copy.Options{Skip: packCtx.skipFunc})
+			if err != nil {
 				log.Warnf("Failed to copy modules from %q: %s", directory, err)
 			}
 		}
@@ -416,8 +411,9 @@ func prepareBundle(cmdCtx *cmdcontext.CmdCtx, packCtx *PackCtx,
 
 	// Copy tcm config, if any.
 	if cmdCtx.Cli.TcmCli.ConfigPath != "" {
-		if err := copy.Copy(cmdCtx.Cli.TcmCli.ConfigPath,
-			path.Join(bundleEnvPath, path.Base(cmdCtx.Cli.TcmCli.ConfigPath))); err != nil {
+		dest := path.Join(bundleEnvPath, path.Base(cmdCtx.Cli.TcmCli.ConfigPath))
+		err = copy.Copy(cmdCtx.Cli.TcmCli.ConfigPath, dest, copy.Options{Skip: packCtx.skipFunc})
+		if err != nil {
 			return "", fmt.Errorf("failed copying tcm config: %s", err)
 		}
 	}
@@ -532,7 +528,8 @@ func copyArtifacts(packCtx PackCtx, basePath string, newOpts *config.CliOpts,
 			}
 			for _, toCopy := range copyInfo {
 				log.Debugf("Copying %q -> %q", toCopy.src, toCopy.dest)
-				if err := copy.Copy(toCopy.src, toCopy.dest); err != nil {
+				err := copy.Copy(toCopy.src, toCopy.dest, copy.Options{Skip: packCtx.skipFunc})
+				if err != nil {
 					log.Warnf("Failed to copy artifacts: %s", err)
 				}
 			}

cli/pack/ignore.go

@@ -3,6 +3,7 @@ package pack
 import (
 	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -12,6 +13,40 @@ import (
 	"strings"
 )
 
+type ignoreFilter struct {
+	patternsFileName string
+	rootNode         *ignoreFilterNode
+}
+
+func createIgnoreFilter(fsys fs.FS, rootDir, patternsFileName string) (*ignoreFilter, error) {
+	patternsFile := filepath.Join(rootDir, patternsFileName)
+	rootNode, err := createIgnoreFilterNode(fsys, patternsFile, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create root node: %w", err)
+	}
+
+	return &ignoreFilter{
+		patternsFileName: patternsFileName,
+		rootNode:         rootNode,
+	}, nil
+}
+
+func (f ignoreFilter) shouldSkip(info os.FileInfo, path string) bool {
+	if !strings.HasPrefix(path, f.rootNode.dir) {
+		return false
+	}
+
+	// Skip ignore pattern file itself.
+	if !info.IsDir() && info.Name() == f.patternsFileName {
+		return true
+	}
+
+	// Find the deepest node that is applicable to path.
+	node := f.rootNode.findNode(path)
+
+	return node.shouldSkip(info, path)
+}
+
 // ignorePattern corresponds to a single ignore pattern from .packignore file.
 type ignorePattern struct {
 	// re holds the "matching" part of ignore pattern (i.e. w/o trailing spaces, directory and
@@ -29,13 +64,13 @@ func turnEscapedToHexCode(s string, c rune) string {
 	return strings.ReplaceAll(s, `\`+string(c), fmt.Sprintf(`\x%x`, c))
 }
 
-func splitIgnorePattern(pattern string) (cleanPattern string, dirOnly, isNegate bool) {
+func splitIgnorePattern(pattern string) (matchingPattern string, dirOnly, isNegate bool) {
 	// Remove trailing spaces (unless escaped one).
-	cleanPattern = turnEscapedToHexCode(pattern, ' ')
-	cleanPattern = strings.TrimRight(cleanPattern, " ")
+	matchingPattern = turnEscapedToHexCode(pattern, ' ')
+	matchingPattern = strings.TrimRight(matchingPattern, " ")
 	// Parse negate and directory markers.
-	cleanPattern, dirOnly = strings.CutSuffix(cleanPattern, "/")
-	cleanPattern, isNegate = strings.CutPrefix(cleanPattern, "!")
+	matchingPattern, dirOnly = strings.CutSuffix(matchingPattern, "/")
+	matchingPattern, isNegate = strings.CutPrefix(matchingPattern, "!")
 	return
 }
 
@@ -45,10 +80,12 @@ func createIgnorePattern(pattern, basepath string) (ignorePattern, error) {
 	// occur as a part of `\\c` sequence which denotes '\' followed by <c>).
 	pattern = turnEscapedToHexCode(pattern, '\\')
 
-	cleanPattern, dirOnly, isNegate := splitIgnorePattern(pattern)
+	matchingPattern, dirOnly, isNegate := splitIgnorePattern(pattern)
+
+	// Translate matching pattern to regex expression.
+	// Translation is performed step by step in a certain sequence (see comments).
+	expr := matchingPattern
 
-	// Translate pattern to regex expression.
-	expr := cleanPattern
 	// Turn escaped '*' and '?' to their hex representation to simplify the translation.
 	expr = turnEscapedToHexCode(expr, '*')
 	expr = turnEscapedToHexCode(expr, '?')
@@ -58,22 +95,35 @@ func createIgnorePattern(pattern, basepath string) (ignorePattern, error) {
 		expr = strings.ReplaceAll(expr, "\\"+s, s)
 		expr = strings.ReplaceAll(expr, s, "\\"+s)
 	}
-	// Replace wildcards with the corresponding regex representation.
-	// Note that '{0,}' (not '*') is used while replacing '**' to avoid confusing
+
+	// Turn '?' to its regex representation here because '?' might be used in the subsequent
+	// transformations to specify non-capturing groups '(?:re)'.
+	expr = strings.ReplaceAll(expr, "?", "[^/]")
+
+	// Replace '**' wildcards with the corresponding regex representation.
+	// Note that '{0,}' rather than '*' is used while replacing '**' to avoid confusing
 	// in the subsequent replacement of a single '*'.
-	expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/){0,}")
+	expr = strings.ReplaceAll(expr, "/**/", "/(?:[^/]+/){0,}")
 	expr, found := strings.CutPrefix(expr, "**/")
-	if found || !strings.Contains(cleanPattern, "/") {
-		expr = "([^/]+/){0,}" + expr
+	if found || !strings.Contains(matchingPattern, "/") {
+		expr = "(?:[^/]+/){0,}" + expr
 	}
+
 	expr, found = strings.CutSuffix(expr, "/**")
+	// Turn '*' to its regex representation before injecting basepath to avoid confusing with '*'
+	// that basepath itself might contain (within basepath it's not a wildcard, but just '*').
+	expr = strings.ReplaceAll(expr, "*", "[^/]*")
+
+	// Construct final expression where the single captured group corresponds to the initial
+	// matchingPattern. This captured group might be used additionally to identify if some path
+	// matches pattern or not (see `ignorePattern.MatchPath`).
 	if found {
-		expr = expr + "/([^/]+/){0,}[^/]+"
+		expr = fmt.Sprintf("(%s(?:/[^/]+){1,})", basepath+expr)
+	} else {
+		expr = fmt.Sprintf("(%s)(?:/[^/]+){0,}", basepath+expr)
 	}
-	expr = strings.ReplaceAll(expr, "*", "[^/]*")
-	expr = strings.ReplaceAll(expr, "?", "[^/]")
 
-	re, err := regexp.Compile("^" + basepath + expr + "$")
+	re, err := regexp.Compile("^" + expr + "$")
 	if err != nil {
 		return ignorePattern{}, fmt.Errorf("failed to compile expression: %w", err)
 	}
@@ -85,6 +135,19 @@ func createIgnorePattern(pattern, basepath string) (ignorePattern, error) {
 	}, nil
 }
 
+// MatchPath checks if path matches this pattern.
+func (p ignorePattern) MatchPath(info fs.FileInfo, path string) bool {
+	submatches := p.re.FindStringSubmatch(path)
+	if submatches == nil {
+		return false
+	}
+
+	// For dirOnly pattern it is needed to check additionally that if path is not a directory
+	// it doesn't match directory part, which is represented with the single captured group
+	// (see `createIgnorePattern`).
+	return !p.dirOnly || info.IsDir() || submatches[1] != path
+}
+
 // loadIgnorePatterns reads ignore patterns from the patternsFile.
 func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error) {
 	contents, err := fs.ReadFile(fsys, patternsFile)
@@ -94,6 +157,14 @@ func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error
 
 	basepath, _ := filepath.Split(patternsFile)
 
+	// basepath is to be the part of every regex based on the patterns from this file,
+	// thus escape symbols \(){}[]+?*| that designate themselves in a path, but have special
+	// meaning in a regex.
+	// Note that '\' is escaped first to avoid confusing with '\' introduced with escaping of
+	// the rest ones.
+	basepath = strings.ReplaceAll(basepath, "\\", "\\\\")
+	basepath = regexp.MustCompile(`([(){}\[\]+?*|])`).ReplaceAllString(basepath, "\\$1")
+
 	var patterns []ignorePattern
 	s := bufio.NewScanner(bytes.NewReader(contents))
 	for s.Scan() {
@@ -109,31 +180,89 @@ func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error
 
 		patterns = append(patterns, p)
 	}
+
 	return patterns, nil
 }
 
-// ignoreFilter returns filter function that implements .gitignore approach of filtering files.
-func ignoreFilter(fsys fs.FS, patternsFile string) (skipFilter, error) {
+type ignoreFilterNode struct {
+	dir      string
+	patterns []ignorePattern
+	parent   *ignoreFilterNode
+	children []*ignoreFilterNode
+}
+
+func createIgnoreFilterNode(fsys fs.FS, patternsFile string, parent *ignoreFilterNode) (
+	*ignoreFilterNode,
+	error,
+) {
 	patterns, err := loadIgnorePatterns(fsys, patternsFile)
 	if err != nil {
-		return nil, err
+		// There is a special condition for the root node that allows patternsFile to be missed
+		// in a root-directory. The other nodes don't have this option since for all of them
+		// construction is initiated by the presence of corresponding patternsFile.
+		if parent != nil || !errors.Is(err, fs.ErrNotExist) {
+			return nil, fmt.Errorf("failed to read patterns: %w", err)
+		}
 	}
 
 	// According to .gitignore documentation "the last matching pattern decides the outcome"
 	// so we need to iterate in reverse order until the first match.
 	slices.Reverse(patterns)
 
-	return func(srcInfo os.FileInfo, src string) bool {
-		// Skip ignore file itself.
-		if src == patternsFile {
-			return true
-		}
-		for _, p := range patterns {
-			isApplicable := srcInfo.IsDir() || !p.dirOnly
-			if isApplicable && p.re.MatchString(src) {
-				return !p.isNegate
+	node := &ignoreFilterNode{
+		dir:      filepath.Dir(patternsFile),
+		patterns: patterns,
+		parent:   parent,
+		children: nil,
+	}
+
+	// Walk down the directories to find all the children.
+	root := filepath.Dir(patternsFile)
+	patternsFileName := filepath.Base(patternsFile)
+	err = fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error {
+		if d.Name() == patternsFileName && !d.IsDir() && path != patternsFile {
+			child, err := createIgnoreFilterNode(fsys, path, node)
+			if err != nil {
+				return fmt.Errorf("failed to create node for %q: %w", path, err)
 			}
+			node.children = append(node.children, child)
+			return fs.SkipDir
 		}
-		return false
-	}, nil
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed walking %q: %w", root, err)
+	}
+
+	return node, nil
+}
+
+// findNode finds the deepest node that is applicable to path.
+func (node *ignoreFilterNode) findNode(path string) *ignoreFilterNode {
+	i := slices.IndexFunc(node.children, func(child *ignoreFilterNode) bool {
+		return strings.HasPrefix(path, child.dir)
+	})
+	if i != -1 {
+		return node.children[i].findNode(path)
+	}
+
+	return node
+}
+
+// shouldSkip searches the tree from this node towards root for the first pattern
+// that matches path and returns true if it's regular one (i.e. non-negative)
+// and false otherwise. If no match found it returns false as well.
+func (node *ignoreFilterNode) shouldSkip(info fs.FileInfo, path string) bool {
+	i := slices.IndexFunc(node.patterns, func(p ignorePattern) bool {
+		return p.MatchPath(info, path)
+	})
+	if i != -1 {
+		return !node.patterns[i].isNegate
+	}
+
+	if node.parent != nil {
+		return node.parent.shouldSkip(info, path)
+	}
+
+	return false
 }