diff --git a/.gitignore b/.gitignore index 9c4d9f1ef..17c882487 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ # Ignore components vendored during tests tests/fixtures/scenarios/vendor/components/** +tests/fixtures/scenarios/vendor-globs/components/** examples/demo-vendoring/components/** diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go new file mode 100644 index 000000000..c7d5636e2 --- /dev/null +++ b/internal/exec/copy_glob.go @@ -0,0 +1,363 @@ +package exec + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + log "github.com/charmbracelet/log" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. +) + +// Named constants to avoid literal duplication. +const ( + logKeyPath = "path" + logKeyError = "error" + shallowCopySuffix = "/*" +) + +// copyFile copies a single file from src to dst while preserving file permissions. +func copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer sourceFile.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return fmt.Errorf("creating destination directory for %q: %w", dst, err) + } + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating destination file %q: %w", dst, err) + } + defer destinationFile.Close() + + if _, err := io.Copy(destinationFile, sourceFile); err != nil { + return fmt.Errorf("copying content from %q to %q: %w", src, dst, err) + } + + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("getting file info for %q: %w", src, err) + } + if err := os.Chmod(dst, info.Mode()); err != nil { + return fmt.Errorf("setting permissions on %q: %w", dst, err) + } + return nil +} + +// shouldExcludePath checks exclusion patterns for a given relative path and file info. +func shouldExcludePath(info os.FileInfo, relPath string, excluded []string) bool { + for _, pattern := range excluded { + // Check plain relative path. + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + log.Debug("Error matching exclusion pattern", logKeyPath, relPath, logKeyError, err) + continue + } + if matched { + log.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, "pattern", pattern) + return true + } + // If a directory, also check with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, relPath+"/") + if err != nil { + log.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, logKeyPath, relPath+"/", logKeyError, err) + continue + } + if matched { + log.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", "pattern", pattern) + return true + } + } + } + return false +} + +// shouldIncludePath checks whether a file should be included based on inclusion patterns. +func shouldIncludePath(info os.FileInfo, relPath string, included []string) bool { + // Directories are generally handled by recursion. + if len(included) == 0 || info.IsDir() { + return true + } + for _, pattern := range included { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + log.Debug("Error matching inclusion pattern", "pattern", pattern, logKeyPath, relPath, logKeyError, err) + continue + } + if matched { + log.Debug("Including path due to inclusion pattern", logKeyPath, relPath, "pattern", pattern) + return true + } + } + log.Debug("Excluding path because it does not match any inclusion pattern", logKeyPath, relPath) + return false +} + +// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { + if info.Name() == ".git" { + return true, nil + } + relPath, err := filepath.Rel(baseDir, srcPath) + if err != nil { + log.Debug("Error computing relative path", "srcPath", srcPath, logKeyError, err) + return true, nil // treat error as a signal to skip + } + relPath = filepath.ToSlash(relPath) + if shouldExcludePath(info, relPath, excluded) { + return true, nil + } + if !shouldIncludePath(info, relPath, included) { + return true, nil + } + return false, nil +} + +// processDirEntry handles a single directory entry for copyDirRecursive. +func processDirEntry(entry os.DirEntry, srcDir, dstDir, baseDir string, excluded, included []string) error { + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) + if err != nil { + return err + } + if skip { + log.Debug("Skipping entry", "srcPath", srcPath) + return nil + } + + // Skip symlinks. + if info.Mode()&os.ModeSymlink != 0 { + log.Debug("Skipping symlink", logKeyPath, srcPath) + return nil + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + return copyDirRecursive(srcPath, dstPath, baseDir, excluded, included) + } + return copyFile(srcPath, dstPath) +} + +// copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + if err := processDirEntry(entry, srcDir, dstDir, baseDir, excluded, included); err != nil { + return err + } + } + return nil +} + +// shouldSkipPrefixEntry checks exclusion patterns for copyDirRecursiveWithPrefix. +func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []string) bool { + for _, pattern := range excluded { + matched, err := u.PathMatch(pattern, fullRelPath) + if err != nil { + log.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, logKeyPath, fullRelPath, logKeyError, err) + continue + } + if matched { + log.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, "pattern", pattern) + return true + } + if info.IsDir() { + matched, err = u.PathMatch(pattern, fullRelPath+"/") + if err != nil { + log.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, logKeyPath, fullRelPath+"/", logKeyError, err) + continue + } + if matched { + log.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", "pattern", pattern) + return true + } + } + } + return false +} + +// processPrefixEntry handles a single entry for copyDirRecursiveWithPrefix. +func processPrefixEntry(entry os.DirEntry, srcDir, dstDir, globalBase, prefix string, excluded []string) error { + fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + if entry.Name() == ".git" { + log.Debug("Skipping .git directory", logKeyPath, fullRelPath) + return nil + } + + if shouldSkipPrefixEntry(info, fullRelPath, excluded) { + return nil + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + return copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded) + } + return copyFile(srcPath, dstPath) +} + +// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. +func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + if err := processPrefixEntry(entry, srcDir, dstDir, globalBase, prefix, excluded); err != nil { + return err + } + } + return nil +} + +// getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. +func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + matches, err := u.GetGlobMatches(fullPattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) + } + if len(matches) == 0 { + // If the pattern ends with "/*" (shallow copy indicator) do not fallback. + if strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") { + log.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) + return []string{}, nil + } + // Fallback for non-shallow patterns. + if strings.HasSuffix(pattern, shallowCopySuffix) { + recursivePattern := strings.TrimSuffix(pattern, shallowCopySuffix) + "/**" + fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) + matches, err = u.GetGlobMatches(fullRecursivePattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) + } + if len(matches) == 0 { + log.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) + return []string{}, nil + } + return matches, nil + } + log.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) + return []string{}, nil + } + return matches, nil +} + +// isShallowPattern determines if a pattern indicates a shallow copy. +func isShallowPattern(pattern string) bool { + return strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") +} + +// processMatch handles a single file/directory match for copyToTargetWithPatterns. +func processMatch(sourceDir, targetPath, file string, shallow bool, excluded []string) error { + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("computing relative path for %q: %w", file, err) + } + relPath = filepath.ToSlash(relPath) + if shouldExcludePath(info, relPath, excluded) { + return nil + } + + dstPath := filepath.Join(targetPath, relPath) + if info.IsDir() { + if shallow { + log.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) + return nil + } + return copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, excluded) + } + return copyFile(file, dstPath) +} + +// processIncludedPattern handles all matches for one inclusion pattern. +func processIncludedPattern(sourceDir, targetPath, pattern string, excluded []string) error { + shallow := isShallowPattern(pattern) + matches, err := getMatchesForPattern(sourceDir, pattern) + if err != nil { + log.Debug("Warning: error getting matches for pattern", "pattern", pattern, logKeyError, err) + return nil + } + if len(matches) == 0 { + log.Debug("No files matched the inclusion pattern", "pattern", pattern) + return nil + } + for _, file := range matches { + if err := processMatch(sourceDir, targetPath, file, shallow, excluded); err != nil { + return err + } + } + return nil +} + +// copyToTargetWithPatterns copies the contents from sourceDir to targetPath, +// applying inclusion and exclusion patterns from the vendor source configuration. +func copyToTargetWithPatterns( + sourceDir, targetPath string, + s *schema.AtmosVendorSource, + sourceIsLocalFile bool, + uri string, +) error { + if sourceIsLocalFile && filepath.Ext(targetPath) == "" { + targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) + } + log.Debug("Copying files", "source", sourceDir, "target", targetPath) + if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { + return fmt.Errorf("creating target directory %q: %w", targetPath, err) + } + + // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library. + if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { + log.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") + return cp.Copy(sourceDir, targetPath) + } + + // Process each inclusion pattern. + for _, pattern := range s.IncludedPaths { + if err := processIncludedPattern(sourceDir, targetPath, pattern, s.ExcludedPaths); err != nil { + return err + } + } + + // If no inclusion patterns are defined, copy everything except those matching excluded items. + if len(s.IncludedPaths) == 0 { + if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) + } + } + return nil +} diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 29e9a415e..4e2cb1db2 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -8,9 +8,11 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" "time" + log "github.com/charmbracelet/log" "github.com/google/uuid" "github.com/hashicorp/go-getter" @@ -64,81 +66,213 @@ func IsValidScheme(scheme string) bool { // CustomGitHubDetector intercepts GitHub URLs and transforms them // into something like git::https://@github.com/... so we can // do a git-based clone with a token. -type CustomGitHubDetector struct { +type CustomGitDetector struct { AtmosConfig schema.AtmosConfiguration + source string } // Detect implements the getter.Detector interface for go-getter v1. -func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { +func (d *CustomGitDetector) Detect(src, _ string) (string, bool, error) { + log.Debug("CustomGitDetector.Detect called") + if len(src) == 0 { return "", false, nil } - if !strings.Contains(src, "://") { - src = "https://" + src - } + // Ensure the URL has an explicit scheme. + src = d.ensureScheme(src) + // Parse the URL to extract the host and path. parsedURL, err := url.Parse(src) if err != nil { - u.LogDebug(fmt.Sprintf("Failed to parse URL %q: %v\n", src, err)) - return "", false, fmt.Errorf("failed to parse URL %q: %w", src, err) + maskedSrc, _ := u.MaskBasicAuth(src) + log.Debug("Failed to parse URL", keyURL, maskedSrc, "error", err) + return "", false, fmt.Errorf("failed to parse URL %q: %w", maskedSrc, err) } - if strings.ToLower(parsedURL.Host) != "github.com" { - u.LogDebug(fmt.Sprintf("Host is %q, not 'github.com', skipping token injection\n", parsedURL.Host)) - return "", false, nil - } + // Normalize the path. + d.normalizePath(parsedURL) - parts := strings.SplitN(parsedURL.Path, "/", 4) - if len(parts) < 3 { - u.LogDebug(fmt.Sprintf("URL path %q doesn't look like /owner/repo\n", parsedURL.Path)) - return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) + // Adjust host check to support GitHub, Bitbucket, GitLab, etc. + host := strings.ToLower(parsedURL.Host) + if host != "github.com" && host != "bitbucket.org" && host != "gitlab.com" { + log.Debug("Skipping token injection for a unsupported host", "host", parsedURL.Host) } - atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") - gitHubToken := os.Getenv("GITHUB_TOKEN") + log.Debug("Reading config param", "InjectGithubToken", d.AtmosConfig.Settings.InjectGithubToken) + // Inject token if available. + d.injectToken(parsedURL, host) - var usedToken string - var tokenSource string + // Adjust subdirectory if needed. + d.adjustSubdir(parsedURL, d.source) + + // Set "depth=1" for a shallow clone if not specified. + q := parsedURL.Query() + if _, exists := q["depth"]; !exists { + q.Set("depth", "1") + } + parsedURL.RawQuery = q.Encode() - // 1. If ATMOS_GITHUB_TOKEN is set, always use that - if atmosGitHubToken != "" { - usedToken = atmosGitHubToken - tokenSource = "ATMOS_GITHUB_TOKEN" - u.LogDebug("ATMOS_GITHUB_TOKEN is set\n") + finalURL := "git::" + parsedURL.String() + maskedFinal, err := u.MaskBasicAuth(strings.TrimPrefix(finalURL, "git::")) + if err != nil { + log.Debug("Masking failed", "error", err) } else { - // 2. Otherwise, only inject GITHUB_TOKEN if cfg.Settings.InjectGithubToken == true - if d.AtmosConfig.Settings.InjectGithubToken && gitHubToken != "" { - usedToken = gitHubToken - tokenSource = "GITHUB_TOKEN" - u.LogTrace("InjectGithubToken=true and GITHUB_TOKEN is set, using it\n") - } else { - u.LogTrace("No ATMOS_GITHUB_TOKEN or GITHUB_TOKEN found\n") + log.Debug("Final URL", "final_url", "git::"+maskedFinal) + } + + return finalURL, true, nil +} + +const ( + // Named constants for regex match indices. + matchIndexUser = 1 + matchIndexHost = 3 + matchIndexPath = 4 + matchIndexSuffix = 5 + matchIndexExtra = 6 + + // Key for logging repeated "url" field. + keyURL = "url" +) + +// ensureScheme checks for an explicit scheme and rewrites SCP-style URLs if needed. +// This version no longer returns an error since it never produces one. +func (d *CustomGitDetector) ensureScheme(src string) string { + if !strings.Contains(src, "://") { + if newSrc, rewritten := rewriteSCPURL(src); rewritten { + maskedOld, _ := u.MaskBasicAuth(src) + maskedNew, _ := u.MaskBasicAuth(newSrc) + log.Debug("Rewriting SCP-style SSH URL", "old_url", maskedOld, "new_url", maskedNew) + return newSrc } + src = "https://" + src + maskedSrc, _ := u.MaskBasicAuth(src) + log.Debug("Defaulting to https scheme", keyURL, maskedSrc) } + return src +} - if usedToken != "" { - user := parsedURL.User.Username() - pass, _ := parsedURL.User.Password() - if user == "" && pass == "" { - u.LogDebug(fmt.Sprintf("Injecting token from %s for %s\n", tokenSource, src)) - parsedURL.User = url.UserPassword("x-access-token", usedToken) +// rewriteSCPURL rewrites SCP-style URLs to a proper SSH URL if they match the expected pattern. +// Returns the rewritten URL and a boolean indicating if rewriting occurred. +func rewriteSCPURL(src string) (string, bool) { + scpPattern := regexp.MustCompile(`^(([\w.-]+)@)?([\w.-]+\.[\w.-]+):([\w./-]+)(\.git)?(.*)$`) + if scpPattern.MatchString(src) { + matches := scpPattern.FindStringSubmatch(src) + newSrc := "ssh://" + if matches[matchIndexUser] != "" { + newSrc += matches[matchIndexUser] // includes username and '@' + } + newSrc += matches[matchIndexHost] + "/" + matches[matchIndexPath] + if matches[matchIndexSuffix] != "" { + newSrc += matches[matchIndexSuffix] + } + if matches[matchIndexExtra] != "" { + newSrc += matches[matchIndexExtra] + } + return newSrc, true + } + return "", false +} + +// normalizePath converts the URL path to use forward slashes. +func (d *CustomGitDetector) normalizePath(parsedURL *url.URL) { + unescapedPath, err := url.PathUnescape(parsedURL.Path) + if err == nil { + parsedURL.Path = filepath.ToSlash(unescapedPath) + } else { + parsedURL.Path = filepath.ToSlash(parsedURL.Path) + } +} + +// injectToken injects a token into the URL if available. +func (d *CustomGitDetector) injectToken(parsedURL *url.URL, host string) { + token, tokenSource := d.resolveToken(host) + if token != "" { + defaultUsername := getDefaultUsername(host) + parsedURL.User = url.UserPassword(defaultUsername, token) + maskedURL, _ := u.MaskBasicAuth(parsedURL.String()) + log.Debug("Injected token", "env", tokenSource, keyURL, maskedURL) + } else { + log.Debug("No token found for injection") + } +} + +// resolveToken returns the token and its source based on the host. +func (d *CustomGitDetector) resolveToken(host string) (string, string) { + var token, tokenSource string + switch host { + case "github.com": + if d.AtmosConfig.Settings.InjectGithubToken { + tokenSource = "ATMOS_GITHUB_TOKEN" + token = os.Getenv(tokenSource) + if token == "" { + tokenSource = "GITHUB_TOKEN" + token = os.Getenv(tokenSource) + } } else { - u.LogDebug("Credentials found, skipping token injection\n") + tokenSource = "GITHUB_TOKEN" + token = os.Getenv(tokenSource) + } + case "bitbucket.org": + tokenSource = "BITBUCKET_TOKEN" + token = os.Getenv(tokenSource) + if token == "" { + tokenSource = "ATMOS_BITBUCKET_TOKEN" + token = os.Getenv(tokenSource) + } + case "gitlab.com": + tokenSource = "GITLAB_TOKEN" + token = os.Getenv(tokenSource) + if token == "" { + tokenSource = "ATMOS_GITLAB_TOKEN" + token = os.Getenv(tokenSource) } } + return token, tokenSource +} - finalURL := "git::" + parsedURL.String() +// getDefaultUsername returns the default username for token injection based on the host. +func getDefaultUsername(host string) string { + switch host { + case "github.com": + return "x-access-token" + case "gitlab.com": + return "oauth2" + case "bitbucket.org": + defaultUsername := os.Getenv("ATMOS_BITBUCKET_USERNAME") + if defaultUsername == "" { + defaultUsername = os.Getenv("BITBUCKET_USERNAME") + if defaultUsername == "" { + return "x-token-auth" + } + } + log.Debug("Using Bitbucket username", "username", defaultUsername) + return defaultUsername + default: + return "x-access-token" + } +} - return finalURL, true, nil +// adjustSubdir appends "//." to the path if no subdirectory is specified. +func (d *CustomGitDetector) adjustSubdir(parsedURL *url.URL, source string) { + normalizedSource := filepath.ToSlash(source) + if normalizedSource != "" && !strings.Contains(normalizedSource, "//") { + parts := strings.SplitN(parsedURL.Path, "/", 4) + if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { + maskedSrc, _ := u.MaskBasicAuth(source) + log.Debug("Detected top-level repo with no subdir: appending '//.'", keyURL, maskedSrc) + parsedURL.Path += "//." + } + } } // RegisterCustomDetectors prepends the custom detector so it runs before // the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { +func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, + &CustomGitDetector{AtmosConfig: atmosConfig, source: source}, }, getter.Detectors..., ) @@ -155,8 +289,11 @@ func GoGetterGet( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors - RegisterCustomDetectors(atmosConfig) + // Register custom detectors, passing the original `src` to the CustomGitHubDetector. + // go-getter typically strips subdirectories before calling the detector, so the + // unaltered source is needed to identify whether a top-level repository or a + // subdirectory was specified (e.g., for appending "//." only when no subdir is present). + RegisterCustomDetectors(atmosConfig, src) client := &getter.Client{ Ctx: ctx, @@ -164,8 +301,18 @@ func GoGetterGet( // Destination where the files will be stored. This will create the directory if it doesn't exist Dst: dest, Mode: clientMode, - } + Getters: map[string]getter.Getter{ + // Overriding 'git' + "git": &CustomGitGetter{}, + "file": &getter.FileGetter{}, + "hg": &getter.HgGetter{}, + "http": &getter.HttpGetter{}, + "https": &getter.HttpGetter{}, + // "s3": &getter.S3Getter{}, // add as needed + // "gcs": &getter.GCSGetter{}, + }, + } if err := client.Get(); err != nil { return err } @@ -173,6 +320,39 @@ func GoGetterGet( return nil } +// CustomGitGetter is a custom getter for git (git::) that removes symlinks. +type CustomGitGetter struct { + getter.GitGetter +} + +// Implements the custom getter logic removing symlinks. +func (c *CustomGitGetter) Get(dst string, url *url.URL) error { + // Normal clone + if err := c.GitGetter.Get(dst, url); err != nil { + return err + } + // Remove symlinks + return removeSymlinks(dst) +} + +// removeSymlinks walks the directory and removes any symlinks +// it encounters. +func removeSymlinks(root string) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + // Symlinks are removed for the entire repo, regardless if there are any subfolders specified + // Thus logging is disabled + // u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) + // It's a symlink, remove it + return os.Remove(path) + } + return nil + }) +} + // DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index b860aa7c9..fdfb68ffc 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -308,7 +308,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - if err := copyToTarget(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to copy package: %w", err), name: p.name, diff --git a/pkg/utils/url_utils.go b/pkg/utils/url_utils.go new file mode 100644 index 000000000..e176c377f --- /dev/null +++ b/pkg/utils/url_utils.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "net/url" +) + +// MaskBasicAuth replaces the username and password in a URL with "xxx" if present. +func MaskBasicAuth(rawURL string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + if parsedURL.User != nil { + parsedURL.User = url.UserPassword("xxx", "xxx") + } + + return parsedURL.String(), nil +} diff --git a/tests/cli_test.go b/tests/cli_test.go index acd33b311..e30e4f499 100644 --- a/tests/cli_test.go +++ b/tests/cli_test.go @@ -46,13 +46,14 @@ var ( var logger *log.Logger type Expectation struct { - Stdout []MatchPattern `yaml:"stdout"` // Expected stdout output - Stderr []MatchPattern `yaml:"stderr"` // Expected stderr output - ExitCode int `yaml:"exit_code"` // Expected exit code - FileExists []string `yaml:"file_exists"` // Files to validate - FileContains map[string][]MatchPattern `yaml:"file_contains"` // File contents to validate (file to patterns map) - Diff []string `yaml:"diff"` // Acceptable differences in snapshot - Timeout string `yaml:"timeout"` // Maximum execution time as a string, e.g., "1s", "1m", "1h", or a number (seconds) + Stdout []MatchPattern `yaml:"stdout"` // Expected stdout output + Stderr []MatchPattern `yaml:"stderr"` // Expected stderr output + ExitCode int `yaml:"exit_code"` // Expected exit code + FileExists []string `yaml:"file_exists"` // Files to validate + FileNotExists []string `yaml:"file_not_exists"` // Files that should not exist + FileContains map[string][]MatchPattern `yaml:"file_contains"` // File contents to validate (file to patterns map) + Diff []string `yaml:"diff"` // Acceptable differences in snapshot + Timeout string `yaml:"timeout"` // Maximum execution time as a string, e.g., "1s", "1m", "1h", or a number (seconds) } type TestCase struct { Name string `yaml:"name"` // Name of the test @@ -658,6 +659,11 @@ func runCLICommandTest(t *testing.T, tc TestCase) { t.Errorf("Description: %s", tc.Description) } + // Validate file not existence + if !verifyFileNotExists(t, tc.Expect.FileNotExists) { + t.Errorf("Description: %s", tc.Description) + } + // Validate file contents if !verifyFileContains(t, tc.Expect.FileContains) { t.Errorf("Description: %s", tc.Description) @@ -766,6 +772,20 @@ func verifyFileExists(t *testing.T, files []string) bool { return success } +func verifyFileNotExists(t *testing.T, files []string) bool { + success := true + for _, file := range files { + if _, err := os.Stat(file); err == nil { + t.Errorf("Reason: File %q exists but it should not.", file) + success = false + } else if !errors.Is(err, os.ErrNotExist) { + t.Errorf("Reason: Unexpected error checking file %q: %v", file, err) + success = false + } + } + return success +} + func verifyFileContains(t *testing.T, filePatterns map[string][]MatchPattern) bool { success := true for file, patterns := range filePatterns { diff --git a/tests/fixtures/scenarios/vendor-globs/atmos.yaml b/tests/fixtures/scenarios/vendor-globs/atmos.yaml new file mode 100644 index 000000000..0f0506e81 --- /dev/null +++ b/tests/fixtures/scenarios/vendor-globs/atmos.yaml @@ -0,0 +1,40 @@ +base_path: "./" + +components: + terraform: + base_path: "components/terraform" + apply_auto_approve: false + deploy_run_init: true + init_run_reconfigure: true + auto_generate_backend_file: false + +stacks: + base_path: "stacks" + included_paths: + - "deploy/**/*" + excluded_paths: + - "**/_defaults.yaml" + name_pattern: "{stage}" + +vendor: + # Single file + base_path: "./vendor.yaml" + + # Directory with multiple files + #base_path: "./vendor" + + # Absolute path + #base_path: "vendor.d/vendor1.yaml" + +logs: + file: "/dev/stderr" + level: Info + +# Custom CLI commands + +# No arguments or flags are required +commands: +- name: "test" + description: "Run all tests" + steps: + - atmos vendor pull --everything diff --git a/tests/fixtures/scenarios/vendor-globs/vendor.yaml b/tests/fixtures/scenarios/vendor-globs/vendor.yaml new file mode 100644 index 000000000..061c06d19 --- /dev/null +++ b/tests/fixtures/scenarios/vendor-globs/vendor.yaml @@ -0,0 +1,43 @@ +apiVersion: atmos/v1 +kind: AtmosVendorConfig +metadata: + name: demo-vendoring + description: Atmos vendoring manifest for Atmos demo component library +spec: + # Import other vendor manifests, if necessary + imports: [] + + sources: + - component: "test globs" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/{demo-library,demo-stacks}/**/*.{tf,md}" + excluded_paths: + - "**/demo-library/**/*.{tfvars,tf}" + targets: + - "components/library/" + tags: + - demo + + - component: "test globs without double stars upfront" + source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" + included_paths: + - "/weather/*.md" + version: "main" + targets: + - "components/library/" + tags: + - demo + + - component: "test shallow globs and folder exclusion" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/demo-localstack/*" + - "**/demo-library/**" + excluded_paths: + - "**/demo-library/**/stargazers/**" + - "**/demo-library/**/*.tf" + targets: + - "components/globs/" + tags: + - demo diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 82cf62016..e88a58f6e 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -51,3 +51,4 @@ spec: - "**/*.tftmpl" - "**/modules/**" excluded_paths: [] + diff --git a/tests/test-cases/demo-globs.yaml b/tests/test-cases/demo-globs.yaml new file mode 100644 index 000000000..996ec870d --- /dev/null +++ b/tests/test-cases/demo-globs.yaml @@ -0,0 +1,60 @@ +tests: + - name: atmos_vendor_pull_with_globs + enabled: true + description: "Ensure atmos vendor pull command executes without errors and files are present." + workdir: "fixtures/scenarios/vendor-globs" + command: "atmos" + args: + - "vendor" + - "pull" + expect: + file_exists: + - "./components/library/examples/demo-library/github/stargazers/README.md" + - "./components/library/examples/demo-library/ipinfo/README.md" + - "./components/library/examples/demo-library/weather/README.md" + - "./components/library/examples/demo-library/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" + - "./components/library/weather/README.md" + - "./components/globs/examples/demo-library/ipinfo/README.md" + - "./components/globs/examples/demo-library/weather/README.md" + - "./components/globs/examples/demo-library/README.md" + - "./components/globs/examples/demo-localstack/.gitignore" + - "./components/globs/examples/demo-localstack/atmos.yaml" + - "./components/globs/examples/demo-localstack/docker-compose.yml" + - "./components/globs/examples/demo-localstack/README.md" + file_not_exists: + - "./components/library/examples/demo-library/github/stargazers/main.tf" + - "./components/library/examples/demo-library/github/stargazers/outputs.tf" + - "./components/library/examples/demo-library/github/stargazers/providers.tf" + - "./components/library/examples/demo-library/github/stargazers/variables.tf" + - "./components/library/examples/demo-library/github/stargazers/versions.tf" + - "./components/library/examples/demo-library/ipinfo/main.tf" + - "./components/library/examples/demo-library/ipinfo/outputs.tf" + - "./components/library/examples/demo-library/ipinfo/providers.tf" + - "./components/library/examples/demo-library/ipinfo/variables.tf" + - "./components/library/examples/demo-library/ipinfo/versions.tf" + - "./components/library/examples/demo-library/weather/main.tf" + - "./components/library/examples/demo-library/weather/outputs.tf" + - "./components/library/examples/demo-library/weather/providers.tf" + - "./components/library/examples/demo-library/weather/variables.tf" + - "./components/library/examples/demo-library/weather/versions.tf" + - "./components/globs/examples/demo-library/github/stargazers/README.md" + - "./components/globs/examples/demo-library/github/stargazers/main.tf" + - "./components/globs/examples/demo-library/github/stargazers/outputs.tf" + - "./components/globs/examples/demo-library/github/stargazers/providers.tf" + - "./components/globs/examples/demo-library/github/stargazers/variables.tf" + - "./components/globs/examples/demo-library/github/stargazers/versions.tf" + - "./components/globs/examples/demo-library/ipinfo/outputs.tf" + - "./components/globs/examples/demo-library/ipinfo/providers.tf" + - "./components/globs/examples/demo-library/ipinfo/variables.tf" + - "./components/globs/examples/demo-library/ipinfo/versions.tf" + - "./components/globs/examples/demo-library/weather/main.tf" + - "./components/globs/examples/demo-library/weather/outputs.tf" + - "./components/globs/examples/demo-library/weather/providers.tf" + - "./components/globs/examples/demo-library/weather/variables.tf" + - "./components/globs/examples/demo-library/weather/versions.tf" + exit_code: 0 diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index c41e2fec0..9de844a01 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -217,7 +217,7 @@ The `vendor.yaml` vendoring manifest supports Kubernetes-style YAML config to de
`included_paths` and `excluded_paths`
- `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). + `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). For more details, see [Vendoring with Globs](#vendoring-with-globs).
`component`
@@ -497,3 +497,92 @@ To vendor the `vpc` component, execute the following command: atmos vendor pull -c vpc ``` + +## Vendoring with Globs + +When defining vendoring rules in Atmos, **glob patterns** determine which files and directories are included or excluded. Understanding how globs behave—especially when using greedy (`**`) versus non-greedy (`*`) patterns—is crucial for precise vendoring. + +### Understanding Greedy and Non-Greedy Globs + +Globs use special wildcard characters: + +- `*` (single asterisk) matches any sequence of characters **within a single path segment**. +- `**` (double asterisk) matches across multiple path segments **recursively**. + +This distinction is important when excluding specific directories or files while vendoring. + +#### Example: Excluding a Subdirectory + +Consider the following configuration: + +```yaml +included_paths: + - "**/demo-library/**" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +How It Works: +- The included_paths rule `**/demo-library/**` ensures all files inside `demo-library` (at any depth) are vendored. +- The excluded_paths rule `**/demo-library/**/stargazers/**` prevents any files inside `stargazers` subdirectories from being vendored. + +This means: +- All files within demo-library except those inside any `stargazers` subdirectory are vendored. +- Any other files outside `stargazers` are unaffected by this exclusion. + +#### Example: A Non-Recursive Pattern That Doesn't Work + +```yaml +included_paths: + - "**/demo-library/*" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +In this case: +- `**/demo-library/*` only matches immediate children of demo-library, not nested files or subdirectories. +- This means `stargazers/` itself could be matched, but its contents might not be explicitly excluded. +- To correctly capture all subdirectories and files while still excluding stargazers, use `**/demo-library/**/*`. + +Using `{...}` for Multiple Extensions or Patterns + +Curly braces `{...}` allow for expanding multiple patterns into separate glob matches. This is useful when selecting multiple file types or directories within a single glob pattern. + +#### Example: Matching Multiple File Extensions + +```yaml +included_paths: + - "**/demo-library/**/*.{tf,md}" +``` + +This is equivalent to writing: + +```yaml +included_paths: + - "**/demo-library/**/*.tf" + - "**/demo-library/**/*.md" +``` + +The `{tf,md}` part expands to both `*.tf` and `*.md`, making the rule more concise. + +#### Example: Excluding Multiple Directories + +```yaml +excluded_paths: + - "**/demo-library/**/{stargazers,archive}/**" +``` + +This excludes both: +- `**/demo-library/**/stargazers/**` +- `**/demo-library/**/archive/**` + +Using `{...}` here prevents the need to write two separate exclusion rules. + +## Key Takeaways + +1. Use `**/` for recursive matching to include everything inside a directory. +2. Use `*` for single-segment matches, which won't include deeper subdirectories. +3. Use `{...}` to match multiple extensions or directories within a single pattern. +4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. + +By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories are omitted. \ No newline at end of file