From ee60848932c52829abfcc9d1392964b5c8a18d33 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 29 Jan 2025 23:20:52 +0500 Subject: [PATCH 01/68] symlinks --- internal/exec/go_getter_utils.go | 46 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index badfb3a67..089f2fe6d 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -99,6 +99,15 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) } + if !strings.Contains(parsedURL.Path, "//") { + // If it ends with .git treat it as wanting the entire repo + if strings.HasSuffix(parsedURL.Path, ".git") || + len(parts) == 3 { // means /owner/repo only + u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.\n") + parsedURL.Path = parsedURL.Path + "//." + } + } + atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") gitHubToken := os.Getenv("GITHUB_TOKEN") @@ -158,7 +167,6 @@ func GoGetterGet( ) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors RegisterCustomDetectors(atmosConfig) @@ -168,8 +176,12 @@ func GoGetterGet( // Destination where the files will be stored. This will create the directory if it doesn't exist Dst: dest, Mode: clientMode, + Getters: map[string]getter.Getter{ + // Overriding 'git' + "git": &CustomGitGetter{}, + }, } - + fmt.Println(src, "src") if err := client.Get(); err != nil { return err } @@ -177,6 +189,36 @@ func GoGetterGet( return nil } +// CustomGitGetter is a custom getter for git (git::) that removes symlinks +type CustomGitGetter struct { + getter.GitGetter +} + +// Implements the custom getter logic removing symlinks +func (c *CustomGitGetter) Get(dst string, url *url.URL) error { + // Normal clone + if err := c.GitGetter.Get(dst, url); err != nil { + return err + } + // Remove symlinks + return removeSymlinks(dst) +} + +// removeSymlinks walks the destination directory and removes any symlinks +// it encounters. +func removeSymlinks(root string) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + // It's a symlink, remove it + return os.Remove(path) + } + return nil + }) +} + // DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() From 940d8769405a155eed76755521b98547a5d759e2 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 29 Jan 2025 23:32:21 +0500 Subject: [PATCH 02/68] comment --- internal/exec/go_getter_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 089f2fe6d..76c364290 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -181,7 +181,6 @@ func GoGetterGet( "git": &CustomGitGetter{}, }, } - fmt.Println(src, "src") if err := client.Get(); err != nil { return err } From 16a72f34e7032b44978d301fb900fc3470a98569 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 29 Jan 2025 23:35:15 +0500 Subject: [PATCH 03/68] log warning --- internal/exec/go_getter_utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 76c364290..001deaab5 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -211,6 +211,7 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { + u.LogWarning(schema.AtmosConfiguration{}, fmt.Sprintf("Removing symlink: %s", path)) // It's a symlink, remove it return os.Remove(path) } From f9b3348d6550f3cf4641964da68aeb657e7ada9c Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 30 Jan 2025 00:15:49 +0500 Subject: [PATCH 04/68] adding back the rest getters --- internal/exec/go_getter_utils.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 001deaab5..1db480e15 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -157,6 +157,15 @@ func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { ) } +func RegisterCustomGetters(atmosConfig schema.AtmosConfiguration) { + getter.Detectors = append( + []getter.Detector{ + &CustomGitHubDetector{AtmosConfig: atmosConfig}, + }, + getter.Detectors..., + ) +} + // GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination func GoGetterGet( atmosConfig schema.AtmosConfiguration, @@ -178,7 +187,14 @@ func GoGetterGet( Mode: clientMode, Getters: map[string]getter.Getter{ // Overriding 'git' - "git": &CustomGitGetter{}, + "git": &CustomGitGetter{}, + "file": &getter.FileGetter{}, + "hg": &getter.HgGetter{}, + "http": &getter.HttpGetter{}, + "https": &getter.HttpGetter{}, + // "s3": &getter.S3Getter{}, // add as needed + // "gcs": &getter.GCSGetter{}, + }, } if err := client.Get(); err != nil { From ed1af3d160740534970120835f2a7ebf8e4fef4a Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 31 Jan 2025 00:20:47 +0500 Subject: [PATCH 05/68] fix for windows path --- internal/exec/go_getter_utils.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 1db480e15..aee7c8c7a 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -70,6 +70,7 @@ func IsValidScheme(scheme string) bool { // do a git-based clone with a token. type CustomGitHubDetector struct { AtmosConfig schema.AtmosConfiguration + source string } // Detect implements the getter.Detector interface for go-getter v1. @@ -99,11 +100,10 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) } - if !strings.Contains(parsedURL.Path, "//") { - // If it ends with .git treat it as wanting the entire repo - if strings.HasSuffix(parsedURL.Path, ".git") || - len(parts) == 3 { // means /owner/repo only - u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.\n") + if !strings.Contains(d.source, "//") { + // means user typed something like "github.com/org/repo.git" with NO subdir + if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { + u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.'\n") parsedURL.Path = parsedURL.Path + "//." } } @@ -148,19 +148,10 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { // RegisterCustomDetectors prepends the custom detector so it runs before // the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) { +func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, - }, - getter.Detectors..., - ) -} - -func RegisterCustomGetters(atmosConfig schema.AtmosConfiguration) { - getter.Detectors = append( - []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig}, + &CustomGitHubDetector{AtmosConfig: atmosConfig, source: source}, }, getter.Detectors..., ) @@ -176,8 +167,12 @@ func GoGetterGet( ) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors - RegisterCustomDetectors(atmosConfig) + + // Register custom detectors, passing the original `src` to the CustomGitHubDetector. + // go-getter typically strips subdirectories before calling the detector, so the + // unaltered source is needed to identify whether a top-level repository or a + // subdirectory was specified (e.g., for appending "//." only when no subdir is present). + RegisterCustomDetectors(atmosConfig, src) client := &getter.Client{ Ctx: ctx, From 5a6789e65f7d5b5ebfe3ed5bb13586aa19b74012 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sun, 2 Feb 2025 00:42:34 +0500 Subject: [PATCH 06/68] globs --- internal/exec/copy_glob.go | 255 ++++++++++++++++++++ internal/exec/vendor_model.go | 2 +- tests/fixtures/scenarios/vendor/vendor.yaml | 11 + tests/test-cases/demo-stacks.yaml | 9 + 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 internal/exec/copy_glob.go diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go new file mode 100644 index 000000000..858d8bd69 --- /dev/null +++ b/internal/exec/copy_glob.go @@ -0,0 +1,255 @@ +package exec + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. +) + +// copyFile copies a single file from src to dst while preserving file permissions. +func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source file %q: %w", src, err) + } + defer sourceFile.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return fmt.Errorf("creating destination directory for %q: %w", dst, err) + } + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating destination file %q: %w", dst, err) + } + defer destinationFile.Close() + + if _, err := io.Copy(destinationFile, sourceFile); err != nil { + return fmt.Errorf("copying content from %q to %q: %w", src, dst, err) + } + + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("getting file info for %q: %w", src, err) + } + if err := os.Chmod(dst, info.Mode()); err != nil { + return fmt.Errorf("setting permissions on %q: %w", dst, err) + } + return nil +} + +// skipFunc determines whether to skip a file/directory based on its relative path to baseDir. +// If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. +func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { + if info.Name() == ".git" { + return true, nil + } + relPath, err := filepath.Rel(baseDir, srcPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) + return true, nil // treat error as a signal to skip + } + relPath = filepath.ToSlash(relPath) + + // Process exclusion patterns. + for _, pattern := range excluded { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) + continue + } else if matched { + u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) + return true, nil + } + } + + // Process inclusion patterns (only for non-directory files). + if len(included) > 0 && !info.IsDir() { + matchedAny := false + for _, pattern := range included { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) + continue + } else if matched { + u.LogTrace(atmosConfig, fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) + matchedAny = true + break + } + } + if !matchedAny { + u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) + return true, nil + } + } + return false, nil +} + +// copyDirRecursive recursively copies srcDir to dstDir using skipFunc filtering. +func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, baseDir string, excluded, included []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + skip, err := skipFunc(atmosConfig, info, srcPath, baseDir, excluded, included) + if err != nil { + return err + } + if skip { + continue + } + + // Skip symlinks. + if info.Mode()&os.ModeSymlink != 0 { + u.LogTrace(atmosConfig, fmt.Sprintf("Skipping symlink: %q", srcPath)) + continue + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + if err := copyDirRecursive(atmosConfig, srcPath, dstPath, baseDir, excluded, included); err != nil { + return err + } + } else { + if err := copyFile(atmosConfig, srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +// getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. +// If no matches are found, it logs a trace and returns an empty slice. +// When the pattern ends with "/*", it retries with a recursive "/**" variant. +func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + matches, err := u.GetGlobMatches(fullPattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) + } + if len(matches) == 0 { + if strings.HasSuffix(pattern, "/*") { + recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" + fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) + matches, err = u.GetGlobMatches(fullRecursivePattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) + } + if len(matches) == 0 { + u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) + return []string{}, nil + } + return matches, nil + } + u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) + return []string{}, nil + } + return matches, nil +} + +// copyToTargetWithPatterns copies the contents from sourceDir to targetPath, +// applying inclusion and exclusion patterns from the vendor source configuration. +// If sourceIsLocalFile is true and targetPath lacks an extension, the sanitized URI is appended. +// If no included paths are defined, all files (except those matching excluded paths) are copied. +// In the special case where neither inclusion nor exclusion patterns are defined, +// the optimized cp library (github.com/otiai10/copy) is used. +func copyToTargetWithPatterns( + atmosConfig schema.AtmosConfiguration, + sourceDir, targetPath string, + s *schema.AtmosVendorSource, + sourceIsLocalFile bool, + uri string, +) error { + if sourceIsLocalFile && filepath.Ext(targetPath) == "" { + targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) + } + u.LogTrace(atmosConfig, fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) + if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { + return fmt.Errorf("creating target directory %q: %w", targetPath, err) + } + + // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. + if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { + u.LogTrace(atmosConfig, "No inclusion or exclusion patterns defined; using cp library for fast copy") + return cp.Copy(sourceDir, targetPath) + } + + // If inclusion patterns are provided, use them to determine which files to copy. + if len(s.IncludedPaths) > 0 { + filesToCopy := make(map[string]struct{}) + for _, pattern := range s.IncludedPaths { + matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) + continue + } + for _, match := range matches { + filesToCopy[match] = struct{}{} + } + } + if len(filesToCopy) == 0 { + u.LogTrace(atmosConfig, "No files matched the inclusion patterns - target directory will be empty") + return nil + } + for file := range filesToCopy { + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("computing relative path for %q: %w", file, err) + } + relPath = filepath.ToSlash(relPath) + skip := false + for _, ex := range s.ExcludedPaths { + matched, err := u.PathMatch(ex, relPath) + if err != nil { + u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) + continue + } else if matched { + u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) + skip = true + break + } + } + if skip { + continue + } + dstPath := filepath.Join(targetPath, relPath) + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } + if info.IsDir() { + if err := copyDirRecursive(atmosConfig, file, dstPath, file, s.ExcludedPaths, nil); err != nil { + return err + } + } else { + if err := copyFile(atmosConfig, file, dstPath); err != nil { + return err + } + } + } + } else { + // No inclusion patterns defined; copy everything except those matching excluded items. + if err := copyDirRecursive(atmosConfig, sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) + } + } + return nil +} diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 6ddf14e71..cbf257d67 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -307,7 +307,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - if err := copyToTarget(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to copy package: %w", err), name: p.name, diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 82cf62016..2c5ffce6c 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -51,3 +51,14 @@ spec: - "**/*.tftmpl" - "**/modules/**" excluded_paths: [] + + - component: "test globs" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/{demo-library,demo-stacks}/**/*.{tf,md}" + excluded_paths: + - "**/demo-library/**/*.{tfvars,tf}" + targets: + - "components/library/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 80923b7be..2d9dfffb1 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -193,4 +193,13 @@ tests: - "./components/terraform/vpc-src/outputs.tf" - "./components/terraform/vpc-src/variables.tf" - "./components/terraform/vpc-src/versions.tf" + - "./components/library/examples/demo-library/github/stargazers/README.md" + - "./components/library/examples/demo-library/ipinfo/README.md" + - "./components/library/examples/demo-library/weather/README.md" + - "./components/library/examples/demo-library/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" exit_code: 0 From e96dd7f3b180aa92d186ba8717ce9e78762d7a06 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 3 Feb 2025 20:36:00 +0500 Subject: [PATCH 07/68] removing atmosconfig from logging --- internal/exec/copy_glob.go | 30 +++++++++++++++--------------- internal/exec/go_getter_utils.go | 6 ++++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 858d8bd69..d04fdc46c 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -52,7 +52,7 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) + u.LogTrace(fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) @@ -61,10 +61,10 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range excluded { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) + u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) continue } else if matched { - u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) + u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) return true, nil } } @@ -75,16 +75,16 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range included { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) + u.LogTrace(fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) continue } else if matched { - u.LogTrace(atmosConfig, fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) + u.LogTrace(fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) matchedAny = true break } } if !matchedAny { - u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) + u.LogTrace(fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) return true, nil } } @@ -116,7 +116,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas // Skip symlinks. if info.Mode()&os.ModeSymlink != 0 { - u.LogTrace(atmosConfig, fmt.Sprintf("Skipping symlink: %q", srcPath)) + u.LogTrace(fmt.Sprintf("Skipping symlink: %q", srcPath)) continue } @@ -154,12 +154,12 @@ func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, patt return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) } if len(matches) == 0 { - u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) + u.LogTrace(fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) return []string{}, nil } return matches, nil } - u.LogTrace(atmosConfig, fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) + u.LogTrace(fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) return []string{}, nil } return matches, nil @@ -181,14 +181,14 @@ func copyToTargetWithPatterns( if sourceIsLocalFile && filepath.Ext(targetPath) == "" { targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) } - u.LogTrace(atmosConfig, fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) + u.LogTrace(fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { return fmt.Errorf("creating target directory %q: %w", targetPath, err) } // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { - u.LogTrace(atmosConfig, "No inclusion or exclusion patterns defined; using cp library for fast copy") + u.LogTrace("No inclusion or exclusion patterns defined; using cp library for fast copy") return cp.Copy(sourceDir, targetPath) } @@ -198,7 +198,7 @@ func copyToTargetWithPatterns( for _, pattern := range s.IncludedPaths { matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) + u.LogTrace(fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) continue } for _, match := range matches { @@ -206,7 +206,7 @@ func copyToTargetWithPatterns( } } if len(filesToCopy) == 0 { - u.LogTrace(atmosConfig, "No files matched the inclusion patterns - target directory will be empty") + u.LogTrace("No files matched the inclusion patterns - target directory will be empty") return nil } for file := range filesToCopy { @@ -219,10 +219,10 @@ func copyToTargetWithPatterns( for _, ex := range s.ExcludedPaths { matched, err := u.PathMatch(ex, relPath) if err != nil { - u.LogTrace(atmosConfig, fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) + u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) continue } else if matched { - u.LogTrace(atmosConfig, fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) + u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) skip = true break } diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index c3b055498..bbffce9cf 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -103,7 +103,7 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { if !strings.Contains(d.source, "//") { // means user typed something like "github.com/org/repo.git" with NO subdir if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { - u.LogDebug(d.AtmosConfig, "Detected top-level repo with no subdir: appending '//.'\n") + u.LogDebug("Detected top-level repo with no subdir: appending '//.'\n") parsedURL.Path = parsedURL.Path + "//." } } @@ -222,7 +222,9 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { - u.LogWarning(schema.AtmosConfiguration{}, fmt.Sprintf("Removing symlink: %s", path)) + //Symlinks are removed for the entire repo, regardless if there are any subfolders specified + //Thus logging is disabled + //u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) // It's a symlink, remove it return os.Remove(path) } From 06f5f3bd762e4e221e44dfd018d3f65d93d2c708 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:37:15 +0000 Subject: [PATCH 08/68] [autofix.ci] apply automated fixes --- internal/exec/go_getter_utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index bbffce9cf..ba1fc6279 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -222,9 +222,9 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { - //Symlinks are removed for the entire repo, regardless if there are any subfolders specified - //Thus logging is disabled - //u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) + // Symlinks are removed for the entire repo, regardless if there are any subfolders specified + // Thus logging is disabled + // u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) // It's a symlink, remove it return os.Remove(path) } From 82caa056392c9f8529a85bb88edbc247f338dd66 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:38:35 +0600 Subject: [PATCH 09/68] Update internal/exec/copy_glob.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- internal/exec/copy_glob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index d04fdc46c..1a0bd2401 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -52,7 +52,7 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error computing relative path for %q: %v", srcPath, err)) + l.LogDebug("Error computing relative path", 'srcPath', srcPath, 'err', err) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) From b9d4e5a4680313919f83e3fe16682939dc5cea06 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 13:01:49 +0500 Subject: [PATCH 10/68] logging to charmbracelet --- internal/exec/copy_glob.go | 47 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 1a0bd2401..e58048f55 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -7,13 +7,14 @@ import ( "path/filepath" "strings" + l "github.com/charmbracelet/log" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. ) // copyFile copies a single file from src to dst while preserving file permissions. -func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { +func copyFile(src, dst string) error { sourceFile, err := os.Open(src) if err != nil { return fmt.Errorf("opening source file %q: %w", src, err) @@ -46,13 +47,13 @@ func copyFile(atmosConfig schema.AtmosConfiguration, src, dst string) error { // skipFunc determines whether to skip a file/directory based on its relative path to baseDir. // If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. -func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { +func skipFunc(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { if info.Name() == ".git" { return true, nil } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - l.LogDebug("Error computing relative path", 'srcPath', srcPath, 'err', err) + l.Debug("Error computing relative path", "srcPath", srcPath, "error", err) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) @@ -61,10 +62,10 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range excluded { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", pattern, relPath, err)) + l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue } else if matched { - u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, pattern)) + l.Debug("Excluding path due to exclusion pattern", "path", relPath, "pattern", pattern) return true, nil } } @@ -75,16 +76,16 @@ func skipFunc(atmosConfig schema.AtmosConfiguration, info os.FileInfo, srcPath, for _, pattern := range included { matched, err := u.PathMatch(pattern, relPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error matching inclusion pattern %q with %q: %v", pattern, relPath, err)) + l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue } else if matched { - u.LogTrace(fmt.Sprintf("Including %q because it matches inclusion pattern %q", relPath, pattern)) + l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern) matchedAny = true break } } if !matchedAny { - u.LogTrace(fmt.Sprintf("Excluding %q because it does not match any inclusion pattern", relPath)) + l.Debug("Excluding path because it does not match any inclusion pattern", "path", relPath) return true, nil } } @@ -106,7 +107,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas return fmt.Errorf("getting info for %q: %w", srcPath, err) } - skip, err := skipFunc(atmosConfig, info, srcPath, baseDir, excluded, included) + skip, err := skipFunc(info, srcPath, baseDir, excluded, included) if err != nil { return err } @@ -116,7 +117,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas // Skip symlinks. if info.Mode()&os.ModeSymlink != 0 { - u.LogTrace(fmt.Sprintf("Skipping symlink: %q", srcPath)) + l.Debug("Skipping symlink", "path", srcPath) continue } @@ -128,7 +129,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas return err } } else { - if err := copyFile(atmosConfig, srcPath, dstPath); err != nil { + if err := copyFile(srcPath, dstPath); err != nil { return err } } @@ -137,9 +138,9 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas } // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. -// If no matches are found, it logs a trace and returns an empty slice. +// If no matches are found, it logs a debug message and returns an empty slice. // When the pattern ends with "/*", it retries with a recursive "/**" variant. -func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, pattern string) ([]string, error) { +func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { fullPattern := filepath.Join(sourceDir, pattern) matches, err := u.GetGlobMatches(fullPattern) if err != nil { @@ -154,12 +155,12 @@ func getMatchesForPattern(atmosConfig schema.AtmosConfiguration, sourceDir, patt return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) } if len(matches) == 0 { - u.LogTrace(fmt.Sprintf("No matches found for recursive pattern %q - target directory will be empty", fullRecursivePattern)) + l.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) return []string{}, nil } return matches, nil } - u.LogTrace(fmt.Sprintf("No matches found for pattern %q - target directory will be empty", fullPattern)) + l.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) return []string{}, nil } return matches, nil @@ -181,14 +182,14 @@ func copyToTargetWithPatterns( if sourceIsLocalFile && filepath.Ext(targetPath) == "" { targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) } - u.LogTrace(fmt.Sprintf("Copying from %q to %q", sourceDir, targetPath)) + l.Debug("Copying files", "source", sourceDir, "target", targetPath) if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { return fmt.Errorf("creating target directory %q: %w", targetPath, err) } // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { - u.LogTrace("No inclusion or exclusion patterns defined; using cp library for fast copy") + l.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") return cp.Copy(sourceDir, targetPath) } @@ -196,9 +197,9 @@ func copyToTargetWithPatterns( if len(s.IncludedPaths) > 0 { filesToCopy := make(map[string]struct{}) for _, pattern := range s.IncludedPaths { - matches, err := getMatchesForPattern(atmosConfig, sourceDir, pattern) + matches, err := getMatchesForPattern(sourceDir, pattern) if err != nil { - u.LogTrace(fmt.Sprintf("Warning: error getting matches for pattern %q: %v", pattern, err)) + l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) continue } for _, match := range matches { @@ -206,7 +207,7 @@ func copyToTargetWithPatterns( } } if len(filesToCopy) == 0 { - u.LogTrace("No files matched the inclusion patterns - target directory will be empty") + l.Debug("No files matched the inclusion patterns - target directory will be empty") return nil } for file := range filesToCopy { @@ -219,10 +220,10 @@ func copyToTargetWithPatterns( for _, ex := range s.ExcludedPaths { matched, err := u.PathMatch(ex, relPath) if err != nil { - u.LogTrace(fmt.Sprintf("Error matching exclusion pattern %q with %q: %v", ex, relPath, err)) + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) continue } else if matched { - u.LogTrace(fmt.Sprintf("Excluding %q because it matches exclusion pattern %q", relPath, ex)) + l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) skip = true break } @@ -240,7 +241,7 @@ func copyToTargetWithPatterns( return err } } else { - if err := copyFile(atmosConfig, file, dstPath); err != nil { + if err := copyFile(file, dstPath); err != nil { return err } } From 0b8e5fa46fbe1033ecb8f8a207d53a504924af1d Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 14:03:27 +0500 Subject: [PATCH 11/68] renaming skipFunc --- internal/exec/copy_glob.go | 17 ++++++++--------- internal/exec/vendor_model.go | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index e58048f55..e9460009f 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -45,9 +45,9 @@ func copyFile(src, dst string) error { return nil } -// skipFunc determines whether to skip a file/directory based on its relative path to baseDir. +// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. // If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. -func skipFunc(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { if info.Name() == ".git" { return true, nil } @@ -92,8 +92,8 @@ func skipFunc(info os.FileInfo, srcPath, baseDir string, excluded, included []st return false, nil } -// copyDirRecursive recursively copies srcDir to dstDir using skipFunc filtering. -func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, baseDir string, excluded, included []string) error { +// copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { entries, err := os.ReadDir(srcDir) if err != nil { return fmt.Errorf("reading directory %q: %w", srcDir, err) @@ -107,7 +107,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas return fmt.Errorf("getting info for %q: %w", srcPath, err) } - skip, err := skipFunc(info, srcPath, baseDir, excluded, included) + skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) if err != nil { return err } @@ -125,7 +125,7 @@ func copyDirRecursive(atmosConfig schema.AtmosConfiguration, srcDir, dstDir, bas if err := os.MkdirAll(dstPath, info.Mode()); err != nil { return fmt.Errorf("creating directory %q: %w", dstPath, err) } - if err := copyDirRecursive(atmosConfig, srcPath, dstPath, baseDir, excluded, included); err != nil { + if err := copyDirRecursive(srcPath, dstPath, baseDir, excluded, included); err != nil { return err } } else { @@ -173,7 +173,6 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { // In the special case where neither inclusion nor exclusion patterns are defined, // the optimized cp library (github.com/otiai10/copy) is used. func copyToTargetWithPatterns( - atmosConfig schema.AtmosConfiguration, sourceDir, targetPath string, s *schema.AtmosVendorSource, sourceIsLocalFile bool, @@ -237,7 +236,7 @@ func copyToTargetWithPatterns( return fmt.Errorf("stating file %q: %w", file, err) } if info.IsDir() { - if err := copyDirRecursive(atmosConfig, file, dstPath, file, s.ExcludedPaths, nil); err != nil { + if err := copyDirRecursive(file, dstPath, file, s.ExcludedPaths, nil); err != nil { return err } } else { @@ -248,7 +247,7 @@ func copyToTargetWithPatterns( } } else { // No inclusion patterns defined; copy everything except those matching excluded items. - if err := copyDirRecursive(atmosConfig, sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) } } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index b48331144..c12b0045b 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -307,7 +307,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - if err := copyToTargetWithPatterns(atmosConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to copy package: %w", err), name: p.name, From e8133be3af6a336c43a7c1687d6505e69cc7baab Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Thu, 6 Feb 2025 15:37:56 +0600 Subject: [PATCH 12/68] Update internal/exec/go_getter_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- internal/exec/go_getter_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index ba1fc6279..c8f9a84b9 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -214,7 +214,7 @@ func (c *CustomGitGetter) Get(dst string, url *url.URL) error { return removeSymlinks(dst) } -// removeSymlinks walks the destination directory and removes any symlinks +// removeSymlinks walks the directory and removes any symlinks // it encounters. func removeSymlinks(root string) error { return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { From 84794096f2e93e673301859c43e81f558dc662bf Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 16:57:00 +0500 Subject: [PATCH 13/68] new testcase --- tests/fixtures/scenarios/vendor/vendor.yaml | 10 ++++++++++ tests/test-cases/demo-stacks.yaml | 1 + 2 files changed, 11 insertions(+) diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 2c5ffce6c..ba3c016f5 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -62,3 +62,13 @@ spec: - "components/library/" tags: - demo + + - component: "test globs without double stars upfront" + source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" + included_paths: + - "/weather/*.md" + version: "main" + targets: + - "components/library/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 4a0ca1a4c..367742db9 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -192,4 +192,5 @@ tests: - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" + - "./components/library/weather/README.md" exit_code: 0 From c4adfb20f5beaaf096c132f6f5a2c19ddd7845db Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 18:35:48 +0500 Subject: [PATCH 14/68] added depth=1 --- internal/exec/go_getter_utils.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 142c32cd5..4a31b3dab 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,6 +136,13 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } + // Ensure that the query parameter "depth" is set to "1" if not already present. + q := parsedURL.Query() + if q.Get("depth") == "" { + q.Set("depth", "1") + } + parsedURL.RawQuery = q.Encode() + finalURL := "git::" + parsedURL.String() return finalURL, true, nil From 7382cab94df827b9b2119ba1477d863e6b359a14 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Feb 2025 19:09:26 +0500 Subject: [PATCH 15/68] depth --- internal/exec/go_getter_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 4a31b3dab..0fc8f8a87 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,9 +136,9 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } - // Ensure that the query parameter "depth" is set to "1" if not already present. + // Ensure that the "depth" parameter is set to "1" if the key does not exist. q := parsedURL.Query() - if q.Get("depth") == "" { + if _, exists := q["depth"]; !exists { q.Set("depth", "1") } parsedURL.RawQuery = q.Encode() From a9c8f48f262f10a953a6a5023d1f60892572b606 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Thu, 6 Feb 2025 21:04:33 +0600 Subject: [PATCH 16/68] Update internal/exec/go_getter_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- internal/exec/go_getter_utils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 0fc8f8a87..38797fb07 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,7 +136,11 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } - // Ensure that the "depth" parameter is set to "1" if the key does not exist. + // Set "depth=1" for a shallow clone if not specified. + // In Go-Getter, "depth" controls how many revisions are cloned: + // - `depth=1` fetches only the latest commit (faster, less bandwidth). + // - `depth=` (empty) performs a full clone (default Git behavior). + // - `depth=N` clones the last N revisions. q := parsedURL.Query() if _, exists := q["depth"]; !exists { q.Set("depth", "1") From a54dafb5c5ce984971bc7d3781abc5a095e501fd Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 6 Feb 2025 15:05:27 +0000 Subject: [PATCH 17/68] [autofix.ci] apply automated fixes --- internal/exec/go_getter_utils.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 38797fb07..f76699f58 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -136,11 +136,11 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } - // Set "depth=1" for a shallow clone if not specified. - // In Go-Getter, "depth" controls how many revisions are cloned: - // - `depth=1` fetches only the latest commit (faster, less bandwidth). - // - `depth=` (empty) performs a full clone (default Git behavior). - // - `depth=N` clones the last N revisions. + // Set "depth=1" for a shallow clone if not specified. + // In Go-Getter, "depth" controls how many revisions are cloned: + // - `depth=1` fetches only the latest commit (faster, less bandwidth). + // - `depth=` (empty) performs a full clone (default Git behavior). + // - `depth=N` clones the last N revisions. q := parsedURL.Query() if _, exists := q["depth"]; !exists { q.Set("depth", "1") From 2bf586a5f3f1e3f2abb39e3e318afc4a9003572f Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Feb 2025 22:20:59 +0500 Subject: [PATCH 18/68] excluded subfolders --- internal/exec/copy_glob.go | 153 +++++++++++++++++++++++++++++++++---- 1 file changed, 137 insertions(+), 16 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index e9460009f..105fa50bd 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -56,21 +56,38 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ l.Debug("Error computing relative path", "srcPath", srcPath, "error", err) return true, nil // treat error as a signal to skip } + // Ensure uniform path separator. relPath = filepath.ToSlash(relPath) // Process exclusion patterns. + // For directories, check with and without a trailing slash. for _, pattern := range excluded { + // First check the plain relative path. matched, err := u.PathMatch(pattern, relPath) if err != nil { l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue - } else if matched { - l.Debug("Excluding path due to exclusion pattern", "path", relPath, "pattern", pattern) + } + if matched { + l.Debug("Excluding path due to exclusion pattern (plain match)", "path", relPath, "pattern", pattern) return true, nil } + // If it is a directory, also try matching with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, relPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, "path", relPath+"/", "error", err) + continue + } + if matched { + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "path", relPath+"/", "pattern", pattern) + return true, nil + } + } } // Process inclusion patterns (only for non-directory files). + // (Directories are generally picked up by the inclusion branch in copyToTargetWithPatterns.) if len(included) > 0 && !info.IsDir() { matchedAny := false for _, pattern := range included { @@ -78,7 +95,8 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ if err != nil { l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err) continue - } else if matched { + } + if matched { l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern) matchedAny = true break @@ -93,6 +111,7 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ } // copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. +// This function is used in cases where the entire sourceDir is the base for relative paths. func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { entries, err := os.ReadDir(srcDir) if err != nil { @@ -107,11 +126,13 @@ func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []strin return fmt.Errorf("getting info for %q: %w", srcPath, err) } + // Check if this entry should be skipped. skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) if err != nil { return err } if skip { + l.Debug("Skipping entry", "srcPath", srcPath) continue } @@ -137,6 +158,80 @@ func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []strin return nil } +// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. +// Instead of using the local srcDir as the base for computing relative paths, this function uses the original +// source directory (globalBase) and an accumulated prefix that represents the relative path from globalBase. +func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + // Compute the full relative path from the original source. + fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + // Skip .git directories. + if entry.Name() == ".git" { + l.Debug("Skipping .git directory", "path", fullRelPath) + continue + } + + // Check exclusion patterns using the full relative path. + skip := false + for _, pattern := range excluded { + // Check plain match. + matched, err := u.PathMatch(pattern, fullRelPath) + if err != nil { + l.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, "path", fullRelPath, "error", err) + continue + } + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (plain match)", "path", fullRelPath, "pattern", pattern) + skip = true + break + } + // For directories, also try with a trailing slash. + if info.IsDir() { + matched, err = u.PathMatch(pattern, fullRelPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, "path", fullRelPath+"/", "error", err) + continue + } + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", "path", fullRelPath+"/", "pattern", pattern) + skip = true + break + } + } + } + if skip { + continue + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + // Recurse with updated prefix. + if err := copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded); err != nil { + return err + } + } else { + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. // If no matches are found, it logs a debug message and returns an empty slice. // When the pattern ends with "/*", it retries with a recursive "/**" variant. @@ -210,33 +305,59 @@ func copyToTargetWithPatterns( return nil } for file := range filesToCopy { + // Retrieve file information early so that we can adjust exclusion checks if this is a directory. + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } relPath, err := filepath.Rel(sourceDir, file) if err != nil { return fmt.Errorf("computing relative path for %q: %w", file, err) } relPath = filepath.ToSlash(relPath) skip := false + // For directories, check both the plain relative path and with a trailing slash. for _, ex := range s.ExcludedPaths { - matched, err := u.PathMatch(ex, relPath) - if err != nil { - l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) - continue - } else if matched { - l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) - skip = true - break + if info.IsDir() { + matched, err := u.PathMatch(ex, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) + } else if matched { + l.Debug("Excluding directory due to exclusion pattern (plain match)", "directory", relPath, "pattern", ex) + skip = true + break + } + // Also try matching with a trailing slash. + matched, err = u.PathMatch(ex, relPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", ex, "path", relPath+"/", "error", err) + } else if matched { + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "directory", relPath+"/", "pattern", ex) + skip = true + break + } + } else { + // For files, just check the plain relative path. + matched, err := u.PathMatch(ex, relPath) + if err != nil { + l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) + } else if matched { + l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) + skip = true + break + } } } if skip { continue } + + // Build the destination path. dstPath := filepath.Join(targetPath, relPath) - info, err := os.Stat(file) - if err != nil { - return fmt.Errorf("stating file %q: %w", file, err) - } if info.IsDir() { - if err := copyDirRecursive(file, dstPath, file, s.ExcludedPaths, nil); err != nil { + // Instead of resetting the base for relative paths, + // use the new recursive function that preserves the global relative path. + if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { return err } } else { From 1fea6edb5a32a7344540f348416f7953ba0408f3 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sun, 9 Feb 2025 01:06:06 +0500 Subject: [PATCH 19/68] shallow copy --- internal/exec/copy_glob.go | 66 ++++++++++++--------- tests/fixtures/scenarios/vendor/vendor.yaml | 13 ++++ tests/test-cases/demo-stacks.yaml | 7 +++ 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 105fa50bd..5528d9303 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -234,7 +234,7 @@ func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, exclu // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. // If no matches are found, it logs a debug message and returns an empty slice. -// When the pattern ends with "/*", it retries with a recursive "/**" variant. +// For patterns ending with "/*" (shallow copy indicator) the function does not fallback to a recursive variant. func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { fullPattern := filepath.Join(sourceDir, pattern) matches, err := u.GetGlobMatches(fullPattern) @@ -242,6 +242,12 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) } if len(matches) == 0 { + // If the pattern ends with "/*" (and not "/**"), do not fallback. + if strings.HasSuffix(pattern, "/*") && !strings.HasSuffix(pattern, "/**") { + l.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) + return []string{}, nil + } + // Fallback for patterns ending with "/*" (non-shallow) or others. if strings.HasSuffix(pattern, "/*") { recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) @@ -287,25 +293,25 @@ func copyToTargetWithPatterns( return cp.Copy(sourceDir, targetPath) } - // If inclusion patterns are provided, use them to determine which files to copy. - if len(s.IncludedPaths) > 0 { - filesToCopy := make(map[string]struct{}) - for _, pattern := range s.IncludedPaths { - matches, err := getMatchesForPattern(sourceDir, pattern) - if err != nil { - l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) - continue - } - for _, match := range matches { - filesToCopy[match] = struct{}{} - } + // If inclusion patterns are provided, process each pattern individually. + for _, pattern := range s.IncludedPaths { + // Determine if the pattern indicates shallow copy. + shallow := false + if strings.HasSuffix(pattern, "/*") && !strings.HasSuffix(pattern, "/**") { + shallow = true + } + + matches, err := getMatchesForPattern(sourceDir, pattern) + if err != nil { + l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) + continue } - if len(filesToCopy) == 0 { - l.Debug("No files matched the inclusion patterns - target directory will be empty") - return nil + if len(matches) == 0 { + l.Debug("No files matched the inclusion pattern", "pattern", pattern) + continue } - for file := range filesToCopy { - // Retrieve file information early so that we can adjust exclusion checks if this is a directory. + for _, file := range matches { + // Retrieve file information. info, err := os.Stat(file) if err != nil { return fmt.Errorf("stating file %q: %w", file, err) @@ -315,8 +321,9 @@ func copyToTargetWithPatterns( return fmt.Errorf("computing relative path for %q: %w", file, err) } relPath = filepath.ToSlash(relPath) + + // Check exclusion patterns (for directories, try both plain and trailing slash). skip := false - // For directories, check both the plain relative path and with a trailing slash. for _, ex := range s.ExcludedPaths { if info.IsDir() { matched, err := u.PathMatch(ex, relPath) @@ -327,7 +334,6 @@ func copyToTargetWithPatterns( skip = true break } - // Also try matching with a trailing slash. matched, err = u.PathMatch(ex, relPath+"/") if err != nil { l.Debug("Error matching exclusion pattern with trailing slash", "pattern", ex, "path", relPath+"/", "error", err) @@ -337,7 +343,6 @@ func copyToTargetWithPatterns( break } } else { - // For files, just check the plain relative path. matched, err := u.PathMatch(ex, relPath) if err != nil { l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) @@ -355,10 +360,14 @@ func copyToTargetWithPatterns( // Build the destination path. dstPath := filepath.Join(targetPath, relPath) if info.IsDir() { - // Instead of resetting the base for relative paths, - // use the new recursive function that preserves the global relative path. - if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { - return err + if shallow { + // Use shallow copy: copy only immediate file entries. + l.Debug("Directory is not copied becasue it is a shallow copy", "directory", relPath) + } else { + // Use the existing recursive copy with prefix. + if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { + return err + } } } else { if err := copyFile(file, dstPath); err != nil { @@ -366,8 +375,11 @@ func copyToTargetWithPatterns( } } } - } else { - // No inclusion patterns defined; copy everything except those matching excluded items. + } + + // If no inclusion patterns are defined; copy everything except those matching excluded items. + // (This branch is preserved from the original logic.) + if len(s.IncludedPaths) == 0 { if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) } diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index ba3c016f5..0d641f09d 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -72,3 +72,16 @@ spec: - "components/library/" tags: - demo + + - component: "test shallow globs and folder exclusion" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/demo-localstack/*" + - "**/demo-library/**" + excluded_paths: + - "**/demo-library/**/stargazers/**" + - "**/demo-library/**/*.tf" + targets: + - "components/globs/" + tags: + - demo diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 367742db9..28ca2b165 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -193,4 +193,11 @@ tests: - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" - "./components/library/weather/README.md" + - "./components/globs/examples/demo-library/ipinfo/README.md" + - "./components/globs/examples/demo-library/weather/README.md" + - "./components/globs/examples/demo-library/README.md" + - "./components/globs/examples/demo-localstack/.gitignore" + - "./components/globs/examples/demo-localstack/atmos.yaml" + - "./components/globs/examples/demo-localstack/docker-compose.yml" + - "./components/globs/examples/demo-localstack/README.md" exit_code: 0 From 618860d12ef3eaee28fc65e3ff4f92dada754d7e Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 10 Feb 2025 16:13:35 +0500 Subject: [PATCH 20/68] docs added --- .../core-concepts/vendor/vendor-manifest.mdx | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index c41e2fec0..ecaa6af4b 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -217,7 +217,7 @@ The `vendor.yaml` vendoring manifest supports Kubernetes-style YAML config to de
`included_paths` and `excluded_paths`
- `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). + `included_paths` and `excluded_paths` support [POSIX-style greedy Globs](https://en.wikipedia.org/wiki/Glob_(programming)) for filenames/paths (double-star/globstar `**` is supported as well). For more details, see [Vendoring with Globs](#vendoring-with-globs).
`component`
@@ -497,3 +497,91 @@ To vendor the `vpc` component, execute the following command: atmos vendor pull -c vpc ``` + +## Vendoring with Globs + +When defining vendoring rules in Atmos, **glob patterns** determine which files and directories are included or excluded. Understanding how globs behave—especially when using greedy (`**`) versus non-greedy (`*`) patterns—is crucial for precise vendoring. + +### Understanding Greedy and Non-Greedy Globs + +Globs use special wildcard characters: + +- `*` (single asterisk) matches any sequence of characters **within a single path segment**. +- `**` (double asterisk) matches across multiple path segments **recursively**. + +This distinction is important when excluding specific directories or files while vendoring. + +#### Example: Excluding a Subdirectory + +Consider the following configuration: + +```yaml +included_paths: + - "**/demo-library/**" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +How It Works: +- The included_paths rule `**/demo-library/**` ensures all files inside `demo-library` (at any depth) are vendored. +- The excluded_paths rule `**/demo-library/**/stargazers/**` prevents any files inside `stargazers` subdirectories from being vendored. + +This means: +- All files within demo-library except those inside any `stargazers` subdirectory are vendored. +- Any other files outside `stargazers` are unaffected by this exclusion. + +Example: A Non-Recursive Pattern That Doesn't Work + +```yaml +included_paths: + - "**/demo-library/*" +excluded_paths: + - "**/demo-library/**/stargazers/**" +``` + +In this case: +- `**/demo-library/*` only matches immediate children of demo-library, not nested files or subdirectories. +- This means `stargazers/` itself could be matched, but its contents might not be explicitly excluded. +- To correctly capture all subdirectories and files while still excluding stargazers, use `**/demo-library/**/*`. + +Using `{...}` for Multiple Extensions or Patterns + +Curly braces `{...}` allow for expanding multiple patterns into separate glob matches. This is useful when selecting multiple file types or directories within a single glob pattern. + +Example: Matching Multiple File Extensions + +```yaml +included_paths: + - "**/demo-library/**/*.{tf,md}" +``` + +This is equivalent to writing: + +```yaml +included_paths: + - "**/demo-library/**/*.tf" + - "**/demo-library/**/*.md" +``` + +The `{tf,md}` part expands to both `*.tf` and `*.md`, making the rule more concise. + +Example: Excluding Multiple Directories + +```yaml +excluded_paths: + - "**/demo-library/**/{stargazers,archive}/**" +``` + +This excludes both: +- `**/demo-library/**/stargazers/**` +- `**/demo-library/**/archive/**` + +Using `{...}` here prevents the need to write two separate exclusion rules. + +Key Takeaways + 1. Use `**/` for recursive matching to include everything inside a directory. + 2. Use `*` for single-segment matches, which won't include deeper subdirectories. + 3. Use `{...}` to match multiple extensions or directories within a single pattern. + 4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. + +By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories like stargazers are omitted. \ No newline at end of file From d2533a7e9d20bc7bcb609bb21a538f0b3a9bfc29 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:37:24 +0600 Subject: [PATCH 21/68] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index ecaa6af4b..0d17306e3 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -579,9 +579,9 @@ This excludes both: Using `{...}` here prevents the need to write two separate exclusion rules. Key Takeaways - 1. Use `**/` for recursive matching to include everything inside a directory. - 2. Use `*` for single-segment matches, which won't include deeper subdirectories. - 3. Use `{...}` to match multiple extensions or directories within a single pattern. - 4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. +1. Use `**/` for recursive matching to include everything inside a directory. +2. Use `*` for single-segment matches, which won't include deeper subdirectories. +3. Use `{...}` to match multiple extensions or directories within a single pattern. +4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories like stargazers are omitted. \ No newline at end of file From 7e7384f9127140c0a24f4621daf491cb1b2d0d87 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 12 Feb 2025 15:10:42 +0500 Subject: [PATCH 22/68] correct ssh-style github URLs that use semicolon as a separator --- internal/exec/go_getter_utils.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index f76699f58..5261a4d6e 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "time" + "unicode" "github.com/google/uuid" "github.com/hashicorp/go-getter" @@ -78,6 +79,20 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { src = "https://" + src } + // Correct SSH-style GitHub URLs that use ":" as separator. + // e.g. transforms "https://git@github.com:cloudposse/terraform-null-label.git?ref=..." + // into "https://git@github.com/cloudposse/terraform-null-label.git?ref=..." + if strings.HasPrefix(src, "https://") { + const search = "github.com:" + if idx := strings.Index(src, search); idx != -1 { + // Only replace if the colon isn't followed by a digit (i.e. not a valid port) + if len(src) > idx+len(search) && !unicode.IsDigit(rune(src[idx+len(search)])) { + src = strings.Replace(src, search, "github.com/", 1) + } + } + } + fmt.Println("src: ", src) + parsedURL, err := url.Parse(src) if err != nil { u.LogDebug(fmt.Sprintf("Failed to parse URL %q: %v\n", src, err)) From b06181e6192559ca340ddb83f5aaeb215f8fa8fc Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 12 Feb 2025 15:39:51 +0500 Subject: [PATCH 23/68] degug verbose removal --- internal/exec/go_getter_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 5261a4d6e..db6a6099c 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -91,7 +91,6 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { } } } - fmt.Println("src: ", src) parsedURL, err := url.Parse(src) if err != nil { From d330de35cc52b1c9e8ae3249f49126f389df1be6 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 13 Feb 2025 12:05:39 +0500 Subject: [PATCH 24/68] reverting the recent changes --- internal/exec/go_getter_utils.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index db6a6099c..f76699f58 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strings" "time" - "unicode" "github.com/google/uuid" "github.com/hashicorp/go-getter" @@ -79,19 +78,6 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { src = "https://" + src } - // Correct SSH-style GitHub URLs that use ":" as separator. - // e.g. transforms "https://git@github.com:cloudposse/terraform-null-label.git?ref=..." - // into "https://git@github.com/cloudposse/terraform-null-label.git?ref=..." - if strings.HasPrefix(src, "https://") { - const search = "github.com:" - if idx := strings.Index(src, search); idx != -1 { - // Only replace if the colon isn't followed by a digit (i.e. not a valid port) - if len(src) > idx+len(search) && !unicode.IsDigit(rune(src[idx+len(search)])) { - src = strings.Replace(src, search, "github.com/", 1) - } - } - } - parsedURL, err := url.Parse(src) if err != nil { u.LogDebug(fmt.Sprintf("Failed to parse URL %q: %v\n", src, err)) From d05a1de44d3352d9f1d91bc96d8cd11734ed06aa Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:23:06 +0600 Subject: [PATCH 25/68] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 0d17306e3..c3651a2b4 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -530,7 +530,7 @@ This means: - All files within demo-library except those inside any `stargazers` subdirectory are vendored. - Any other files outside `stargazers` are unaffected by this exclusion. -Example: A Non-Recursive Pattern That Doesn't Work +#### Example: A Non-Recursive Pattern That Doesn't Work ```yaml included_paths: From be1c8ceb3542b92bdcbecdd00c955c849a6a16c5 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:23:41 +0600 Subject: [PATCH 26/68] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index c3651a2b4..9530faf1c 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -548,7 +548,7 @@ Using `{...}` for Multiple Extensions or Patterns Curly braces `{...}` allow for expanding multiple patterns into separate glob matches. This is useful when selecting multiple file types or directories within a single glob pattern. -Example: Matching Multiple File Extensions +#### Example: Matching Multiple File Extensions ```yaml included_paths: From a08f63eff572db646705a3f0cd21f1d81f6538ea Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:24:19 +0600 Subject: [PATCH 27/68] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 9530faf1c..40d991cf2 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -565,7 +565,7 @@ included_paths: The `{tf,md}` part expands to both `*.tf` and `*.md`, making the rule more concise. -Example: Excluding Multiple Directories +#### Example: Excluding Multiple Directories ```yaml excluded_paths: From 10118a12596a887216e8492b7c733ccf34075038 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:24:43 +0600 Subject: [PATCH 28/68] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 40d991cf2..c13063af7 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -578,7 +578,8 @@ This excludes both: Using `{...}` here prevents the need to write two separate exclusion rules. -Key Takeaways +## Key Takeaways + 1. Use `**/` for recursive matching to include everything inside a directory. 2. Use `*` for single-segment matches, which won't include deeper subdirectories. 3. Use `{...}` to match multiple extensions or directories within a single pattern. From 89827fdf83e286ba82c4d82b79f6705bb1b4f2a4 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:25:21 +0600 Subject: [PATCH 29/68] Update website/docs/core-concepts/vendor/vendor-manifest.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- website/docs/core-concepts/vendor/vendor-manifest.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index c13063af7..9de844a01 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -585,4 +585,4 @@ Using `{...}` here prevents the need to write two separate exclusion rules. 3. Use `{...}` to match multiple extensions or directories within a single pattern. 4. Exclusion rules must match nested paths explicitly when trying to exclude deep directories. -By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories like stargazers are omitted. \ No newline at end of file +By carefully combining `included_paths`, `excluded_paths`, and `{...}` expansion, you can precisely control which files are vendored while ensuring unwanted directories are omitted. \ No newline at end of file From 4c511040f10687e51fbf455e6796d6a69b249510 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sat, 22 Feb 2025 17:31:17 +0500 Subject: [PATCH 30/68] new testcases --- tests/cli_test.go | 34 ++++++++++++++---- tests/test-cases/demo-globs.yaml | 60 +++++++++++++++++++++++++++++++ tests/test-cases/demo-stacks.yaml | 17 --------- 3 files changed, 87 insertions(+), 24 deletions(-) create mode 100644 tests/test-cases/demo-globs.yaml diff --git a/tests/cli_test.go b/tests/cli_test.go index acd33b311..e30e4f499 100644 --- a/tests/cli_test.go +++ b/tests/cli_test.go @@ -46,13 +46,14 @@ var ( var logger *log.Logger type Expectation struct { - Stdout []MatchPattern `yaml:"stdout"` // Expected stdout output - Stderr []MatchPattern `yaml:"stderr"` // Expected stderr output - ExitCode int `yaml:"exit_code"` // Expected exit code - FileExists []string `yaml:"file_exists"` // Files to validate - FileContains map[string][]MatchPattern `yaml:"file_contains"` // File contents to validate (file to patterns map) - Diff []string `yaml:"diff"` // Acceptable differences in snapshot - Timeout string `yaml:"timeout"` // Maximum execution time as a string, e.g., "1s", "1m", "1h", or a number (seconds) + Stdout []MatchPattern `yaml:"stdout"` // Expected stdout output + Stderr []MatchPattern `yaml:"stderr"` // Expected stderr output + ExitCode int `yaml:"exit_code"` // Expected exit code + FileExists []string `yaml:"file_exists"` // Files to validate + FileNotExists []string `yaml:"file_not_exists"` // Files that should not exist + FileContains map[string][]MatchPattern `yaml:"file_contains"` // File contents to validate (file to patterns map) + Diff []string `yaml:"diff"` // Acceptable differences in snapshot + Timeout string `yaml:"timeout"` // Maximum execution time as a string, e.g., "1s", "1m", "1h", or a number (seconds) } type TestCase struct { Name string `yaml:"name"` // Name of the test @@ -658,6 +659,11 @@ func runCLICommandTest(t *testing.T, tc TestCase) { t.Errorf("Description: %s", tc.Description) } + // Validate file not existence + if !verifyFileNotExists(t, tc.Expect.FileNotExists) { + t.Errorf("Description: %s", tc.Description) + } + // Validate file contents if !verifyFileContains(t, tc.Expect.FileContains) { t.Errorf("Description: %s", tc.Description) @@ -766,6 +772,20 @@ func verifyFileExists(t *testing.T, files []string) bool { return success } +func verifyFileNotExists(t *testing.T, files []string) bool { + success := true + for _, file := range files { + if _, err := os.Stat(file); err == nil { + t.Errorf("Reason: File %q exists but it should not.", file) + success = false + } else if !errors.Is(err, os.ErrNotExist) { + t.Errorf("Reason: Unexpected error checking file %q: %v", file, err) + success = false + } + } + return success +} + func verifyFileContains(t *testing.T, filePatterns map[string][]MatchPattern) bool { success := true for file, patterns := range filePatterns { diff --git a/tests/test-cases/demo-globs.yaml b/tests/test-cases/demo-globs.yaml new file mode 100644 index 000000000..17092ee74 --- /dev/null +++ b/tests/test-cases/demo-globs.yaml @@ -0,0 +1,60 @@ +tests: + - name: atmos_vendor_pull_with_globs + enabled: true + description: "Ensure atmos vendor pull command executes without errors and files are present." + workdir: "fixtures/scenarios/vendor" + command: "atmos" + args: + - "vendor" + - "pull" + expect: + file_exists: + - "./components/library/examples/demo-library/github/stargazers/README.md" + - "./components/library/examples/demo-library/ipinfo/README.md" + - "./components/library/examples/demo-library/weather/README.md" + - "./components/library/examples/demo-library/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" + - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" + - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" + - "./components/library/weather/README.md" + - "./components/globs/examples/demo-library/ipinfo/README.md" + - "./components/globs/examples/demo-library/weather/README.md" + - "./components/globs/examples/demo-library/README.md" + - "./components/globs/examples/demo-localstack/.gitignore" + - "./components/globs/examples/demo-localstack/atmos.yaml" + - "./components/globs/examples/demo-localstack/docker-compose.yml" + - "./components/globs/examples/demo-localstack/README.md" + file_not_exists: + - "./components/library/examples/demo-library/github/stargazers/main.tf" + - "./components/library/examples/demo-library/github/stargazers/outputs.tf" + - "./components/library/examples/demo-library/github/stargazers/providers.tf" + - "./components/library/examples/demo-library/github/stargazers/variables.tf" + - "./components/library/examples/demo-library/github/stargazers/versions.tf" + - "./components/library/examples/demo-library/ipinfo/main.tf" + - "./components/library/examples/demo-library/ipinfo/outputs.tf" + - "./components/library/examples/demo-library/ipinfo/providers.tf" + - "./components/library/examples/demo-library/ipinfo/variables.tf" + - "./components/library/examples/demo-library/ipinfo/versions.tf" + - "./components/library/examples/demo-library/weather/main.tf" + - "./components/library/examples/demo-library/weather/outputs.tf" + - "./components/library/examples/demo-library/weather/providers.tf" + - "./components/library/examples/demo-library/weather/variables.tf" + - "./components/library/examples/demo-library/weather/versions.tf" + - "./components/globs/examples/demo-library/github/stargazers/README.md" + - "./components/globs/examples/demo-library/github/stargazers/main.tf" + - "./components/globs/examples/demo-library/github/stargazers/outputs.tf" + - "./components/globs/examples/demo-library/github/stargazers/providers.tf" + - "./components/globs/examples/demo-library/github/stargazers/variables.tf" + - "./components/globs/examples/demo-library/github/stargazers/versions.tf" + - "./components/globs/examples/demo-library/ipinfo/outputs.tf" + - "./components/globs/examples/demo-library/ipinfo/providers.tf" + - "./components/globs/examples/demo-library/ipinfo/variables.tf" + - "./components/globs/examples/demo-library/ipinfo/versions.tf" + - "./components/globs/examples/demo-library/weather/main.tf" + - "./components/globs/examples/demo-library/weather/outputs.tf" + - "./components/globs/examples/demo-library/weather/providers.tf" + - "./components/globs/examples/demo-library/weather/variables.tf" + - "./components/globs/examples/demo-library/weather/versions.tf" + exit_code: 0 diff --git a/tests/test-cases/demo-stacks.yaml b/tests/test-cases/demo-stacks.yaml index 28ca2b165..3fc5c07a3 100644 --- a/tests/test-cases/demo-stacks.yaml +++ b/tests/test-cases/demo-stacks.yaml @@ -183,21 +183,4 @@ tests: - "./components/terraform/vpc-src/outputs.tf" - "./components/terraform/vpc-src/variables.tf" - "./components/terraform/vpc-src/versions.tf" - - "./components/library/examples/demo-library/github/stargazers/README.md" - - "./components/library/examples/demo-library/ipinfo/README.md" - - "./components/library/examples/demo-library/weather/README.md" - - "./components/library/examples/demo-library/README.md" - - "./components/library/examples/demo-stacks/components/terraform/myapp/main.tf" - - "./components/library/examples/demo-stacks/components/terraform/myapp/outputs.tf" - - "./components/library/examples/demo-stacks/components/terraform/myapp/README.md" - - "./components/library/examples/demo-stacks/components/terraform/myapp/variables.tf" - - "./components/library/examples/demo-stacks/components/terraform/myapp/versions.tf" - - "./components/library/weather/README.md" - - "./components/globs/examples/demo-library/ipinfo/README.md" - - "./components/globs/examples/demo-library/weather/README.md" - - "./components/globs/examples/demo-library/README.md" - - "./components/globs/examples/demo-localstack/.gitignore" - - "./components/globs/examples/demo-localstack/atmos.yaml" - - "./components/globs/examples/demo-localstack/docker-compose.yml" - - "./components/globs/examples/demo-localstack/README.md" exit_code: 0 From d854d4bd45b41bbcfde0af22b2bf2f571736d333 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sat, 22 Feb 2025 19:08:23 +0500 Subject: [PATCH 31/68] tests/golangci --- .gitignore | 1 + internal/exec/copy_glob.go | 427 +++++++++--------- internal/exec/go_getter_utils.go | 226 ++++++--- pkg/utils/url_utils.go | 20 + .../scenarios/vendor-globs/atmos.yaml | 40 ++ .../scenarios/vendor-globs/vendor.yaml | 43 ++ tests/fixtures/scenarios/vendor/vendor.yaml | 33 -- tests/test-cases/demo-globs.yaml | 2 +- 8 files changed, 476 insertions(+), 316 deletions(-) create mode 100644 pkg/utils/url_utils.go create mode 100644 tests/fixtures/scenarios/vendor-globs/atmos.yaml create mode 100644 tests/fixtures/scenarios/vendor-globs/vendor.yaml diff --git a/.gitignore b/.gitignore index 9c4d9f1ef..17c882487 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ # Ignore components vendored during tests tests/fixtures/scenarios/vendor/components/** +tests/fixtures/scenarios/vendor-globs/components/** examples/demo-vendoring/components/** diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 5528d9303..b009bf824 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -13,6 +13,13 @@ import ( cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. ) +// Named constants to avoid literal duplication. +const ( + logKeyPath = "path" + logKeyError = "error" + shallowCopySuffix = "/*" +) + // copyFile copies a single file from src to dst while preserving file permissions. func copyFile(src, dst string) error { sourceFile, err := os.Open(src) @@ -45,196 +52,195 @@ func copyFile(src, dst string) error { return nil } -// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. -// If an error occurs during matching for an exclusion or inclusion pattern, it logs the error and proceeds. -func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { - if info.Name() == ".git" { - return true, nil - } - relPath, err := filepath.Rel(baseDir, srcPath) - if err != nil { - l.Debug("Error computing relative path", "srcPath", srcPath, "error", err) - return true, nil // treat error as a signal to skip - } - // Ensure uniform path separator. - relPath = filepath.ToSlash(relPath) - - // Process exclusion patterns. - // For directories, check with and without a trailing slash. +// shouldExcludePath checks exclusion patterns for a given relative path and file info. +func shouldExcludePath(info os.FileInfo, relPath string, excluded []string) bool { for _, pattern := range excluded { - // First check the plain relative path. + // Check plain relative path. matched, err := u.PathMatch(pattern, relPath) if err != nil { - l.Debug("Error matching exclusion pattern", "pattern", pattern, "path", relPath, "error", err) + l.Debug("Error matching exclusion pattern", logKeyPath, relPath, logKeyError, err) continue } if matched { - l.Debug("Excluding path due to exclusion pattern (plain match)", "path", relPath, "pattern", pattern) - return true, nil + l.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, "pattern", pattern) + return true } - // If it is a directory, also try matching with a trailing slash. + // If a directory, also check with a trailing slash. if info.IsDir() { matched, err = u.PathMatch(pattern, relPath+"/") if err != nil { - l.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, "path", relPath+"/", "error", err) + l.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, logKeyPath, relPath+"/", logKeyError, err) continue } if matched { - l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "path", relPath+"/", "pattern", pattern) - return true, nil + l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", "pattern", pattern) + return true } } } + return false +} - // Process inclusion patterns (only for non-directory files). - // (Directories are generally picked up by the inclusion branch in copyToTargetWithPatterns.) - if len(included) > 0 && !info.IsDir() { - matchedAny := false - for _, pattern := range included { - matched, err := u.PathMatch(pattern, relPath) - if err != nil { - l.Debug("Error matching inclusion pattern", "pattern", pattern, "path", relPath, "error", err) - continue - } - if matched { - l.Debug("Including path due to inclusion pattern", "path", relPath, "pattern", pattern) - matchedAny = true - break - } +// shouldIncludePath checks whether a file should be included based on inclusion patterns. +func shouldIncludePath(info os.FileInfo, relPath string, included []string) bool { + // Directories are generally handled by recursion. + if len(included) == 0 || info.IsDir() { + return true + } + for _, pattern := range included { + matched, err := u.PathMatch(pattern, relPath) + if err != nil { + l.Debug("Error matching inclusion pattern", "pattern", pattern, logKeyPath, relPath, logKeyError, err) + continue } - if !matchedAny { - l.Debug("Excluding path because it does not match any inclusion pattern", "path", relPath) - return true, nil + if matched { + l.Debug("Including path due to inclusion pattern", logKeyPath, relPath, "pattern", pattern) + return true } } + l.Debug("Excluding path because it does not match any inclusion pattern", logKeyPath, relPath) + return false +} + +// shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { + if info.Name() == ".git" { + return true, nil + } + relPath, err := filepath.Rel(baseDir, srcPath) + if err != nil { + l.Debug("Error computing relative path", "srcPath", srcPath, logKeyError, err) + return true, nil // treat error as a signal to skip + } + relPath = filepath.ToSlash(relPath) + if shouldExcludePath(info, relPath, excluded) { + return true, nil + } + if !shouldIncludePath(info, relPath, included) { + return true, nil + } return false, nil } +// processDirEntry handles a single directory entry for copyDirRecursive. +func processDirEntry(entry os.DirEntry, srcDir, dstDir, baseDir string, excluded, included []string) error { + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() + if err != nil { + return fmt.Errorf("getting info for %q: %w", srcPath, err) + } + + skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) + if err != nil { + return err + } + if skip { + l.Debug("Skipping entry", "srcPath", srcPath) + return nil + } + + // Skip symlinks. + if info.Mode()&os.ModeSymlink != 0 { + l.Debug("Skipping symlink", logKeyPath, srcPath) + return nil + } + + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) + } + return copyDirRecursive(srcPath, dstPath, baseDir, excluded, included) + } + return copyFile(srcPath, dstPath) +} + // copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. -// This function is used in cases where the entire sourceDir is the base for relative paths. func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { entries, err := os.ReadDir(srcDir) if err != nil { return fmt.Errorf("reading directory %q: %w", srcDir, err) } for _, entry := range entries { - srcPath := filepath.Join(srcDir, entry.Name()) - dstPath := filepath.Join(dstDir, entry.Name()) - - info, err := entry.Info() - if err != nil { - return fmt.Errorf("getting info for %q: %w", srcPath, err) + if err := processDirEntry(entry, srcDir, dstDir, baseDir, excluded, included); err != nil { + return err } + } + return nil +} - // Check if this entry should be skipped. - skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) +// shouldSkipPrefixEntry checks exclusion patterns for copyDirRecursiveWithPrefix. +func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []string) bool { + for _, pattern := range excluded { + matched, err := u.PathMatch(pattern, fullRelPath) if err != nil { - return err - } - if skip { - l.Debug("Skipping entry", "srcPath", srcPath) + l.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, logKeyPath, fullRelPath, logKeyError, err) continue } - - // Skip symlinks. - if info.Mode()&os.ModeSymlink != 0 { - l.Debug("Skipping symlink", "path", srcPath) - continue + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, "pattern", pattern) + return true } - if info.IsDir() { - if err := os.MkdirAll(dstPath, info.Mode()); err != nil { - return fmt.Errorf("creating directory %q: %w", dstPath, err) - } - if err := copyDirRecursive(srcPath, dstPath, baseDir, excluded, included); err != nil { - return err + matched, err = u.PathMatch(pattern, fullRelPath+"/") + if err != nil { + l.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, logKeyPath, fullRelPath+"/", logKeyError, err) + continue } - } else { - if err := copyFile(srcPath, dstPath); err != nil { - return err + if matched { + l.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", "pattern", pattern) + return true } } } - return nil + return false } -// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. -// Instead of using the local srcDir as the base for computing relative paths, this function uses the original -// source directory (globalBase) and an accumulated prefix that represents the relative path from globalBase. -func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { - entries, err := os.ReadDir(srcDir) +// processPrefixEntry handles a single entry for copyDirRecursiveWithPrefix. +func processPrefixEntry(entry os.DirEntry, srcDir, dstDir, globalBase, prefix string, excluded []string) error { + fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) + srcPath := filepath.Join(srcDir, entry.Name()) + dstPath := filepath.Join(dstDir, entry.Name()) + + info, err := entry.Info() if err != nil { - return fmt.Errorf("reading directory %q: %w", srcDir, err) + return fmt.Errorf("getting info for %q: %w", srcPath, err) } - for _, entry := range entries { - // Compute the full relative path from the original source. - fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) - srcPath := filepath.Join(srcDir, entry.Name()) - dstPath := filepath.Join(dstDir, entry.Name()) - info, err := entry.Info() - if err != nil { - return fmt.Errorf("getting info for %q: %w", srcPath, err) - } + if entry.Name() == ".git" { + l.Debug("Skipping .git directory", logKeyPath, fullRelPath) + return nil + } - // Skip .git directories. - if entry.Name() == ".git" { - l.Debug("Skipping .git directory", "path", fullRelPath) - continue - } + if shouldSkipPrefixEntry(info, fullRelPath, excluded) { + return nil + } - // Check exclusion patterns using the full relative path. - skip := false - for _, pattern := range excluded { - // Check plain match. - matched, err := u.PathMatch(pattern, fullRelPath) - if err != nil { - l.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, "path", fullRelPath, "error", err) - continue - } - if matched { - l.Debug("Excluding (prefix) due to exclusion pattern (plain match)", "path", fullRelPath, "pattern", pattern) - skip = true - break - } - // For directories, also try with a trailing slash. - if info.IsDir() { - matched, err = u.PathMatch(pattern, fullRelPath+"/") - if err != nil { - l.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, "path", fullRelPath+"/", "error", err) - continue - } - if matched { - l.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", "path", fullRelPath+"/", "pattern", pattern) - skip = true - break - } - } - } - if skip { - continue + if info.IsDir() { + if err := os.MkdirAll(dstPath, info.Mode()); err != nil { + return fmt.Errorf("creating directory %q: %w", dstPath, err) } + return copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded) + } + return copyFile(srcPath, dstPath) +} - if info.IsDir() { - if err := os.MkdirAll(dstPath, info.Mode()); err != nil { - return fmt.Errorf("creating directory %q: %w", dstPath, err) - } - // Recurse with updated prefix. - if err := copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded); err != nil { - return err - } - } else { - if err := copyFile(srcPath, dstPath); err != nil { - return err - } +// copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. +func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return fmt.Errorf("reading directory %q: %w", srcDir, err) + } + for _, entry := range entries { + if err := processPrefixEntry(entry, srcDir, dstDir, globalBase, prefix, excluded); err != nil { + return err } } return nil } // getMatchesForPattern returns files/directories matching a pattern relative to sourceDir. -// If no matches are found, it logs a debug message and returns an empty slice. -// For patterns ending with "/*" (shallow copy indicator) the function does not fallback to a recursive variant. func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { fullPattern := filepath.Join(sourceDir, pattern) matches, err := u.GetGlobMatches(fullPattern) @@ -242,14 +248,14 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) } if len(matches) == 0 { - // If the pattern ends with "/*" (and not "/**"), do not fallback. - if strings.HasSuffix(pattern, "/*") && !strings.HasSuffix(pattern, "/**") { + // If the pattern ends with "/*" (shallow copy indicator) do not fallback. + if strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") { l.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) return []string{}, nil } - // Fallback for patterns ending with "/*" (non-shallow) or others. - if strings.HasSuffix(pattern, "/*") { - recursivePattern := strings.TrimSuffix(pattern, "/*") + "/**" + // Fallback for non-shallow patterns. + if strings.HasSuffix(pattern, shallowCopySuffix) { + recursivePattern := strings.TrimSuffix(pattern, shallowCopySuffix) + "/**" fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) matches, err = u.GetGlobMatches(fullRecursivePattern) if err != nil { @@ -267,12 +273,59 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { return matches, nil } +// isShallowPattern determines if a pattern indicates a shallow copy. +func isShallowPattern(pattern string) bool { + return strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") +} + +// processMatch handles a single file/directory match for copyToTargetWithPatterns. +func processMatch(sourceDir, targetPath, file string, shallow bool, excluded []string) error { + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("stating file %q: %w", file, err) + } + relPath, err := filepath.Rel(sourceDir, file) + if err != nil { + return fmt.Errorf("computing relative path for %q: %w", file, err) + } + relPath = filepath.ToSlash(relPath) + if shouldExcludePath(info, relPath, excluded) { + return nil + } + + dstPath := filepath.Join(targetPath, relPath) + if info.IsDir() { + if shallow { + l.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) + return nil + } + return copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, excluded) + } + return copyFile(file, dstPath) +} + +// processIncludedPattern handles all matches for one inclusion pattern. +func processIncludedPattern(sourceDir, targetPath, pattern string, excluded []string) error { + shallow := isShallowPattern(pattern) + matches, err := getMatchesForPattern(sourceDir, pattern) + if err != nil { + l.Debug("Warning: error getting matches for pattern", "pattern", pattern, logKeyError, err) + return nil + } + if len(matches) == 0 { + l.Debug("No files matched the inclusion pattern", "pattern", pattern) + return nil + } + for _, file := range matches { + if err := processMatch(sourceDir, targetPath, file, shallow, excluded); err != nil { + return err + } + } + return nil +} + // copyToTargetWithPatterns copies the contents from sourceDir to targetPath, // applying inclusion and exclusion patterns from the vendor source configuration. -// If sourceIsLocalFile is true and targetPath lacks an extension, the sanitized URI is appended. -// If no included paths are defined, all files (except those matching excluded paths) are copied. -// In the special case where neither inclusion nor exclusion patterns are defined, -// the optimized cp library (github.com/otiai10/copy) is used. func copyToTargetWithPatterns( sourceDir, targetPath string, s *schema.AtmosVendorSource, @@ -287,98 +340,20 @@ func copyToTargetWithPatterns( return fmt.Errorf("creating target directory %q: %w", targetPath, err) } - // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library for fast copying. + // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library. if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { l.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") return cp.Copy(sourceDir, targetPath) } - // If inclusion patterns are provided, process each pattern individually. + // Process each inclusion pattern. for _, pattern := range s.IncludedPaths { - // Determine if the pattern indicates shallow copy. - shallow := false - if strings.HasSuffix(pattern, "/*") && !strings.HasSuffix(pattern, "/**") { - shallow = true - } - - matches, err := getMatchesForPattern(sourceDir, pattern) - if err != nil { - l.Debug("Warning: error getting matches for pattern", "pattern", pattern, "error", err) - continue - } - if len(matches) == 0 { - l.Debug("No files matched the inclusion pattern", "pattern", pattern) - continue - } - for _, file := range matches { - // Retrieve file information. - info, err := os.Stat(file) - if err != nil { - return fmt.Errorf("stating file %q: %w", file, err) - } - relPath, err := filepath.Rel(sourceDir, file) - if err != nil { - return fmt.Errorf("computing relative path for %q: %w", file, err) - } - relPath = filepath.ToSlash(relPath) - - // Check exclusion patterns (for directories, try both plain and trailing slash). - skip := false - for _, ex := range s.ExcludedPaths { - if info.IsDir() { - matched, err := u.PathMatch(ex, relPath) - if err != nil { - l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) - } else if matched { - l.Debug("Excluding directory due to exclusion pattern (plain match)", "directory", relPath, "pattern", ex) - skip = true - break - } - matched, err = u.PathMatch(ex, relPath+"/") - if err != nil { - l.Debug("Error matching exclusion pattern with trailing slash", "pattern", ex, "path", relPath+"/", "error", err) - } else if matched { - l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", "directory", relPath+"/", "pattern", ex) - skip = true - break - } - } else { - matched, err := u.PathMatch(ex, relPath) - if err != nil { - l.Debug("Error matching exclusion pattern", "pattern", ex, "path", relPath, "error", err) - } else if matched { - l.Debug("Excluding file due to exclusion pattern", "file", relPath, "pattern", ex) - skip = true - break - } - } - } - if skip { - continue - } - - // Build the destination path. - dstPath := filepath.Join(targetPath, relPath) - if info.IsDir() { - if shallow { - // Use shallow copy: copy only immediate file entries. - l.Debug("Directory is not copied becasue it is a shallow copy", "directory", relPath) - } else { - // Use the existing recursive copy with prefix. - if err := copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, s.ExcludedPaths); err != nil { - return err - } - } - } else { - if err := copyFile(file, dstPath); err != nil { - return err - } - } + if err := processIncludedPattern(sourceDir, targetPath, pattern, s.ExcludedPaths); err != nil { + return err } } - // If no inclusion patterns are defined; copy everything except those matching excluded items. - // (This branch is preserved from the original logic.) + // If no inclusion patterns are defined, copy everything except those matching excluded items. if len(s.IncludedPaths) == 0 { if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 00a2d7558..24403deee 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -8,9 +8,11 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" "time" + "github.com/charmbracelet/log" "github.com/google/uuid" "github.com/hashicorp/go-getter" @@ -64,93 +66,205 @@ func IsValidScheme(scheme string) bool { // CustomGitHubDetector intercepts GitHub URLs and transforms them // into something like git::https://@github.com/... so we can // do a git-based clone with a token. -type CustomGitHubDetector struct { +type CustomGitDetector struct { AtmosConfig schema.AtmosConfiguration source string } // Detect implements the getter.Detector interface for go-getter v1. -func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { +func (d *CustomGitDetector) Detect(src, _ string) (string, bool, error) { + log.Debug("CustomGitDetector.Detect called") + if len(src) == 0 { return "", false, nil } - if !strings.Contains(src, "://") { - src = "https://" + src - } + // Ensure the URL has an explicit scheme. + src = d.ensureScheme(src) + // Parse the URL to extract the host and path. parsedURL, err := url.Parse(src) if err != nil { - u.LogDebug(fmt.Sprintf("Failed to parse URL %q: %v\n", src, err)) - return "", false, fmt.Errorf("failed to parse URL %q: %w", src, err) + maskedSrc, _ := u.MaskBasicAuth(src) + log.Debug("Failed to parse URL", keyURL, maskedSrc, "error", err) + return "", false, fmt.Errorf("failed to parse URL %q: %w", maskedSrc, err) } - if strings.ToLower(parsedURL.Host) != "github.com" { - u.LogDebug(fmt.Sprintf("Host is %q, not 'github.com', skipping token injection\n", parsedURL.Host)) - return "", false, nil + // Normalize the path. + d.normalizePath(parsedURL) + + // Adjust host check to support GitHub, Bitbucket, GitLab, etc. + host := strings.ToLower(parsedURL.Host) + if host != "github.com" && host != "bitbucket.org" && host != "gitlab.com" { + log.Debug("Skipping token injection for a unsupported host", "host", parsedURL.Host) } - parts := strings.SplitN(parsedURL.Path, "/", 4) - if len(parts) < 3 { - u.LogDebug(fmt.Sprintf("URL path %q doesn't look like /owner/repo\n", parsedURL.Path)) - return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path) + log.Debug("Reading config param", "InjectGithubToken", d.AtmosConfig.Settings.InjectGithubToken) + // Inject token if available. + d.injectToken(parsedURL, host) + + // Adjust subdirectory if needed. + d.adjustSubdir(parsedURL, d.source) + + // Set "depth=1" for a shallow clone if not specified. + q := parsedURL.Query() + if _, exists := q["depth"]; !exists { + q.Set("depth", "1") } + parsedURL.RawQuery = q.Encode() - if !strings.Contains(d.source, "//") { - // means user typed something like "github.com/org/repo.git" with NO subdir - if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { - u.LogDebug("Detected top-level repo with no subdir: appending '//.'\n") - parsedURL.Path = parsedURL.Path + "//." + finalURL := "git::" + parsedURL.String() + maskedFinal, err := u.MaskBasicAuth(strings.TrimPrefix(finalURL, "git::")) + if err != nil { + log.Debug("Masking failed", "error", err) + } else { + log.Debug("Final URL", "final_url", "git::"+maskedFinal) + } + + return finalURL, true, nil +} + +const ( + // Named constants for regex match indices. + matchIndexUser = 1 + matchIndexHost = 3 + matchIndexPath = 4 + matchIndexSuffix = 5 + matchIndexExtra = 6 + + // Key for logging repeated "url" field. + keyURL = "url" +) + +// ensureScheme checks for an explicit scheme and rewrites SCP-style URLs if needed. +// This version no longer returns an error since it never produces one. +func (d *CustomGitDetector) ensureScheme(src string) string { + if !strings.Contains(src, "://") { + if newSrc, rewritten := rewriteSCPURL(src); rewritten { + maskedOld, _ := u.MaskBasicAuth(src) + maskedNew, _ := u.MaskBasicAuth(newSrc) + log.Debug("Rewriting SCP-style SSH URL", "old_url", maskedOld, "new_url", maskedNew) + return newSrc } + src = "https://" + src + maskedSrc, _ := u.MaskBasicAuth(src) + log.Debug("Defaulting to https scheme", keyURL, maskedSrc) } + return src +} - atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN") - gitHubToken := os.Getenv("GITHUB_TOKEN") +// rewriteSCPURL rewrites SCP-style URLs to a proper SSH URL if they match the expected pattern. +// Returns the rewritten URL and a boolean indicating if rewriting occurred. +func rewriteSCPURL(src string) (string, bool) { + scpPattern := regexp.MustCompile(`^(([\w.-]+)@)?([\w.-]+\.[\w.-]+):([\w./-]+)(\.git)?(.*)$`) + if scpPattern.MatchString(src) { + matches := scpPattern.FindStringSubmatch(src) + newSrc := "ssh://" + if matches[matchIndexUser] != "" { + newSrc += matches[matchIndexUser] // includes username and '@' + } + newSrc += matches[matchIndexHost] + "/" + matches[matchIndexPath] + if matches[matchIndexSuffix] != "" { + newSrc += matches[matchIndexSuffix] + } + if matches[matchIndexExtra] != "" { + newSrc += matches[matchIndexExtra] + } + return newSrc, true + } + return "", false +} - var usedToken string - var tokenSource string +// normalizePath converts the URL path to use forward slashes. +func (d *CustomGitDetector) normalizePath(parsedURL *url.URL) { + unescapedPath, err := url.PathUnescape(parsedURL.Path) + if err == nil { + parsedURL.Path = filepath.ToSlash(unescapedPath) + } else { + parsedURL.Path = filepath.ToSlash(parsedURL.Path) + } +} - // 1. If ATMOS_GITHUB_TOKEN is set, always use that - if atmosGitHubToken != "" { - usedToken = atmosGitHubToken - tokenSource = "ATMOS_GITHUB_TOKEN" - u.LogDebug("ATMOS_GITHUB_TOKEN is set\n") +// injectToken injects a token into the URL if available. +func (d *CustomGitDetector) injectToken(parsedURL *url.URL, host string) { + token, tokenSource := d.resolveToken(host) + if token != "" { + defaultUsername := getDefaultUsername(host) + parsedURL.User = url.UserPassword(defaultUsername, token) + maskedURL, _ := u.MaskBasicAuth(parsedURL.String()) + log.Debug("Injected token", "env", tokenSource, keyURL, maskedURL) } else { - // 2. Otherwise, only inject GITHUB_TOKEN if cfg.Settings.InjectGithubToken == true - if d.AtmosConfig.Settings.InjectGithubToken && gitHubToken != "" { - usedToken = gitHubToken - tokenSource = "GITHUB_TOKEN" - u.LogTrace("InjectGithubToken=true and GITHUB_TOKEN is set, using it\n") - } else { - u.LogTrace("No ATMOS_GITHUB_TOKEN or GITHUB_TOKEN found\n") - } + log.Debug("No token found for injection") } +} - if usedToken != "" { - user := parsedURL.User.Username() - pass, _ := parsedURL.User.Password() - if user == "" && pass == "" { - u.LogDebug(fmt.Sprintf("Injecting token from %s for %s\n", tokenSource, src)) - parsedURL.User = url.UserPassword("x-access-token", usedToken) +// resolveToken returns the token and its source based on the host. +func (d *CustomGitDetector) resolveToken(host string) (string, string) { + var token, tokenSource string + switch host { + case "github.com": + if d.AtmosConfig.Settings.InjectGithubToken { + tokenSource = "ATMOS_GITHUB_TOKEN" + token = os.Getenv(tokenSource) + if token == "" { + tokenSource = "GITHUB_TOKEN" + token = os.Getenv(tokenSource) + } } else { - u.LogDebug("Credentials found, skipping token injection\n") + tokenSource = "GITHUB_TOKEN" + token = os.Getenv(tokenSource) + } + case "bitbucket.org": + tokenSource = "BITBUCKET_TOKEN" + token = os.Getenv(tokenSource) + if token == "" { + tokenSource = "ATMOS_BITBUCKET_TOKEN" + token = os.Getenv(tokenSource) + } + case "gitlab.com": + tokenSource = "GITLAB_TOKEN" + token = os.Getenv(tokenSource) + if token == "" { + tokenSource = "ATMOS_GITLAB_TOKEN" + token = os.Getenv(tokenSource) } } + return token, tokenSource +} - // Set "depth=1" for a shallow clone if not specified. - // In Go-Getter, "depth" controls how many revisions are cloned: - // - `depth=1` fetches only the latest commit (faster, less bandwidth). - // - `depth=` (empty) performs a full clone (default Git behavior). - // - `depth=N` clones the last N revisions. - q := parsedURL.Query() - if _, exists := q["depth"]; !exists { - q.Set("depth", "1") +// getDefaultUsername returns the default username for token injection based on the host. +func getDefaultUsername(host string) string { + switch host { + case "github.com": + return "x-access-token" + case "gitlab.com": + return "oauth2" + case "bitbucket.org": + defaultUsername := os.Getenv("ATMOS_BITBUCKET_USERNAME") + if defaultUsername == "" { + defaultUsername = os.Getenv("BITBUCKET_USERNAME") + if defaultUsername == "" { + return "x-token-auth" + } + } + log.Debug("Using Bitbucket username", "username", defaultUsername) + return defaultUsername + default: + return "x-access-token" } - parsedURL.RawQuery = q.Encode() - - finalURL := "git::" + parsedURL.String() +} - return finalURL, true, nil +// adjustSubdir appends "//." to the path if no subdirectory is specified. +func (d *CustomGitDetector) adjustSubdir(parsedURL *url.URL, source string) { + normalizedSource := filepath.ToSlash(source) + if normalizedSource != "" && !strings.Contains(normalizedSource, "//") { + parts := strings.SplitN(parsedURL.Path, "/", 4) + if strings.HasSuffix(parsedURL.Path, ".git") || len(parts) == 3 { + maskedSrc, _ := u.MaskBasicAuth(source) + log.Debug("Detected top-level repo with no subdir: appending '//.'", keyURL, maskedSrc) + parsedURL.Path += "//." + } + } } // RegisterCustomDetectors prepends the custom detector so it runs before @@ -158,7 +272,7 @@ func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) { func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ - &CustomGitHubDetector{AtmosConfig: atmosConfig, source: source}, + &CustomGitDetector{AtmosConfig: atmosConfig, source: source}, }, getter.Detectors..., ) diff --git a/pkg/utils/url_utils.go b/pkg/utils/url_utils.go new file mode 100644 index 000000000..e176c377f --- /dev/null +++ b/pkg/utils/url_utils.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "net/url" +) + +// MaskBasicAuth replaces the username and password in a URL with "xxx" if present. +func MaskBasicAuth(rawURL string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + if parsedURL.User != nil { + parsedURL.User = url.UserPassword("xxx", "xxx") + } + + return parsedURL.String(), nil +} diff --git a/tests/fixtures/scenarios/vendor-globs/atmos.yaml b/tests/fixtures/scenarios/vendor-globs/atmos.yaml new file mode 100644 index 000000000..0f0506e81 --- /dev/null +++ b/tests/fixtures/scenarios/vendor-globs/atmos.yaml @@ -0,0 +1,40 @@ +base_path: "./" + +components: + terraform: + base_path: "components/terraform" + apply_auto_approve: false + deploy_run_init: true + init_run_reconfigure: true + auto_generate_backend_file: false + +stacks: + base_path: "stacks" + included_paths: + - "deploy/**/*" + excluded_paths: + - "**/_defaults.yaml" + name_pattern: "{stage}" + +vendor: + # Single file + base_path: "./vendor.yaml" + + # Directory with multiple files + #base_path: "./vendor" + + # Absolute path + #base_path: "vendor.d/vendor1.yaml" + +logs: + file: "/dev/stderr" + level: Info + +# Custom CLI commands + +# No arguments or flags are required +commands: +- name: "test" + description: "Run all tests" + steps: + - atmos vendor pull --everything diff --git a/tests/fixtures/scenarios/vendor-globs/vendor.yaml b/tests/fixtures/scenarios/vendor-globs/vendor.yaml new file mode 100644 index 000000000..061c06d19 --- /dev/null +++ b/tests/fixtures/scenarios/vendor-globs/vendor.yaml @@ -0,0 +1,43 @@ +apiVersion: atmos/v1 +kind: AtmosVendorConfig +metadata: + name: demo-vendoring + description: Atmos vendoring manifest for Atmos demo component library +spec: + # Import other vendor manifests, if necessary + imports: [] + + sources: + - component: "test globs" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/{demo-library,demo-stacks}/**/*.{tf,md}" + excluded_paths: + - "**/demo-library/**/*.{tfvars,tf}" + targets: + - "components/library/" + tags: + - demo + + - component: "test globs without double stars upfront" + source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" + included_paths: + - "/weather/*.md" + version: "main" + targets: + - "components/library/" + tags: + - demo + + - component: "test shallow globs and folder exclusion" + source: "github.com/cloudposse/atmos.git" + included_paths: + - "**/demo-localstack/*" + - "**/demo-library/**" + excluded_paths: + - "**/demo-library/**/stargazers/**" + - "**/demo-library/**/*.tf" + targets: + - "components/globs/" + tags: + - demo diff --git a/tests/fixtures/scenarios/vendor/vendor.yaml b/tests/fixtures/scenarios/vendor/vendor.yaml index 0d641f09d..e88a58f6e 100644 --- a/tests/fixtures/scenarios/vendor/vendor.yaml +++ b/tests/fixtures/scenarios/vendor/vendor.yaml @@ -52,36 +52,3 @@ spec: - "**/modules/**" excluded_paths: [] - - component: "test globs" - source: "github.com/cloudposse/atmos.git" - included_paths: - - "**/{demo-library,demo-stacks}/**/*.{tf,md}" - excluded_paths: - - "**/demo-library/**/*.{tfvars,tf}" - targets: - - "components/library/" - tags: - - demo - - - component: "test globs without double stars upfront" - source: "github.com/cloudposse/atmos.git//examples/demo-library?ref={{.Version}}" - included_paths: - - "/weather/*.md" - version: "main" - targets: - - "components/library/" - tags: - - demo - - - component: "test shallow globs and folder exclusion" - source: "github.com/cloudposse/atmos.git" - included_paths: - - "**/demo-localstack/*" - - "**/demo-library/**" - excluded_paths: - - "**/demo-library/**/stargazers/**" - - "**/demo-library/**/*.tf" - targets: - - "components/globs/" - tags: - - demo diff --git a/tests/test-cases/demo-globs.yaml b/tests/test-cases/demo-globs.yaml index 17092ee74..996ec870d 100644 --- a/tests/test-cases/demo-globs.yaml +++ b/tests/test-cases/demo-globs.yaml @@ -2,7 +2,7 @@ tests: - name: atmos_vendor_pull_with_globs enabled: true description: "Ensure atmos vendor pull command executes without errors and files are present." - workdir: "fixtures/scenarios/vendor" + workdir: "fixtures/scenarios/vendor-globs" command: "atmos" args: - "vendor" From 1882f98256b77727e5e87ea1ce9d07b00ba6023e Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Sat, 22 Feb 2025 19:26:05 +0500 Subject: [PATCH 32/68] more tests/golangci --- internal/exec/copy_glob.go | 48 ++++++++++++++++---------------- internal/exec/go_getter_utils.go | 6 ++-- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index b009bf824..c7d5636e2 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" - l "github.com/charmbracelet/log" + log "github.com/charmbracelet/log" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" cp "github.com/otiai10/copy" // Using the optimized copy library when no filtering is required. @@ -58,22 +58,22 @@ func shouldExcludePath(info os.FileInfo, relPath string, excluded []string) bool // Check plain relative path. matched, err := u.PathMatch(pattern, relPath) if err != nil { - l.Debug("Error matching exclusion pattern", logKeyPath, relPath, logKeyError, err) + log.Debug("Error matching exclusion pattern", logKeyPath, relPath, logKeyError, err) continue } if matched { - l.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, "pattern", pattern) + log.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, "pattern", pattern) return true } // If a directory, also check with a trailing slash. if info.IsDir() { matched, err = u.PathMatch(pattern, relPath+"/") if err != nil { - l.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, logKeyPath, relPath+"/", logKeyError, err) + log.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, logKeyPath, relPath+"/", logKeyError, err) continue } if matched { - l.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", "pattern", pattern) + log.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", "pattern", pattern) return true } } @@ -90,15 +90,15 @@ func shouldIncludePath(info os.FileInfo, relPath string, included []string) bool for _, pattern := range included { matched, err := u.PathMatch(pattern, relPath) if err != nil { - l.Debug("Error matching inclusion pattern", "pattern", pattern, logKeyPath, relPath, logKeyError, err) + log.Debug("Error matching inclusion pattern", "pattern", pattern, logKeyPath, relPath, logKeyError, err) continue } if matched { - l.Debug("Including path due to inclusion pattern", logKeyPath, relPath, "pattern", pattern) + log.Debug("Including path due to inclusion pattern", logKeyPath, relPath, "pattern", pattern) return true } } - l.Debug("Excluding path because it does not match any inclusion pattern", logKeyPath, relPath) + log.Debug("Excluding path because it does not match any inclusion pattern", logKeyPath, relPath) return false } @@ -109,7 +109,7 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { - l.Debug("Error computing relative path", "srcPath", srcPath, logKeyError, err) + log.Debug("Error computing relative path", "srcPath", srcPath, logKeyError, err) return true, nil // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) @@ -137,13 +137,13 @@ func processDirEntry(entry os.DirEntry, srcDir, dstDir, baseDir string, excluded return err } if skip { - l.Debug("Skipping entry", "srcPath", srcPath) + log.Debug("Skipping entry", "srcPath", srcPath) return nil } // Skip symlinks. if info.Mode()&os.ModeSymlink != 0 { - l.Debug("Skipping symlink", logKeyPath, srcPath) + log.Debug("Skipping symlink", logKeyPath, srcPath) return nil } @@ -175,21 +175,21 @@ func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []stri for _, pattern := range excluded { matched, err := u.PathMatch(pattern, fullRelPath) if err != nil { - l.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, logKeyPath, fullRelPath, logKeyError, err) + log.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, logKeyPath, fullRelPath, logKeyError, err) continue } if matched { - l.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, "pattern", pattern) + log.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, "pattern", pattern) return true } if info.IsDir() { matched, err = u.PathMatch(pattern, fullRelPath+"/") if err != nil { - l.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, logKeyPath, fullRelPath+"/", logKeyError, err) + log.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, logKeyPath, fullRelPath+"/", logKeyError, err) continue } if matched { - l.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", "pattern", pattern) + log.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", "pattern", pattern) return true } } @@ -209,7 +209,7 @@ func processPrefixEntry(entry os.DirEntry, srcDir, dstDir, globalBase, prefix st } if entry.Name() == ".git" { - l.Debug("Skipping .git directory", logKeyPath, fullRelPath) + log.Debug("Skipping .git directory", logKeyPath, fullRelPath) return nil } @@ -250,7 +250,7 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { if len(matches) == 0 { // If the pattern ends with "/*" (shallow copy indicator) do not fallback. if strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") { - l.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) + log.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) return []string{}, nil } // Fallback for non-shallow patterns. @@ -262,12 +262,12 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) } if len(matches) == 0 { - l.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) + log.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) return []string{}, nil } return matches, nil } - l.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) + log.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) return []string{}, nil } return matches, nil @@ -296,7 +296,7 @@ func processMatch(sourceDir, targetPath, file string, shallow bool, excluded []s dstPath := filepath.Join(targetPath, relPath) if info.IsDir() { if shallow { - l.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) + log.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) return nil } return copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, excluded) @@ -309,11 +309,11 @@ func processIncludedPattern(sourceDir, targetPath, pattern string, excluded []st shallow := isShallowPattern(pattern) matches, err := getMatchesForPattern(sourceDir, pattern) if err != nil { - l.Debug("Warning: error getting matches for pattern", "pattern", pattern, logKeyError, err) + log.Debug("Warning: error getting matches for pattern", "pattern", pattern, logKeyError, err) return nil } if len(matches) == 0 { - l.Debug("No files matched the inclusion pattern", "pattern", pattern) + log.Debug("No files matched the inclusion pattern", "pattern", pattern) return nil } for _, file := range matches { @@ -335,14 +335,14 @@ func copyToTargetWithPatterns( if sourceIsLocalFile && filepath.Ext(targetPath) == "" { targetPath = filepath.Join(targetPath, SanitizeFileName(uri)) } - l.Debug("Copying files", "source", sourceDir, "target", targetPath) + log.Debug("Copying files", "source", sourceDir, "target", targetPath) if err := os.MkdirAll(targetPath, os.ModePerm); err != nil { return fmt.Errorf("creating target directory %q: %w", targetPath, err) } // Optimization: if no inclusion and no exclusion patterns are defined, use the cp library. if len(s.IncludedPaths) == 0 && len(s.ExcludedPaths) == 0 { - l.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") + log.Debug("No inclusion or exclusion patterns defined; using cp library for fast copy") return cp.Copy(sourceDir, targetPath) } diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 24403deee..4e2cb1db2 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "github.com/charmbracelet/log" + log "github.com/charmbracelet/log" "github.com/google/uuid" "github.com/hashicorp/go-getter" @@ -320,12 +320,12 @@ func GoGetterGet( return nil } -// CustomGitGetter is a custom getter for git (git::) that removes symlinks +// CustomGitGetter is a custom getter for git (git::) that removes symlinks. type CustomGitGetter struct { getter.GitGetter } -// Implements the custom getter logic removing symlinks +// Implements the custom getter logic removing symlinks. func (c *CustomGitGetter) Get(dst string, url *url.URL) error { // Normal clone if err := c.GitGetter.Get(dst, url); err != nil { From b839cd41ad6da415300b74e48b80ebe9765414cb Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 16:40:04 +0500 Subject: [PATCH 33/68] globs refactoring --- internal/exec/copy_glob.go | 167 +++++++++++++++++++++++-------------- 1 file changed, 105 insertions(+), 62 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index c7d5636e2..1dede98a5 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -17,9 +17,28 @@ import ( const ( logKeyPath = "path" logKeyError = "error" + logKeyPattern = "pattern" shallowCopySuffix = "/*" ) +// PrefixCopyContext groups parameters for prefix-based copy operations. +type PrefixCopyContext struct { + SrcDir string + DstDir string + GlobalBase string + Prefix string + Excluded []string +} + +// CopyContext groups parameters for directory copy operations. +type CopyContext struct { + SrcDir string + DstDir string + BaseDir string + Excluded []string + Included []string +} + // copyFile copies a single file from src to dst while preserving file permissions. func copyFile(src, dst string) error { sourceFile, err := os.Open(src) @@ -62,18 +81,18 @@ func shouldExcludePath(info os.FileInfo, relPath string, excluded []string) bool continue } if matched { - log.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, "pattern", pattern) + log.Debug("Excluding path due to exclusion pattern (plain match)", logKeyPath, relPath, logKeyPattern, pattern) return true } // If a directory, also check with a trailing slash. if info.IsDir() { matched, err = u.PathMatch(pattern, relPath+"/") if err != nil { - log.Debug("Error matching exclusion pattern with trailing slash", "pattern", pattern, logKeyPath, relPath+"/", logKeyError, err) + log.Debug("Error matching exclusion pattern with trailing slash", logKeyPattern, pattern, logKeyPath, relPath+"/", logKeyError, err) continue } if matched { - log.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", "pattern", pattern) + log.Debug("Excluding directory due to exclusion pattern (with trailing slash)", logKeyPath, relPath+"/", logKeyPattern, pattern) return true } } @@ -90,11 +109,11 @@ func shouldIncludePath(info os.FileInfo, relPath string, included []string) bool for _, pattern := range included { matched, err := u.PathMatch(pattern, relPath) if err != nil { - log.Debug("Error matching inclusion pattern", "pattern", pattern, logKeyPath, relPath, logKeyError, err) + log.Debug("Error matching inclusion pattern", logKeyPattern, pattern, logKeyPath, relPath, logKeyError, err) continue } if matched { - log.Debug("Including path due to inclusion pattern", logKeyPath, relPath, "pattern", pattern) + log.Debug("Including path due to inclusion pattern", logKeyPath, relPath, logKeyPattern, pattern) return true } } @@ -103,40 +122,36 @@ func shouldIncludePath(info os.FileInfo, relPath string, included []string) bool } // shouldSkipEntry determines whether to skip a file/directory based on its relative path to baseDir. -func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) (bool, error) { +func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, included []string) bool { if info.Name() == ".git" { - return true, nil + return true } relPath, err := filepath.Rel(baseDir, srcPath) if err != nil { log.Debug("Error computing relative path", "srcPath", srcPath, logKeyError, err) - return true, nil // treat error as a signal to skip + return true // treat error as a signal to skip } relPath = filepath.ToSlash(relPath) if shouldExcludePath(info, relPath, excluded) { - return true, nil + return true } if !shouldIncludePath(info, relPath, included) { - return true, nil + return true } - return false, nil + return false } // processDirEntry handles a single directory entry for copyDirRecursive. -func processDirEntry(entry os.DirEntry, srcDir, dstDir, baseDir string, excluded, included []string) error { - srcPath := filepath.Join(srcDir, entry.Name()) - dstPath := filepath.Join(dstDir, entry.Name()) +func processDirEntry(entry os.DirEntry, ctx CopyContext) error { + srcPath := filepath.Join(ctx.SrcDir, entry.Name()) + dstPath := filepath.Join(ctx.DstDir, entry.Name()) info, err := entry.Info() if err != nil { return fmt.Errorf("getting info for %q: %w", srcPath, err) } - skip, err := shouldSkipEntry(info, srcPath, baseDir, excluded, included) - if err != nil { - return err - } - if skip { + if shouldSkipEntry(info, srcPath, ctx.BaseDir, ctx.Excluded, ctx.Included) { log.Debug("Skipping entry", "srcPath", srcPath) return nil } @@ -151,19 +166,27 @@ func processDirEntry(entry os.DirEntry, srcDir, dstDir, baseDir string, excluded if err := os.MkdirAll(dstPath, info.Mode()); err != nil { return fmt.Errorf("creating directory %q: %w", dstPath, err) } - return copyDirRecursive(srcPath, dstPath, baseDir, excluded, included) + // Recurse with the same context but with updated source and destination directories. + newCtx := CopyContext{ + SrcDir: srcPath, + DstDir: dstPath, + BaseDir: ctx.BaseDir, + Excluded: ctx.Excluded, + Included: ctx.Included, + } + return copyDirRecursive(newCtx) } return copyFile(srcPath, dstPath) } // copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. -func copyDirRecursive(srcDir, dstDir, baseDir string, excluded, included []string) error { - entries, err := os.ReadDir(srcDir) +func copyDirRecursive(ctx CopyContext) error { + entries, err := os.ReadDir(ctx.SrcDir) if err != nil { - return fmt.Errorf("reading directory %q: %w", srcDir, err) + return fmt.Errorf("reading directory %q: %w", ctx.SrcDir, err) } for _, entry := range entries { - if err := processDirEntry(entry, srcDir, dstDir, baseDir, excluded, included); err != nil { + if err := processDirEntry(entry, ctx); err != nil { return err } } @@ -175,21 +198,21 @@ func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []stri for _, pattern := range excluded { matched, err := u.PathMatch(pattern, fullRelPath) if err != nil { - log.Debug("Error matching exclusion pattern in prefix function", "pattern", pattern, logKeyPath, fullRelPath, logKeyError, err) + log.Debug("Error matching exclusion pattern in prefix function", logKeyPattern, pattern, logKeyPath, fullRelPath, logKeyError, err) continue } if matched { - log.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, "pattern", pattern) + log.Debug("Excluding (prefix) due to exclusion pattern (plain match)", logKeyPath, fullRelPath, logKeyPattern, pattern) return true } if info.IsDir() { matched, err = u.PathMatch(pattern, fullRelPath+"/") if err != nil { - log.Debug("Error matching exclusion pattern with trailing slash in prefix function", "pattern", pattern, logKeyPath, fullRelPath+"/", logKeyError, err) + log.Debug("Error matching exclusion pattern with trailing slash in prefix function", logKeyPattern, pattern, logKeyPath, fullRelPath+"/", logKeyError, err) continue } if matched { - log.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", "pattern", pattern) + log.Debug("Excluding (prefix) due to exclusion pattern (with trailing slash)", logKeyPath, fullRelPath+"/", logKeyPattern, pattern) return true } } @@ -198,10 +221,10 @@ func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []stri } // processPrefixEntry handles a single entry for copyDirRecursiveWithPrefix. -func processPrefixEntry(entry os.DirEntry, srcDir, dstDir, globalBase, prefix string, excluded []string) error { - fullRelPath := filepath.ToSlash(filepath.Join(prefix, entry.Name())) - srcPath := filepath.Join(srcDir, entry.Name()) - dstPath := filepath.Join(dstDir, entry.Name()) +func processPrefixEntry(entry os.DirEntry, ctx PrefixCopyContext) error { + fullRelPath := filepath.ToSlash(filepath.Join(ctx.Prefix, entry.Name())) + srcPath := filepath.Join(ctx.SrcDir, entry.Name()) + dstPath := filepath.Join(ctx.DstDir, entry.Name()) info, err := entry.Info() if err != nil { @@ -213,7 +236,7 @@ func processPrefixEntry(entry os.DirEntry, srcDir, dstDir, globalBase, prefix st return nil } - if shouldSkipPrefixEntry(info, fullRelPath, excluded) { + if shouldSkipPrefixEntry(info, fullRelPath, ctx.Excluded) { return nil } @@ -221,19 +244,26 @@ func processPrefixEntry(entry os.DirEntry, srcDir, dstDir, globalBase, prefix st if err := os.MkdirAll(dstPath, info.Mode()); err != nil { return fmt.Errorf("creating directory %q: %w", dstPath, err) } - return copyDirRecursiveWithPrefix(srcPath, dstPath, globalBase, fullRelPath, excluded) + newCtx := PrefixCopyContext{ + SrcDir: srcPath, + DstDir: dstPath, + GlobalBase: ctx.GlobalBase, + Prefix: fullRelPath, + Excluded: ctx.Excluded, + } + return copyDirRecursiveWithPrefix(newCtx) } return copyFile(srcPath, dstPath) } // copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. -func copyDirRecursiveWithPrefix(srcDir, dstDir, globalBase, prefix string, excluded []string) error { - entries, err := os.ReadDir(srcDir) +func copyDirRecursiveWithPrefix(ctx PrefixCopyContext) error { + entries, err := os.ReadDir(ctx.SrcDir) if err != nil { - return fmt.Errorf("reading directory %q: %w", srcDir, err) + return fmt.Errorf("reading directory %q: %w", ctx.SrcDir, err) } for _, entry := range entries { - if err := processPrefixEntry(entry, srcDir, dstDir, globalBase, prefix, excluded); err != nil { + if err := processPrefixEntry(entry, ctx); err != nil { return err } } @@ -247,30 +277,31 @@ func getMatchesForPattern(sourceDir, pattern string) ([]string, error) { if err != nil { return nil, fmt.Errorf("error getting glob matches for %q: %w", fullPattern, err) } - if len(matches) == 0 { - // If the pattern ends with "/*" (shallow copy indicator) do not fallback. - if strings.HasSuffix(pattern, shallowCopySuffix) && !strings.HasSuffix(pattern, "/**") { - log.Debug("No matches found for shallow pattern; target directory will be empty", "pattern", fullPattern) + if len(matches) != 0 { + return matches, nil + } + + // Handle shallow copy indicator. + if strings.HasSuffix(pattern, shallowCopySuffix) { + if !strings.HasSuffix(pattern, "/**") { + log.Debug("No matches found for shallow pattern; target directory will be empty", logKeyPattern, fullPattern) return []string{}, nil } - // Fallback for non-shallow patterns. - if strings.HasSuffix(pattern, shallowCopySuffix) { - recursivePattern := strings.TrimSuffix(pattern, shallowCopySuffix) + "/**" - fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) - matches, err = u.GetGlobMatches(fullRecursivePattern) - if err != nil { - return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) - } - if len(matches) == 0 { - log.Debug("No matches found for recursive pattern; target directory will be empty", "pattern", fullRecursivePattern) - return []string{}, nil - } - return matches, nil + recursivePattern := strings.TrimSuffix(pattern, shallowCopySuffix) + "/**" + fullRecursivePattern := filepath.Join(sourceDir, recursivePattern) + matches, err = u.GetGlobMatches(fullRecursivePattern) + if err != nil { + return nil, fmt.Errorf("error getting glob matches for recursive pattern %q: %w", fullRecursivePattern, err) } - log.Debug("No matches found for pattern; target directory will be empty", "pattern", fullPattern) - return []string{}, nil + if len(matches) == 0 { + log.Debug("No matches found for recursive pattern; target directory will be empty", logKeyPattern, fullRecursivePattern) + return []string{}, nil + } + return matches, nil } - return matches, nil + + log.Debug("No matches found for pattern; target directory will be empty", logKeyPattern, fullPattern) + return []string{}, nil } // isShallowPattern determines if a pattern indicates a shallow copy. @@ -299,7 +330,13 @@ func processMatch(sourceDir, targetPath, file string, shallow bool, excluded []s log.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) return nil } - return copyDirRecursiveWithPrefix(file, dstPath, sourceDir, relPath, excluded) + return copyDirRecursiveWithPrefix(PrefixCopyContext{ + SrcDir: file, + DstDir: dstPath, + GlobalBase: sourceDir, + Prefix: relPath, + Excluded: excluded, + }) } return copyFile(file, dstPath) } @@ -309,11 +346,11 @@ func processIncludedPattern(sourceDir, targetPath, pattern string, excluded []st shallow := isShallowPattern(pattern) matches, err := getMatchesForPattern(sourceDir, pattern) if err != nil { - log.Debug("Warning: error getting matches for pattern", "pattern", pattern, logKeyError, err) + log.Debug("Warning: error getting matches for pattern", logKeyPattern, pattern, logKeyError, err) return nil } if len(matches) == 0 { - log.Debug("No files matched the inclusion pattern", "pattern", pattern) + log.Debug("No files matched the inclusion pattern", logKeyPattern, pattern) return nil } for _, file := range matches { @@ -355,7 +392,13 @@ func copyToTargetWithPatterns( // If no inclusion patterns are defined, copy everything except those matching excluded items. if len(s.IncludedPaths) == 0 { - if err := copyDirRecursive(sourceDir, targetPath, sourceDir, s.ExcludedPaths, s.IncludedPaths); err != nil { + if err := copyDirRecursive(CopyContext{ + SrcDir: sourceDir, + DstDir: targetPath, + BaseDir: sourceDir, + Excluded: s.ExcludedPaths, + Included: s.IncludedPaths, + }); err != nil { return fmt.Errorf("error copying from %q to %q: %w", sourceDir, targetPath, err) } } From deb228cd1d53e5e176550e136da804b36815c30b Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 16:58:00 +0500 Subject: [PATCH 34/68] refactoring globs --- internal/exec/copy_glob.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/exec/copy_glob.go b/internal/exec/copy_glob.go index 1dede98a5..4850d6fd3 100644 --- a/internal/exec/copy_glob.go +++ b/internal/exec/copy_glob.go @@ -142,7 +142,7 @@ func shouldSkipEntry(info os.FileInfo, srcPath, baseDir string, excluded, includ } // processDirEntry handles a single directory entry for copyDirRecursive. -func processDirEntry(entry os.DirEntry, ctx CopyContext) error { +func processDirEntry(entry os.DirEntry, ctx *CopyContext) error { srcPath := filepath.Join(ctx.SrcDir, entry.Name()) dstPath := filepath.Join(ctx.DstDir, entry.Name()) @@ -167,7 +167,7 @@ func processDirEntry(entry os.DirEntry, ctx CopyContext) error { return fmt.Errorf("creating directory %q: %w", dstPath, err) } // Recurse with the same context but with updated source and destination directories. - newCtx := CopyContext{ + newCtx := &CopyContext{ SrcDir: srcPath, DstDir: dstPath, BaseDir: ctx.BaseDir, @@ -180,7 +180,7 @@ func processDirEntry(entry os.DirEntry, ctx CopyContext) error { } // copyDirRecursive recursively copies srcDir to dstDir using shouldSkipEntry filtering. -func copyDirRecursive(ctx CopyContext) error { +func copyDirRecursive(ctx *CopyContext) error { entries, err := os.ReadDir(ctx.SrcDir) if err != nil { return fmt.Errorf("reading directory %q: %w", ctx.SrcDir, err) @@ -221,7 +221,7 @@ func shouldSkipPrefixEntry(info os.FileInfo, fullRelPath string, excluded []stri } // processPrefixEntry handles a single entry for copyDirRecursiveWithPrefix. -func processPrefixEntry(entry os.DirEntry, ctx PrefixCopyContext) error { +func processPrefixEntry(entry os.DirEntry, ctx *PrefixCopyContext) error { fullRelPath := filepath.ToSlash(filepath.Join(ctx.Prefix, entry.Name())) srcPath := filepath.Join(ctx.SrcDir, entry.Name()) dstPath := filepath.Join(ctx.DstDir, entry.Name()) @@ -244,7 +244,7 @@ func processPrefixEntry(entry os.DirEntry, ctx PrefixCopyContext) error { if err := os.MkdirAll(dstPath, info.Mode()); err != nil { return fmt.Errorf("creating directory %q: %w", dstPath, err) } - newCtx := PrefixCopyContext{ + newCtx := &PrefixCopyContext{ SrcDir: srcPath, DstDir: dstPath, GlobalBase: ctx.GlobalBase, @@ -257,7 +257,7 @@ func processPrefixEntry(entry os.DirEntry, ctx PrefixCopyContext) error { } // copyDirRecursiveWithPrefix recursively copies srcDir to dstDir while preserving the global relative path. -func copyDirRecursiveWithPrefix(ctx PrefixCopyContext) error { +func copyDirRecursiveWithPrefix(ctx *PrefixCopyContext) error { entries, err := os.ReadDir(ctx.SrcDir) if err != nil { return fmt.Errorf("reading directory %q: %w", ctx.SrcDir, err) @@ -330,7 +330,7 @@ func processMatch(sourceDir, targetPath, file string, shallow bool, excluded []s log.Debug("Directory is not copied because it is a shallow copy", "directory", relPath) return nil } - return copyDirRecursiveWithPrefix(PrefixCopyContext{ + return copyDirRecursiveWithPrefix(&PrefixCopyContext{ SrcDir: file, DstDir: dstPath, GlobalBase: sourceDir, @@ -392,7 +392,7 @@ func copyToTargetWithPatterns( // If no inclusion patterns are defined, copy everything except those matching excluded items. if len(s.IncludedPaths) == 0 { - if err := copyDirRecursive(CopyContext{ + if err := copyDirRecursive(&CopyContext{ SrcDir: sourceDir, DstDir: targetPath, BaseDir: sourceDir, From 9a75bd4e8555b255d38640252d1384ca3047870a Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 17:51:47 +0500 Subject: [PATCH 35/68] atmosConfig is now a pointer in GoGetter --- internal/exec/go_getter_utils.go | 57 +++++++++++-------------- internal/exec/validate_stacks.go | 2 +- internal/exec/vendor_component_utils.go | 2 +- internal/exec/vendor_model.go | 4 +- internal/exec/vendor_model_component.go | 36 ++++++++++------ internal/exec/yaml_func_include.go | 2 +- 6 files changed, 54 insertions(+), 49 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 4e2cb1db2..3c78ec6e5 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -63,11 +63,11 @@ func IsValidScheme(scheme string) bool { return validSchemes[scheme] } -// CustomGitHubDetector intercepts GitHub URLs and transforms them +// CustomGitDetector intercepts GitHub URLs and transforms them // into something like git::https://@github.com/... so we can // do a git-based clone with a token. type CustomGitDetector struct { - AtmosConfig schema.AtmosConfiguration + AtmosConfig *schema.AtmosConfiguration source string } @@ -269,7 +269,7 @@ func (d *CustomGitDetector) adjustSubdir(parsedURL *url.URL, source string) { // RegisterCustomDetectors prepends the custom detector so it runs before // the built-in ones. Any code that calls go-getter should invoke this. -func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source string) { +func RegisterCustomDetectors(atmosConfig *schema.AtmosConfiguration, source string) { getter.Detectors = append( []getter.Detector{ &CustomGitDetector{AtmosConfig: atmosConfig, source: source}, @@ -280,7 +280,7 @@ func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration, source strin // GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination func GoGetterGet( - atmosConfig schema.AtmosConfiguration, + atmosConfig *schema.AtmosConfiguration, src string, dest string, clientMode getter.ClientMode, @@ -289,7 +289,7 @@ func GoGetterGet( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - // Register custom detectors, passing the original `src` to the CustomGitHubDetector. + // Register custom detectors, passing the original `src` to the CustomGitDetector. // go-getter typically strips subdirectories before calling the detector, so the // unaltered source is needed to identify whether a top-level repository or a // subdirectory was specified (e.g., for appending "//." only when no subdir is present). @@ -310,7 +310,6 @@ func GoGetterGet( "https": &getter.HttpGetter{}, // "s3": &getter.S3Getter{}, // add as needed // "gcs": &getter.GCSGetter{}, - }, } if err := client.Get(); err != nil { @@ -325,7 +324,7 @@ type CustomGitGetter struct { getter.GitGetter } -// Implements the custom getter logic removing symlinks. +// Get implements the custom getter logic removing symlinks. func (c *CustomGitGetter) Get(dst string, url *url.URL) error { // Normal clone if err := c.GitGetter.Get(dst, url); err != nil { @@ -344,9 +343,6 @@ func removeSymlinks(root string) error { } if info.Mode()&os.ModeSymlink != 0 { // Symlinks are removed for the entire repo, regardless if there are any subfolders specified - // Thus logging is disabled - // u.LogWarning(fmt.Sprintf("Removing symlink: %s", path)) - // It's a symlink, remove it return os.Remove(path) } return nil @@ -354,11 +350,11 @@ func removeSymlinks(root string) error { } // DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type -func DownloadDetectFormatAndParseFile(atmosConfig schema.AtmosConfiguration, file string) (any, error) { +func DownloadDetectFormatAndParseFile(atmosConfig *schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() f := filepath.Join(tempDir, uuid.New().String()) - if err := GoGetterGet(atmosConfig, file, f, getter.ClientModeFile, time.Second*30); err != nil { + if err := GoGetterGet(atmosConfig, file, f, getter.ClientModeFile, 30*time.Second); err != nil { return nil, fmt.Errorf("failed to download the file '%s': %w", file, err) } @@ -382,41 +378,40 @@ scp, sftp Shortcuts like github.com, bitbucket.org - File-related Schemes: -file - Local filesystem paths -dir - Local directories -tar - Tar files, potentially compressed (tar.gz, tar.bz2, etc.) -zip - Zip files + file - Local filesystem paths + dir - Local directories + tar - Tar files, potentially compressed (tar.gz, tar.bz2, etc.) + zip - Zip files - HTTP/HTTPS: -http - HTTP URLs -https - HTTPS URLs + http - HTTP URLs + https - HTTPS URLs - Git: -git - Git repositories, which can be accessed via HTTPS or SSH + git - Git repositories, which can be accessed via HTTPS or SSH - Mercurial: -hg - Mercurial repositories, accessed via HTTP/S or SSH + hg - Mercurial repositories, accessed via HTTP/S or SSH - Amazon S3: -s3 - Amazon S3 bucket URLs + s3 - Amazon S3 bucket URLs - Google Cloud Storage: -gcs - Google Cloud Storage URLs + gcs - Google Cloud Storage URLs - OCI: -oci - Open Container Initiative (OCI) images + oci - Open Container Initiative (OCI) images - Other Protocols: -scp - Secure Copy Protocol for SSH-based transfers -sftp - SSH File Transfer Protocol + scp - Secure Copy Protocol for SSH-based transfers + sftp - SSH File Transfer Protocol - GitHub/Bitbucket/Other Shortcuts: -github.com - Direct GitHub repository shortcuts -bitbucket.org - Direct Bitbucket repository shortcuts + github.com - Direct GitHub repository shortcuts + bitbucket.org - Direct Bitbucket repository shortcuts - Composite Schemes: -go-getter allows for composite schemes, where multiple operations can be combined. For example: -git::https://github.com/user/repo - Forces the use of git over an HTTPS URL. -tar::http://example.com/archive.tar.gz - Treats the HTTP resource as a tarball. - + go-getter allows for composite schemes, where multiple operations can be combined. For example: + git::https://github.com/user/repo - Forces the use of git over an HTTPS URL. + tar::http://example.com/archive.tar.gz - Treats the HTTP resource as a tarball. */ diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go index e1f40e6f6..28fcfbe30 100644 --- a/internal/exec/validate_stacks.go +++ b/internal/exec/validate_stacks.go @@ -401,7 +401,7 @@ func downloadSchemaFromURL(atmosConfig schema.AtmosConfiguration) (string, error atmosManifestJsonSchemaFilePath := filepath.Join(tempDir, fileName) - if err = GoGetterGet(atmosConfig, manifestURL, atmosManifestJsonSchemaFilePath, getter.ClientModeFile, time.Second*30); err != nil { + if err = GoGetterGet(&atmosConfig, manifestURL, atmosManifestJsonSchemaFilePath, getter.ClientModeFile, time.Second*30); err != nil { return "", fmt.Errorf("failed to download the Atmos JSON Schema file '%s' from the URL '%s': %w", fileName, manifestURL, err) } diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 8c3a4714f..94d532e5f 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -357,7 +357,7 @@ func ExecuteComponentVendorInternal( // Run TUI to process packages if len(packages) > 0 { - model, err := newModelComponentVendorInternal(packages, dryRun, atmosConfig) + model, err := newModelComponentVendorInternal(packages, dryRun, &atmosConfig) if err != nil { return fmt.Errorf("error initializing model: %v", err) } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index fdfb68ffc..e1545e379 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -269,7 +269,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos switch p.pkgType { case pkgTypeRemote: // Use go-getter to download remote packages - if err := GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, 10*time.Minute); err != nil { + if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, 10*time.Minute); err != nil { return installedPkgMsg{ err: fmt.Errorf("failed to download package: %w", err), name: p.name, @@ -327,7 +327,7 @@ func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig schema.AtmosCo } if installer.componentPackage != nil { - return downloadComponentAndInstall(installer.componentPackage, dryRun, atmosConfig) + return downloadComponentAndInstall(installer.componentPackage, dryRun, &atmosConfig) } // No valid package provided diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index f64aee4a7..d5c2bac05 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -18,6 +18,7 @@ import ( "github.com/cloudposse/atmos/pkg/ui/theme" ) +// pkgComponentVendor defines a vendor package. type pkgComponentVendor struct { uri string name string @@ -31,7 +32,8 @@ type pkgComponentVendor struct { mixinFilename string } -func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) (modelVendor, error) { +// newModelComponentVendorInternal creates a new vendor model. +func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) (modelVendor, error) { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(30), @@ -40,7 +42,14 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm s := spinner.New() s.Style = theme.Styles.Link if len(pkgs) == 0 { - return modelVendor{done: true}, nil + return modelVendor{ + packages: nil, + spinner: s, + progress: p, + dryRun: dryRun, + atmosConfig: *atmosConfig, + isTTY: term.IsTTYSupportForStdout(), + }, nil } vendorPks := []pkgVendor{} for _, pkg := range pkgs { @@ -50,7 +59,6 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm componentPackage: &pkg, } vendorPks = append(vendorPks, vendorPkg) - } tty := term.IsTTYSupportForStdout() return modelVendor{ @@ -58,12 +66,13 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm spinner: s, progress: p, dryRun: dryRun, - atmosConfig: atmosConfig, + atmosConfig: *atmosConfig, isTTY: tty, }, nil } -func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) tea.Cmd { +// downloadComponentAndInstall returns a command to download and install a component. +func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { // Simulate the action @@ -106,7 +115,8 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig } } -func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) error { +// installComponent downloads and installs a component. +func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { // Create temp folder // We are using a temp folder for the following reasons: // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) @@ -121,7 +131,7 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati return fmt.Errorf("failed to set temp directory permissions: %w", err) } - defer removeTempDir(atmosConfig, tempDir) + defer removeTempDir(*atmosConfig, tempDir) switch p.pkgType { case pkgTypeRemote: @@ -133,7 +143,7 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati case pkgTypeOci: // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(atmosConfig, p.uri, tempDir) + err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { return fmt.Errorf("Failed to process OCI image %s error %s", p.name, err) } @@ -159,22 +169,22 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati } default: return fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name) - } - if err = copyComponentToDestination(atmosConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { + if err = copyComponentToDestination(*atmosConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { return fmt.Errorf("failed to copy package %s error %s", p.name, err) } return nil } -func installMixin(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) error { +// installMixin downloads and installs a mixin. +func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { return fmt.Errorf("Failed to create temp directory %s", err) } - defer removeTempDir(atmosConfig, tempDir) + defer removeTempDir(*atmosConfig, tempDir) switch p.pkgType { case pkgTypeRemote: @@ -184,7 +194,7 @@ func installMixin(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration) case pkgTypeOci: // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(atmosConfig, p.uri, tempDir) + err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { return fmt.Errorf("failed to process OCI image %s error %s", p.name, err) } diff --git a/internal/exec/yaml_func_include.go b/internal/exec/yaml_func_include.go index 4d75f3db0..5cc18a46c 100644 --- a/internal/exec/yaml_func_include.go +++ b/internal/exec/yaml_func_include.go @@ -47,7 +47,7 @@ func processTagInclude( if fileType == u.AtmosYamlFuncIncludeLocalFile { res, err = u.DetectFormatAndParseFile(f) } else if fileType == u.AtmosYamlFuncIncludeGoGetter { - res, err = DownloadDetectFormatAndParseFile(atmosConfig, f) + res, err = DownloadDetectFormatAndParseFile(&atmosConfig, f) } if err != nil { From 42ff520b44aea81c5e1833dff975cd9d4b3bf145 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 18:32:51 +0500 Subject: [PATCH 36/68] cyclomatic counter temp bump to 13 --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 719177c63..1034b9987 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -100,7 +100,7 @@ linters-settings: - name: cognitive-complexity arguments: [25] # Max cognitive complexity - name: cyclomatic - arguments: [10] # Max cyclomatic complexity + arguments: [13] # Max cyclomatic complexity - name: add-constant arguments: - maxLitCount: "3" From f8e949e2fb5fb33faaf43bd9a1dca8e9e33bed72 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 18:57:23 +0500 Subject: [PATCH 37/68] remaining linter feedback is fixed --- internal/exec/go_getter_utils.go | 8 +++++--- internal/exec/vendor_component_utils.go | 5 +---- internal/exec/vendor_model_component.go | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 3c78ec6e5..de051cc43 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -63,9 +63,11 @@ func IsValidScheme(scheme string) bool { return validSchemes[scheme] } -// CustomGitDetector intercepts GitHub URLs and transforms them -// into something like git::https://@github.com/... so we can -// do a git-based clone with a token. +/* +CustomGitDetector intercepts GitHub URLs and transforms them +into something like git::https://@github.com/... so we can +do a git-based clone with a token. +*/ type CustomGitDetector struct { AtmosConfig *schema.AtmosConfiguration source string diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index 94d532e5f..692fb525c 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -357,10 +357,7 @@ func ExecuteComponentVendorInternal( // Run TUI to process packages if len(packages) > 0 { - model, err := newModelComponentVendorInternal(packages, dryRun, &atmosConfig) - if err != nil { - return fmt.Errorf("error initializing model: %v", err) - } + model := newModelComponentVendorInternal(packages, dryRun, &atmosConfig) var opts []tea.ProgramOption // Disable TUI if no TTY support is available if !term.IsTTYSupportForStdout() { diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index d5c2bac05..c7bf0e1df 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -33,7 +33,7 @@ type pkgComponentVendor struct { } // newModelComponentVendorInternal creates a new vendor model. -func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) (modelVendor, error) { +func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) modelVendor { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(30), @@ -49,7 +49,7 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm dryRun: dryRun, atmosConfig: *atmosConfig, isTTY: term.IsTTYSupportForStdout(), - }, nil + } } vendorPks := []pkgVendor{} for _, pkg := range pkgs { @@ -68,7 +68,7 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm dryRun: dryRun, atmosConfig: *atmosConfig, isTTY: tty, - }, nil + } } // downloadComponentAndInstall returns a command to download and install a component. From eb8b7fac0260f7e7b99f2fa585f6ff00b0d99f35 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 20:00:52 +0500 Subject: [PATCH 38/68] linter fix --- internal/exec/go_getter_utils.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index de051cc43..4aba03fc2 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -63,11 +63,7 @@ func IsValidScheme(scheme string) bool { return validSchemes[scheme] } -/* -CustomGitDetector intercepts GitHub URLs and transforms them -into something like git::https://@github.com/... so we can -do a git-based clone with a token. -*/ +// CustomGitDetector intercepts GitHub URLs and transforms them into something like git::https://@github.com/ so we can do a git-based clone with a token. type CustomGitDetector struct { AtmosConfig *schema.AtmosConfiguration source string From 086c79b1a86dffb9ecb3603f155c52e4c9f781ab Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Wed, 26 Feb 2025 22:05:57 +0500 Subject: [PATCH 39/68] symmlynk debug --- internal/exec/go_getter_utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 4aba03fc2..13efa65e3 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -340,6 +340,7 @@ func removeSymlinks(root string) error { return err } if info.Mode()&os.ModeSymlink != 0 { + log.Debug("Removing symlink", "path", path) // Symlinks are removed for the entire repo, regardless if there are any subfolders specified return os.Remove(path) } From ec93039e6fb785dccd8bb51d5ba8bb0062258013 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:41:30 +0600 Subject: [PATCH 40/68] Update .gitignore Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 17c882487..9c4d9f1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,4 @@ # Ignore components vendored during tests tests/fixtures/scenarios/vendor/components/** -tests/fixtures/scenarios/vendor-globs/components/** examples/demo-vendoring/components/** From 4b88dfc5252b4a8cdea6a44fbef51a2e147d54d3 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 3 Mar 2025 21:15:38 +0600 Subject: [PATCH 41/68] unit tests for copy_glob.go --- internal/exec/copy_glob_test.go | 187 ++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 internal/exec/copy_glob_test.go diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go new file mode 100644 index 000000000..7df691e3a --- /dev/null +++ b/internal/exec/copy_glob_test.go @@ -0,0 +1,187 @@ +// copy_glob_test.go +package exec + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestCopyFile verifies that copyFile correctly copies file contents and preserves permissions. +func TestCopyFile(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copyfile-src") + if err != nil { + t.Fatalf("Failed to create source dir: %v", err) + } + defer os.RemoveAll(srcDir) + + dstDir, err := os.MkdirTemp("", "copyfile-dst") + if err != nil { + t.Fatalf("Failed to create destination dir: %v", err) + } + defer os.RemoveAll(dstDir) + + srcFile := filepath.Join(srcDir, "test.txt") + content := "copyFileTest" + if err := os.WriteFile(srcFile, []byte(content), 0644); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + + dstFile := filepath.Join(dstDir, "test.txt") + if err := copyFile(srcFile, dstFile); err != nil { + t.Fatalf("copyFile failed: %v", err) + } + + copiedContent, err := os.ReadFile(dstFile) + if err != nil { + t.Fatalf("Failed to read destination file: %v", err) + } + + if string(copiedContent) != content { + t.Errorf("Expected content %q, got %q", content, string(copiedContent)) + } +} + +// TestShouldExcludePath tests that shouldExcludePath returns true when a file matches an exclusion pattern. +func TestShouldExcludePath(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.log") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat temp file: %v", err) + } + + excluded := []string{"**/*.log"} + relPath := "app/test.log" + if !shouldExcludePath(info, relPath, excluded) { + t.Errorf("Expected path %q to be excluded", relPath) + } +} + +// TestShouldIncludePath verifies that shouldIncludePath returns true when the file matches an inclusion pattern. +func TestShouldIncludePath(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.txt") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat temp file: %v", err) + } + + included := []string{"**/*.txt"} + relPath := "docs/readme.txt" + if !shouldIncludePath(info, relPath, included) { + t.Errorf("Expected path %q to be included", relPath) + } +} + +// TestShouldSkipEntry creates a temporary directory structure and verifies that shouldSkipEntry correctly skips a file. +func TestShouldSkipEntry(t *testing.T) { + baseDir, err := os.MkdirTemp("", "base") + if err != nil { + t.Fatalf("Failed to create base dir: %v", err) + } + defer os.RemoveAll(baseDir) + + subDir := filepath.Join(baseDir, "sub") + if err := os.Mkdir(subDir, 0755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + filePath := filepath.Join(subDir, "sample.txt") + if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + info, err := os.Stat(filePath) + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + + excluded := []string{"**/*.txt"} + included := []string{} + if !shouldSkipEntry(info, filePath, baseDir, excluded, included) { + t.Errorf("Expected file %q to be skipped", filePath) + } +} + +// TestCopyDirRecursive sets up a temporary directory tree and verifies that copyDirRecursive copies all files. +func TestCopyDirRecursive(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copydir-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + + dstDir, err := os.MkdirTemp("", "copydir-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + + subDir := filepath.Join(srcDir, "sub") + if err := os.Mkdir(subDir, 0755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + file1 := filepath.Join(srcDir, "file1.txt") + file2 := filepath.Join(subDir, "file2.txt") + if err := os.WriteFile(file1, []byte("file1"), 0644); err != nil { + t.Fatalf("Failed to write file1: %v", err) + } + if err := os.WriteFile(file2, []byte("file2"), 0644); err != nil { + t.Fatalf("Failed to write file2: %v", err) + } + + ctx := &CopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + BaseDir: srcDir, + Excluded: []string{}, + Included: []string{}, + } + if err := copyDirRecursive(ctx); err != nil { + t.Fatalf("copyDirRecursive failed: %v", err) + } + + dstFile1 := filepath.Join(dstDir, "file1.txt") + dstFile2 := filepath.Join(dstDir, "sub", "file2.txt") + if _, err := os.Stat(dstFile1); os.IsNotExist(err) { + t.Errorf("Expected %q to exist", dstFile1) + } + if _, err := os.Stat(dstFile2); os.IsNotExist(err) { + t.Errorf("Expected %q to exist", dstFile2) + } +} + +// TestGetMatchesForPattern verifies that getMatchesForPattern returns the correct file matches. +func TestGetMatchesForPattern(t *testing.T) { + srcDir, err := os.MkdirTemp("", "glob-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + + fileA := filepath.Join(srcDir, "a.txt") + fileB := filepath.Join(srcDir, "b.log") + if err := os.WriteFile(fileA, []byte("content"), 0644); err != nil { + t.Fatalf("Failed to write fileA: %v", err) + } + if err := os.WriteFile(fileB, []byte("content"), 0644); err != nil { + t.Fatalf("Failed to write fileB: %v", err) + } + + pattern := "*.txt" + matches, err := getMatchesForPattern(srcDir, pattern) + if err != nil { + t.Fatalf("getMatchesForPattern error: %v", err) + } + if len(matches) != 1 || !strings.Contains(matches[0], "a.txt") { + t.Errorf("Expected one match for a.txt, got %v", matches) + } +} From 2be732e4a23e9e72ab5cef8fa596437435e09ad3 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 15:16:42 +0000 Subject: [PATCH 42/68] [autofix.ci] apply automated fixes --- internal/exec/copy_glob_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 7df691e3a..f338f6198 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -24,7 +24,7 @@ func TestCopyFile(t *testing.T) { srcFile := filepath.Join(srcDir, "test.txt") content := "copyFileTest" - if err := os.WriteFile(srcFile, []byte(content), 0644); err != nil { + if err := os.WriteFile(srcFile, []byte(content), 0o644); err != nil { t.Fatalf("Failed to write source file: %v", err) } @@ -92,11 +92,11 @@ func TestShouldSkipEntry(t *testing.T) { defer os.RemoveAll(baseDir) subDir := filepath.Join(baseDir, "sub") - if err := os.Mkdir(subDir, 0755); err != nil { + if err := os.Mkdir(subDir, 0o755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } filePath := filepath.Join(subDir, "sample.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { + if err := os.WriteFile(filePath, []byte("test"), 0o644); err != nil { t.Fatalf("Failed to write file: %v", err) } info, err := os.Stat(filePath) @@ -126,15 +126,15 @@ func TestCopyDirRecursive(t *testing.T) { defer os.RemoveAll(dstDir) subDir := filepath.Join(srcDir, "sub") - if err := os.Mkdir(subDir, 0755); err != nil { + if err := os.Mkdir(subDir, 0o755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } file1 := filepath.Join(srcDir, "file1.txt") file2 := filepath.Join(subDir, "file2.txt") - if err := os.WriteFile(file1, []byte("file1"), 0644); err != nil { + if err := os.WriteFile(file1, []byte("file1"), 0o644); err != nil { t.Fatalf("Failed to write file1: %v", err) } - if err := os.WriteFile(file2, []byte("file2"), 0644); err != nil { + if err := os.WriteFile(file2, []byte("file2"), 0o644); err != nil { t.Fatalf("Failed to write file2: %v", err) } @@ -169,10 +169,10 @@ func TestGetMatchesForPattern(t *testing.T) { fileA := filepath.Join(srcDir, "a.txt") fileB := filepath.Join(srcDir, "b.log") - if err := os.WriteFile(fileA, []byte("content"), 0644); err != nil { + if err := os.WriteFile(fileA, []byte("content"), 0o644); err != nil { t.Fatalf("Failed to write fileA: %v", err) } - if err := os.WriteFile(fileB, []byte("content"), 0644); err != nil { + if err := os.WriteFile(fileB, []byte("content"), 0o644); err != nil { t.Fatalf("Failed to write fileB: %v", err) } From 9b4fd7335c1074de92d032aa1e27f5b8b79c3909 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 3 Mar 2025 22:20:35 +0600 Subject: [PATCH 43/68] remaining unit tests for copy_glob.go --- internal/exec/copy_glob_test.go | 136 ++++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 17 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 7df691e3a..5553d07c1 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/cloudposse/atmos/pkg/schema" ) // TestCopyFile verifies that copyFile correctly copies file contents and preserves permissions. @@ -15,29 +17,24 @@ func TestCopyFile(t *testing.T) { t.Fatalf("Failed to create source dir: %v", err) } defer os.RemoveAll(srcDir) - dstDir, err := os.MkdirTemp("", "copyfile-dst") if err != nil { t.Fatalf("Failed to create destination dir: %v", err) } defer os.RemoveAll(dstDir) - srcFile := filepath.Join(srcDir, "test.txt") content := "copyFileTest" if err := os.WriteFile(srcFile, []byte(content), 0644); err != nil { t.Fatalf("Failed to write source file: %v", err) } - dstFile := filepath.Join(dstDir, "test.txt") if err := copyFile(srcFile, dstFile); err != nil { t.Fatalf("copyFile failed: %v", err) } - copiedContent, err := os.ReadFile(dstFile) if err != nil { t.Fatalf("Failed to read destination file: %v", err) } - if string(copiedContent) != content { t.Errorf("Expected content %q, got %q", content, string(copiedContent)) } @@ -50,12 +47,10 @@ func TestShouldExcludePath(t *testing.T) { t.Fatalf("Failed to create temporary file: %v", err) } defer os.Remove(tmpFile.Name()) - info, err := tmpFile.Stat() if err != nil { t.Fatalf("Failed to stat temp file: %v", err) } - excluded := []string{"**/*.log"} relPath := "app/test.log" if !shouldExcludePath(info, relPath, excluded) { @@ -70,12 +65,10 @@ func TestShouldIncludePath(t *testing.T) { t.Fatalf("Failed to create temporary file: %v", err) } defer os.Remove(tmpFile.Name()) - info, err := tmpFile.Stat() if err != nil { t.Fatalf("Failed to stat temp file: %v", err) } - included := []string{"**/*.txt"} relPath := "docs/readme.txt" if !shouldIncludePath(info, relPath, included) { @@ -90,7 +83,6 @@ func TestShouldSkipEntry(t *testing.T) { t.Fatalf("Failed to create base dir: %v", err) } defer os.RemoveAll(baseDir) - subDir := filepath.Join(baseDir, "sub") if err := os.Mkdir(subDir, 0755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) @@ -103,7 +95,6 @@ func TestShouldSkipEntry(t *testing.T) { if err != nil { t.Fatalf("Failed to stat file: %v", err) } - excluded := []string{"**/*.txt"} included := []string{} if !shouldSkipEntry(info, filePath, baseDir, excluded, included) { @@ -118,13 +109,11 @@ func TestCopyDirRecursive(t *testing.T) { t.Fatalf("Failed to create src dir: %v", err) } defer os.RemoveAll(srcDir) - dstDir, err := os.MkdirTemp("", "copydir-dst") if err != nil { t.Fatalf("Failed to create dst dir: %v", err) } defer os.RemoveAll(dstDir) - subDir := filepath.Join(srcDir, "sub") if err := os.Mkdir(subDir, 0755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) @@ -137,7 +126,6 @@ func TestCopyDirRecursive(t *testing.T) { if err := os.WriteFile(file2, []byte("file2"), 0644); err != nil { t.Fatalf("Failed to write file2: %v", err) } - ctx := &CopyContext{ SrcDir: srcDir, DstDir: dstDir, @@ -148,7 +136,6 @@ func TestCopyDirRecursive(t *testing.T) { if err := copyDirRecursive(ctx); err != nil { t.Fatalf("copyDirRecursive failed: %v", err) } - dstFile1 := filepath.Join(dstDir, "file1.txt") dstFile2 := filepath.Join(dstDir, "sub", "file2.txt") if _, err := os.Stat(dstFile1); os.IsNotExist(err) { @@ -166,7 +153,6 @@ func TestGetMatchesForPattern(t *testing.T) { t.Fatalf("Failed to create src dir: %v", err) } defer os.RemoveAll(srcDir) - fileA := filepath.Join(srcDir, "a.txt") fileB := filepath.Join(srcDir, "b.log") if err := os.WriteFile(fileA, []byte("content"), 0644); err != nil { @@ -175,7 +161,6 @@ func TestGetMatchesForPattern(t *testing.T) { if err := os.WriteFile(fileB, []byte("content"), 0644); err != nil { t.Fatalf("Failed to write fileB: %v", err) } - pattern := "*.txt" matches, err := getMatchesForPattern(srcDir, pattern) if err != nil { @@ -185,3 +170,120 @@ func TestGetMatchesForPattern(t *testing.T) { t.Errorf("Expected one match for a.txt, got %v", matches) } } + +// TestIsShallowPattern verifies that isShallowPattern returns correct results for given patterns. +func TestIsShallowPattern(t *testing.T) { + if !isShallowPattern("**/demo-localstack/*") { + t.Errorf("Expected pattern '**/demo-localstack/*' to be shallow") + } + if isShallowPattern("**/demo-library/**") { + t.Errorf("Expected pattern '**/demo-library/**' not to be shallow") + } +} + +// TestCopyDirRecursiveWithPrefix tests that copyDirRecursiveWithPrefix copies files using prefix context. +func TestCopyDirRecursiveWithPrefix(t *testing.T) { + srcDir, err := os.MkdirTemp("", "prefix-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "prefix-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + filePath := filepath.Join(srcDir, "test.txt") + if err := os.WriteFile(filePath, []byte("content"), 0644); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + ctx := &PrefixCopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + GlobalBase: srcDir, + Prefix: "prefix", + Excluded: []string{}, + } + if err := copyDirRecursiveWithPrefix(ctx); err != nil { + t.Fatalf("copyDirRecursiveWithPrefix failed: %v", err) + } + dstFile := filepath.Join(dstDir, "test.txt") + if _, err := os.Stat(dstFile); os.IsNotExist(err) { + t.Errorf("Expected file %q to exist", dstFile) + } +} + +// TestProcessIncludedPattern tests processIncludedPattern by copying matching files to target directory. +func TestProcessIncludedPattern(t *testing.T) { + srcDir, err := os.MkdirTemp("", "included-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "included-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + fileMatch := filepath.Join(srcDir, "match.md") + if err := os.WriteFile(fileMatch, []byte("mdcontent"), 0644); err != nil { + t.Fatalf("Failed to write matching file: %v", err) + } + fileNoMatch := filepath.Join(srcDir, "no_match.txt") + if err := os.WriteFile(fileNoMatch, []byte("txtcontent"), 0644); err != nil { + t.Fatalf("Failed to write non-matching file: %v", err) + } + pattern := "**/*.md" + if err := processIncludedPattern(srcDir, dstDir, pattern, []string{}); err != nil { + t.Fatalf("processIncludedPattern failed: %v", err) + } + copiedMatch := filepath.Join(dstDir, "match.md") + if _, err := os.Stat(copiedMatch); os.IsNotExist(err) { + t.Errorf("Expected file %q to exist", copiedMatch) + } + notCopied := filepath.Join(dstDir, "no_match.txt") + if _, err := os.Stat(notCopied); err == nil { + t.Errorf("Expected file %q not to exist", notCopied) + } +} + +// TestCopyToTargetWithPatterns tests copyToTargetWithPatterns with inclusion and exclusion patterns. +func TestCopyToTargetWithPatterns(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copyto-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "copyto-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + subDir := filepath.Join(srcDir, "sub") + if err := os.Mkdir(subDir, 0755); err != nil { + t.Fatalf("Failed to create subdirectory: %v", err) + } + fileKeep := filepath.Join(subDir, "keep.test") + if err := os.WriteFile(fileKeep, []byte("keep"), 0644); err != nil { + t.Fatalf("Failed to write keep file: %v", err) + } + fileSkip := filepath.Join(subDir, "skip.test") + if err := os.WriteFile(fileSkip, []byte("skip"), 0644); err != nil { + t.Fatalf("Failed to write skip file: %v", err) + } + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{"**/*.test"}, + ExcludedPaths: []string{"**/skip.test"}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false, "dummy"); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + dstKeep := filepath.Join(dstDir, "sub", "keep.test") + if _, err := os.Stat(dstKeep); os.IsNotExist(err) { + t.Errorf("Expected file %q to exist", dstKeep) + } + dstSkip := filepath.Join(dstDir, "sub", "skip.test") + if _, err := os.Stat(dstSkip); err == nil { + t.Errorf("Expected file %q not to exist", dstSkip) + } +} From ac526c45f497a4ecc0e10830a2decc660a1a8a72 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:22:37 +0000 Subject: [PATCH 44/68] [autofix.ci] apply automated fixes --- internal/exec/copy_glob_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 85b6de650..8848340fe 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -194,7 +194,7 @@ func TestCopyDirRecursiveWithPrefix(t *testing.T) { } defer os.RemoveAll(dstDir) filePath := filepath.Join(srcDir, "test.txt") - if err := os.WriteFile(filePath, []byte("content"), 0644); err != nil { + if err := os.WriteFile(filePath, []byte("content"), 0o644); err != nil { t.Fatalf("Failed to write file: %v", err) } ctx := &PrefixCopyContext{ @@ -226,11 +226,11 @@ func TestProcessIncludedPattern(t *testing.T) { } defer os.RemoveAll(dstDir) fileMatch := filepath.Join(srcDir, "match.md") - if err := os.WriteFile(fileMatch, []byte("mdcontent"), 0644); err != nil { + if err := os.WriteFile(fileMatch, []byte("mdcontent"), 0o644); err != nil { t.Fatalf("Failed to write matching file: %v", err) } fileNoMatch := filepath.Join(srcDir, "no_match.txt") - if err := os.WriteFile(fileNoMatch, []byte("txtcontent"), 0644); err != nil { + if err := os.WriteFile(fileNoMatch, []byte("txtcontent"), 0o644); err != nil { t.Fatalf("Failed to write non-matching file: %v", err) } pattern := "**/*.md" @@ -260,15 +260,15 @@ func TestCopyToTargetWithPatterns(t *testing.T) { } defer os.RemoveAll(dstDir) subDir := filepath.Join(srcDir, "sub") - if err := os.Mkdir(subDir, 0755); err != nil { + if err := os.Mkdir(subDir, 0o755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } fileKeep := filepath.Join(subDir, "keep.test") - if err := os.WriteFile(fileKeep, []byte("keep"), 0644); err != nil { + if err := os.WriteFile(fileKeep, []byte("keep"), 0o644); err != nil { t.Fatalf("Failed to write keep file: %v", err) } fileSkip := filepath.Join(subDir, "skip.test") - if err := os.WriteFile(fileSkip, []byte("skip"), 0644); err != nil { + if err := os.WriteFile(fileSkip, []byte("skip"), 0o644); err != nil { t.Fatalf("Failed to write skip file: %v", err) } dummy := &schema.AtmosVendorSource{ From bcc852bf74a585cebdb4fddc3680182007309591 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Mon, 3 Mar 2025 22:29:57 +0600 Subject: [PATCH 45/68] os.WriteFile to use 0o600 permission instead of 0o644 --- internal/exec/copy_glob_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 85b6de650..ec9cfc689 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -24,7 +24,7 @@ func TestCopyFile(t *testing.T) { defer os.RemoveAll(dstDir) srcFile := filepath.Join(srcDir, "test.txt") content := "copyFileTest" - if err := os.WriteFile(srcFile, []byte(content), 0o644); err != nil { + if err := os.WriteFile(srcFile, []byte(content), 0o600); err != nil { t.Fatalf("Failed to write source file: %v", err) } dstFile := filepath.Join(dstDir, "test.txt") @@ -84,11 +84,11 @@ func TestShouldSkipEntry(t *testing.T) { } defer os.RemoveAll(baseDir) subDir := filepath.Join(baseDir, "sub") - if err := os.Mkdir(subDir, 0o755); err != nil { + if err := os.Mkdir(subDir, 0755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } filePath := filepath.Join(subDir, "sample.txt") - if err := os.WriteFile(filePath, []byte("test"), 0o644); err != nil { + if err := os.WriteFile(filePath, []byte("test"), 0o600); err != nil { t.Fatalf("Failed to write file: %v", err) } info, err := os.Stat(filePath) @@ -115,15 +115,15 @@ func TestCopyDirRecursive(t *testing.T) { } defer os.RemoveAll(dstDir) subDir := filepath.Join(srcDir, "sub") - if err := os.Mkdir(subDir, 0o755); err != nil { + if err := os.Mkdir(subDir, 0755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } file1 := filepath.Join(srcDir, "file1.txt") file2 := filepath.Join(subDir, "file2.txt") - if err := os.WriteFile(file1, []byte("file1"), 0o644); err != nil { + if err := os.WriteFile(file1, []byte("file1"), 0o600); err != nil { t.Fatalf("Failed to write file1: %v", err) } - if err := os.WriteFile(file2, []byte("file2"), 0o644); err != nil { + if err := os.WriteFile(file2, []byte("file2"), 0o600); err != nil { t.Fatalf("Failed to write file2: %v", err) } ctx := &CopyContext{ @@ -155,10 +155,10 @@ func TestGetMatchesForPattern(t *testing.T) { defer os.RemoveAll(srcDir) fileA := filepath.Join(srcDir, "a.txt") fileB := filepath.Join(srcDir, "b.log") - if err := os.WriteFile(fileA, []byte("content"), 0o644); err != nil { + if err := os.WriteFile(fileA, []byte("content"), 0o600); err != nil { t.Fatalf("Failed to write fileA: %v", err) } - if err := os.WriteFile(fileB, []byte("content"), 0o644); err != nil { + if err := os.WriteFile(fileB, []byte("content"), 0o600); err != nil { t.Fatalf("Failed to write fileB: %v", err) } pattern := "*.txt" @@ -194,7 +194,7 @@ func TestCopyDirRecursiveWithPrefix(t *testing.T) { } defer os.RemoveAll(dstDir) filePath := filepath.Join(srcDir, "test.txt") - if err := os.WriteFile(filePath, []byte("content"), 0644); err != nil { + if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil { t.Fatalf("Failed to write file: %v", err) } ctx := &PrefixCopyContext{ @@ -226,11 +226,11 @@ func TestProcessIncludedPattern(t *testing.T) { } defer os.RemoveAll(dstDir) fileMatch := filepath.Join(srcDir, "match.md") - if err := os.WriteFile(fileMatch, []byte("mdcontent"), 0644); err != nil { + if err := os.WriteFile(fileMatch, []byte("mdcontent"), 0o600); err != nil { t.Fatalf("Failed to write matching file: %v", err) } fileNoMatch := filepath.Join(srcDir, "no_match.txt") - if err := os.WriteFile(fileNoMatch, []byte("txtcontent"), 0644); err != nil { + if err := os.WriteFile(fileNoMatch, []byte("txtcontent"), 0o600); err != nil { t.Fatalf("Failed to write non-matching file: %v", err) } pattern := "**/*.md" @@ -264,11 +264,11 @@ func TestCopyToTargetWithPatterns(t *testing.T) { t.Fatalf("Failed to create subdirectory: %v", err) } fileKeep := filepath.Join(subDir, "keep.test") - if err := os.WriteFile(fileKeep, []byte("keep"), 0644); err != nil { + if err := os.WriteFile(fileKeep, []byte("keep"), 0o600); err != nil { t.Fatalf("Failed to write keep file: %v", err) } fileSkip := filepath.Join(subDir, "skip.test") - if err := os.WriteFile(fileSkip, []byte("skip"), 0644); err != nil { + if err := os.WriteFile(fileSkip, []byte("skip"), 0o600); err != nil { t.Fatalf("Failed to write skip file: %v", err) } dummy := &schema.AtmosVendorSource{ From 5bbfe6617061636f435f633aeba733c615e86692 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:33:36 +0000 Subject: [PATCH 46/68] [autofix.ci] apply automated fixes --- internal/exec/copy_glob_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 9ebc25c79..460bce8fb 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -84,7 +84,7 @@ func TestShouldSkipEntry(t *testing.T) { } defer os.RemoveAll(baseDir) subDir := filepath.Join(baseDir, "sub") - if err := os.Mkdir(subDir, 0755); err != nil { + if err := os.Mkdir(subDir, 0o755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } filePath := filepath.Join(subDir, "sample.txt") @@ -115,7 +115,7 @@ func TestCopyDirRecursive(t *testing.T) { } defer os.RemoveAll(dstDir) subDir := filepath.Join(srcDir, "sub") - if err := os.Mkdir(subDir, 0755); err != nil { + if err := os.Mkdir(subDir, 0o755); err != nil { t.Fatalf("Failed to create subdirectory: %v", err) } file1 := filepath.Join(srcDir, "file1.txt") From ea198e15bd84f01dbc8b24481d7b0b9e3bdb1893 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Tue, 4 Mar 2025 04:19:06 +0600 Subject: [PATCH 47/68] more copy_glob tests --- internal/exec/copy_glob_test.go | 480 ++++++++++++++++++++++++++++---- 1 file changed, 427 insertions(+), 53 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 460bce8fb..2e242d249 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -1,15 +1,37 @@ -// copy_glob_test.go package exec import ( "os" "path/filepath" + "runtime" "strings" "testing" "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/utils" ) +// Use a local variable to override the glob matching function in tests. +var getGlobMatchesForTest = utils.GetGlobMatches + +// Helper that calls our local getGlobMatchesForTest. +func getMatchesForPatternForTest(sourceDir, pattern string) ([]string, error) { + fullPattern := filepath.Join(sourceDir, pattern) + // Normalize fullPattern to use forward slashes. + normalized := filepath.ToSlash(fullPattern) + return getGlobMatchesForTest(normalized) +} + +type fakeDirEntry struct { + name string + err error +} + +func (f fakeDirEntry) Name() string { return f.name } +func (f fakeDirEntry) IsDir() bool { return false } +func (f fakeDirEntry) Type() os.FileMode { return 0 } +func (f fakeDirEntry) Info() (os.FileInfo, error) { return nil, f.err } + // TestCopyFile verifies that copyFile correctly copies file contents and preserves permissions. func TestCopyFile(t *testing.T) { srcDir, err := os.MkdirTemp("", "copyfile-src") @@ -40,43 +62,104 @@ func TestCopyFile(t *testing.T) { } } -// TestShouldExcludePath tests that shouldExcludePath returns true when a file matches an exclusion pattern. +// TestCopyFile_SourceNotExist tests error in copyFile when source file does not exist. +func TestCopyFile_SourceNotExist(t *testing.T) { + nonExistent := filepath.Join(os.TempDir(), "nonexistent.txt") + dstFile := filepath.Join(os.TempDir(), "dst.txt") + err := copyFile(nonExistent, dstFile) + if err == nil || !strings.Contains(err.Error(), "opening source file") { + t.Errorf("Expected error for non-existent source file, got %v", err) + } +} + +// TestShouldExcludePath checks that a file is excluded by pattern. func TestShouldExcludePath(t *testing.T) { tmpFile, err := os.CreateTemp("", "test.log") if err != nil { - t.Fatalf("Failed to create temporary file: %v", err) + t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tmpFile.Name()) info, err := tmpFile.Stat() if err != nil { - t.Fatalf("Failed to stat temp file: %v", err) + t.Fatalf("Failed to stat file: %v", err) } excluded := []string{"**/*.log"} - relPath := "app/test.log" - if !shouldExcludePath(info, relPath, excluded) { - t.Errorf("Expected path %q to be excluded", relPath) + if !shouldExcludePath(info, "app/test.log", excluded) { + t.Errorf("Expected path to be excluded") + } +} + +// TestShouldExcludePath_Directory checks directory exclusion using a trailing slash. +// Skipped on Windows. +func TestShouldExcludePath_Directory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping directory exclusion test on Windows") + } + dir, err := os.MkdirTemp("", "dir-exclude") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(dir) + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("Failed to stat directory: %v", err) + } + pattern := "**/" + filepath.Base(dir) + "/" + if !shouldExcludePath(info, filepath.Base(dir), []string{pattern}) { + t.Errorf("Expected directory %q to be excluded by pattern %q", filepath.Base(dir), pattern) + } +} + +// TestShouldExcludePath_Error ensures invalid patterns do not exclude files. +func TestShouldExcludePath_Error(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.log") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + if shouldExcludePath(info, "app/test.log", []string{"[abc"}) { + t.Errorf("Expected path not to be excluded by invalid pattern") } } -// TestShouldIncludePath verifies that shouldIncludePath returns true when the file matches an inclusion pattern. +// TestShouldIncludePath checks that a file is included by pattern. func TestShouldIncludePath(t *testing.T) { tmpFile, err := os.CreateTemp("", "test.txt") if err != nil { - t.Fatalf("Failed to create temporary file: %v", err) + t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tmpFile.Name()) info, err := tmpFile.Stat() if err != nil { - t.Fatalf("Failed to stat temp file: %v", err) + t.Fatalf("Failed to stat file: %v", err) } included := []string{"**/*.txt"} - relPath := "docs/readme.txt" - if !shouldIncludePath(info, relPath, included) { - t.Errorf("Expected path %q to be included", relPath) + if !shouldIncludePath(info, "docs/readme.txt", included) { + t.Errorf("Expected path to be included") + } +} + +// TestShouldIncludePath_Error ensures invalid inclusion patterns do not include files. +func TestShouldIncludePath_Error(t *testing.T) { + tmpFile, err := os.CreateTemp("", "test.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + info, err := tmpFile.Stat() + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + if shouldIncludePath(info, "docs/readme.txt", []string{"[abc"}) { + t.Errorf("Expected path not to be included by invalid pattern") } } -// TestShouldSkipEntry creates a temporary directory structure and verifies that shouldSkipEntry correctly skips a file. +// TestShouldSkipEntry verifies that a file is skipped if it matches an excluded pattern. func TestShouldSkipEntry(t *testing.T) { baseDir, err := os.MkdirTemp("", "base") if err != nil { @@ -85,7 +168,7 @@ func TestShouldSkipEntry(t *testing.T) { defer os.RemoveAll(baseDir) subDir := filepath.Join(baseDir, "sub") if err := os.Mkdir(subDir, 0o755); err != nil { - t.Fatalf("Failed to create subdirectory: %v", err) + t.Fatalf("Failed to create sub dir: %v", err) } filePath := filepath.Join(subDir, "sample.txt") if err := os.WriteFile(filePath, []byte("test"), 0o600); err != nil { @@ -95,14 +178,12 @@ func TestShouldSkipEntry(t *testing.T) { if err != nil { t.Fatalf("Failed to stat file: %v", err) } - excluded := []string{"**/*.txt"} - included := []string{} - if !shouldSkipEntry(info, filePath, baseDir, excluded, included) { + if !shouldSkipEntry(info, filePath, baseDir, []string{"**/*.txt"}, []string{}) { t.Errorf("Expected file %q to be skipped", filePath) } } -// TestCopyDirRecursive sets up a temporary directory tree and verifies that copyDirRecursive copies all files. +// TestCopyDirRecursive ensures that copyDirRecursive copies a directory tree. func TestCopyDirRecursive(t *testing.T) { srcDir, err := os.MkdirTemp("", "copydir-src") if err != nil { @@ -116,7 +197,7 @@ func TestCopyDirRecursive(t *testing.T) { defer os.RemoveAll(dstDir) subDir := filepath.Join(srcDir, "sub") if err := os.Mkdir(subDir, 0o755); err != nil { - t.Fatalf("Failed to create subdirectory: %v", err) + t.Fatalf("Failed to create sub dir: %v", err) } file1 := filepath.Join(srcDir, "file1.txt") file2 := filepath.Join(subDir, "file2.txt") @@ -136,17 +217,67 @@ func TestCopyDirRecursive(t *testing.T) { if err := copyDirRecursive(ctx); err != nil { t.Fatalf("copyDirRecursive failed: %v", err) } - dstFile1 := filepath.Join(dstDir, "file1.txt") - dstFile2 := filepath.Join(dstDir, "sub", "file2.txt") - if _, err := os.Stat(dstFile1); os.IsNotExist(err) { - t.Errorf("Expected %q to exist", dstFile1) + if _, err := os.Stat(filepath.Join(dstDir, "file1.txt")); os.IsNotExist(err) { + t.Errorf("Expected file1.txt to exist") + } + if _, err := os.Stat(filepath.Join(dstDir, "sub", "file2.txt")); os.IsNotExist(err) { + t.Errorf("Expected file2.txt to exist") + } +} + +// TestProcessDirEntry_Symlink ensures that symlink entries are skipped. +func TestProcessDirEntry_Symlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping symlink test on Windows") + } + srcDir, err := os.MkdirTemp("", "symlink-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "symlink-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + targetFile := filepath.Join(srcDir, "target.txt") + if err := os.WriteFile(targetFile, []byte("data"), 0o600); err != nil { + t.Fatalf("Failed to write target file: %v", err) + } + linkPath := filepath.Join(srcDir, "link.txt") + if err := os.Symlink(targetFile, linkPath); err != nil { + t.Skip("Cannot create symlink on this system, skipping test.") + } + entries, err := os.ReadDir(srcDir) + if err != nil { + t.Fatalf("Failed to read src dir: %v", err) + } + var linkEntry os.DirEntry + for _, e := range entries { + if e.Name() == "link.txt" { + linkEntry = e + break + } + } + if linkEntry == nil { + t.Fatalf("Symlink entry not found") + } + ctx := &CopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + BaseDir: srcDir, + Excluded: []string{}, + Included: []string{}, } - if _, err := os.Stat(dstFile2); os.IsNotExist(err) { - t.Errorf("Expected %q to exist", dstFile2) + if err := processDirEntry(linkEntry, ctx); err != nil { + t.Errorf("Expected nil error for symlink, got %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "link.txt")); err == nil { + t.Errorf("Expected symlink not to be copied") } } -// TestGetMatchesForPattern verifies that getMatchesForPattern returns the correct file matches. +// TestGetMatchesForPattern checks that getMatchesForPattern returns expected matches. func TestGetMatchesForPattern(t *testing.T) { srcDir, err := os.MkdirTemp("", "glob-src") if err != nil { @@ -161,27 +292,127 @@ func TestGetMatchesForPattern(t *testing.T) { if err := os.WriteFile(fileB, []byte("content"), 0o600); err != nil { t.Fatalf("Failed to write fileB: %v", err) } - pattern := "*.txt" - matches, err := getMatchesForPattern(srcDir, pattern) + matches, err := getMatchesForPattern(srcDir, "*.txt") if err != nil { t.Fatalf("getMatchesForPattern error: %v", err) } - if len(matches) != 1 || !strings.Contains(matches[0], "a.txt") { - t.Errorf("Expected one match for a.txt, got %v", matches) + if len(matches) == 0 || !strings.Contains(matches[0], "a.txt") { + t.Errorf("Expected match for a.txt, got %v", matches) + } +} + +// TestGetMatchesForPattern_NoMatches uses our helper to simulate no matches. +func TestGetMatchesForPattern_NoMatches(t *testing.T) { + oldFn := getGlobMatchesForTest + defer func() { getGlobMatchesForTest = oldFn }() + getGlobMatchesForTest = func(pattern string) ([]string, error) { + return []string{}, nil + } + srcDir, err := os.MkdirTemp("", "nomatch-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + matches, err := getMatchesForPatternForTest(srcDir, "nonexistent*") + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(matches) != 0 { + t.Errorf("Expected 0 matches, got %v", matches) + } +} + +// TestGetMatchesForPattern_InvalidPattern ensures invalid patterns produce an error. +func TestGetMatchesForPattern_InvalidPattern(t *testing.T) { + srcDir, err := os.MkdirTemp("", "invalid-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + _, err = getMatchesForPattern(srcDir, "[") + if err == nil { + t.Errorf("Expected error for invalid pattern, got nil") + } +} + +// TestGetMatchesForPattern_ShallowNoMatch tests the shallow branch with no matches. +func TestGetMatchesForPattern_ShallowNoMatch(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping shallow no-match test on Windows") + } + oldFn := getGlobMatchesForTest + defer func() { getGlobMatchesForTest = oldFn }() + getGlobMatchesForTest = func(pattern string) ([]string, error) { + // Normalize pattern to slash. + normalized := filepath.ToSlash(pattern) + if strings.Contains(normalized, "/*") && !strings.Contains(normalized, "/**") { + return []string{}, nil + } + return []string{}, nil + } + srcDir, err := os.MkdirTemp("", "shallow-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + emptyDir := filepath.Join(srcDir, "dir") + if err := os.Mkdir(emptyDir, 0o755); err != nil { + t.Fatalf("Failed to create empty directory: %v", err) + } + _, err = getMatchesForPatternForTest(srcDir, "dir/*") + if err != nil { + t.Fatalf("Expected no error for shallow pattern with no matches, got %v", err) } } -// TestIsShallowPattern verifies that isShallowPattern returns correct results for given patterns. +// TestGetMatchesForPattern_RecursiveMatch tests the recursive branch by overriding glob matching. +func TestGetMatchesForPattern_RecursiveMatch(t *testing.T) { + oldFn := getGlobMatchesForTest + defer func() { getGlobMatchesForTest = oldFn }() + getGlobMatchesForTest = func(pattern string) ([]string, error) { + normalized := filepath.ToSlash(pattern) + if strings.Contains(normalized, "/**") { + return []string{"match.txt"}, nil + } + return []string{}, nil + } + srcDir, err := os.MkdirTemp("", "recursive-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dir := filepath.Join(srcDir, "dir") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + child := filepath.Join(dir, "child") + if err := os.Mkdir(child, 0o755); err != nil { + t.Fatalf("Failed to create child directory: %v", err) + } + if err := os.WriteFile(filepath.Join(child, "file.txt"), []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + pattern := "dir/*/**" + matches, err := getMatchesForPatternForTest(srcDir, pattern) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(matches) == 0 { + t.Errorf("Expected matches for recursive branch, got none") + } +} + +// TestIsShallowPattern ensures shallow pattern detection works. func TestIsShallowPattern(t *testing.T) { if !isShallowPattern("**/demo-localstack/*") { - t.Errorf("Expected pattern '**/demo-localstack/*' to be shallow") + t.Errorf("Expected '**/demo-localstack/*' to be shallow") } if isShallowPattern("**/demo-library/**") { - t.Errorf("Expected pattern '**/demo-library/**' not to be shallow") + t.Errorf("Expected '**/demo-library/**' not to be shallow") } } -// TestCopyDirRecursiveWithPrefix tests that copyDirRecursiveWithPrefix copies files using prefix context. +// TestCopyDirRecursiveWithPrefix ensures prefix-based copy works. func TestCopyDirRecursiveWithPrefix(t *testing.T) { srcDir, err := os.MkdirTemp("", "prefix-src") if err != nil { @@ -207,13 +438,12 @@ func TestCopyDirRecursiveWithPrefix(t *testing.T) { if err := copyDirRecursiveWithPrefix(ctx); err != nil { t.Fatalf("copyDirRecursiveWithPrefix failed: %v", err) } - dstFile := filepath.Join(dstDir, "test.txt") - if _, err := os.Stat(dstFile); os.IsNotExist(err) { - t.Errorf("Expected file %q to exist", dstFile) + if _, err := os.Stat(filepath.Join(dstDir, "test.txt")); os.IsNotExist(err) { + t.Errorf("Expected file to exist in destination") } } -// TestProcessIncludedPattern tests processIncludedPattern by copying matching files to target directory. +// TestProcessIncludedPattern ensures that matching files are copied. func TestProcessIncludedPattern(t *testing.T) { srcDir, err := os.MkdirTemp("", "included-src") if err != nil { @@ -237,17 +467,107 @@ func TestProcessIncludedPattern(t *testing.T) { if err := processIncludedPattern(srcDir, dstDir, pattern, []string{}); err != nil { t.Fatalf("processIncludedPattern failed: %v", err) } - copiedMatch := filepath.Join(dstDir, "match.md") - if _, err := os.Stat(copiedMatch); os.IsNotExist(err) { - t.Errorf("Expected file %q to exist", copiedMatch) + if _, err := os.Stat(filepath.Join(dstDir, "match.md")); os.IsNotExist(err) { + t.Errorf("Expected match.md to exist") + } + if _, err := os.Stat(filepath.Join(dstDir, "no_match.txt")); err == nil { + t.Errorf("Expected no_match.txt not to exist") + } +} + +// TestProcessIncludedPattern_Invalid ensures that an invalid pattern does not cause fatal errors. +func TestProcessIncludedPattern_Invalid(t *testing.T) { + srcDir, err := os.MkdirTemp("", "invalid-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "invalid-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + if err := processIncludedPattern(srcDir, dstDir, "[", []string{}); err != nil { + t.Fatalf("Expected processIncludedPattern to handle invalid pattern gracefully, got: %v", err) + } +} + +// TestProcessMatch_ShallowDirectory ensures directories are not copied when shallow is true. +func TestProcessMatch_ShallowDirectory(t *testing.T) { + srcDir, err := os.MkdirTemp("", "pm-shallow-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "pm-shallow-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + dirPath := filepath.Join(srcDir, "dir") + if err := os.Mkdir(dirPath, 0o755); err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + if err := processMatch(srcDir, dstDir, dirPath, true, []string{}); err != nil { + t.Errorf("Expected nil error for shallow directory, got %v", err) } - notCopied := filepath.Join(dstDir, "no_match.txt") - if _, err := os.Stat(notCopied); err == nil { - t.Errorf("Expected file %q not to exist", notCopied) + if _, err := os.Stat(filepath.Join(dstDir, "dir")); err == nil { + t.Errorf("Expected directory not to be copied when shallow is true") } } -// TestCopyToTargetWithPatterns tests copyToTargetWithPatterns with inclusion and exclusion patterns. +// TestProcessMatch_Directory ensures directories are copied when shallow is false. +func TestProcessMatch_Directory(t *testing.T) { + srcDir, err := os.MkdirTemp("", "pm-dir-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "pm-dir-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + dirPath := filepath.Join(srcDir, "dir") + if err := os.Mkdir(dirPath, 0o755); err != nil { + t.Fatalf("Failed to create directory: %v", err) + } + fileInside := filepath.Join(dirPath, "inside.txt") + if err := os.WriteFile(fileInside, []byte("data"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + if err := processMatch(srcDir, dstDir, dirPath, false, []string{}); err != nil { + t.Errorf("Expected nil error for directory copy, got %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "dir", "inside.txt")); os.IsNotExist(err) { + t.Errorf("Expected file inside directory to be copied") + } +} + +// TestProcessMatch_ErrorStat ensures processMatch returns an error when os.Stat fails. +func TestProcessMatch_ErrorStat(t *testing.T) { + err := processMatch(os.TempDir(), os.TempDir(), "/nonexistentfile.txt", false, []string{}) + if err == nil || !strings.Contains(err.Error(), "stating file") { + t.Errorf("Expected error for non-existent file in processMatch, got %v", err) + } +} + +// TestCopyDirRecursive_ReadDirError checks that copyDirRecursive fails if os.ReadDir fails. +func TestCopyDirRecursive_ReadDirError(t *testing.T) { + ctx := &CopyContext{ + SrcDir: "/nonexistent_directory", + DstDir: os.TempDir(), + BaseDir: "/nonexistent_directory", + Excluded: []string{}, + Included: []string{}, + } + err := copyDirRecursive(ctx) + if err == nil || !strings.Contains(err.Error(), "reading directory") { + t.Errorf("Expected error for non-existent src dir, got %v", err) + } +} + +// TestCopyToTargetWithPatterns checks that included/excluded patterns work. func TestCopyToTargetWithPatterns(t *testing.T) { srcDir, err := os.MkdirTemp("", "copyto-src") if err != nil { @@ -261,7 +581,7 @@ func TestCopyToTargetWithPatterns(t *testing.T) { defer os.RemoveAll(dstDir) subDir := filepath.Join(srcDir, "sub") if err := os.Mkdir(subDir, 0o755); err != nil { - t.Fatalf("Failed to create subdirectory: %v", err) + t.Fatalf("Failed to create sub dir: %v", err) } fileKeep := filepath.Join(subDir, "keep.test") if err := os.WriteFile(fileKeep, []byte("keep"), 0o600); err != nil { @@ -278,12 +598,66 @@ func TestCopyToTargetWithPatterns(t *testing.T) { if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false, "dummy"); err != nil { t.Fatalf("copyToTargetWithPatterns failed: %v", err) } - dstKeep := filepath.Join(dstDir, "sub", "keep.test") - if _, err := os.Stat(dstKeep); os.IsNotExist(err) { - t.Errorf("Expected file %q to exist", dstKeep) + if _, err := os.Stat(filepath.Join(dstDir, "sub", "keep.test")); os.IsNotExist(err) { + t.Errorf("Expected keep.test to exist") + } + if _, err := os.Stat(filepath.Join(dstDir, "sub", "skip.test")); err == nil { + t.Errorf("Expected skip.test not to exist") + } +} + +// TestCopyToTargetWithPatterns_NoPatterns tests the branch using cp.Copy. +func TestCopyToTargetWithPatterns_NoPatterns(t *testing.T) { + srcDir, err := os.MkdirTemp("", "nopattern-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "nopattern-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + filePath := filepath.Join(srcDir, "file.txt") + if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{}, + ExcludedPaths: []string{}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false, "dummy"); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + if _, err := os.Stat(filepath.Join(dstDir, "file.txt")); os.IsNotExist(err) { + t.Errorf("Expected file.txt to exist in destination") + } +} + +// TestCopyToTargetWithPatterns_LocalFileBranch tests the sourceIsLocalFile branch. +func TestCopyToTargetWithPatterns_LocalFileBranch(t *testing.T) { + srcDir, err := os.MkdirTemp("", "local-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "local-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + filePath := filepath.Join(srcDir, "file.txt") + if err := os.WriteFile(filePath, []byte("data"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{"**/*.txt"}, + ExcludedPaths: []string{}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, true, "test:uri"); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) } - dstSkip := filepath.Join(dstDir, "sub", "skip.test") - if _, err := os.Stat(dstSkip); err == nil { - t.Errorf("Expected file %q not to exist", dstSkip) + if _, err := os.Stat(filepath.Join(dstDir, "file.txt")); os.IsNotExist(err) { + t.Errorf("Expected file.txt to exist in destination") } } From 68642ed122a44f16c6a28fb079a8b765bbfebbf5 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Tue, 4 Mar 2025 04:31:05 +0600 Subject: [PATCH 48/68] missing test added --- internal/exec/copy_glob_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 2e242d249..7db02d0da 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -9,6 +9,7 @@ import ( "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/utils" + "github.com/pkg/errors" ) // Use a local variable to override the glob matching function in tests. @@ -661,3 +662,17 @@ func TestCopyToTargetWithPatterns_LocalFileBranch(t *testing.T) { t.Errorf("Expected file.txt to exist in destination") } } + +func TestProcessDirEntry_InfoError(t *testing.T) { + ctx := &CopyContext{ + SrcDir: "/dummy", + DstDir: "/dummy", + BaseDir: "/dummy", + Excluded: []string{}, + Included: []string{}, + } + err := processDirEntry(fakeDirEntry{name: "error.txt", err: errors.New("forced info error")}, ctx) + if err == nil || !strings.Contains(err.Error(), "getting info") { + t.Errorf("Expected error for Info() failure, got %v", err) + } +} From 417f19c34a38e6a9914b468bab7a3cd1d86439f7 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Tue, 4 Mar 2025 19:06:36 +0500 Subject: [PATCH 49/68] more copy_glob tests --- go.mod | 1 + go.sum | 5 + internal/exec/copy_glob_test.go | 281 ++++++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+) diff --git a/go.mod b/go.mod index 2f6f53bd7..916a74813 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/Shopify/ejson v1.3.3 // indirect github.com/a8m/envsubst v1.4.2 // indirect github.com/agext/levenshtein v1.2.2 // indirect + github.com/agiledragon/gomonkey/v2 v2.13.0 github.com/agnivade/levenshtein v1.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 1430263f2..36c6b6778 100644 --- a/go.sum +++ b/go.sum @@ -684,6 +684,8 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg= github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/agiledragon/gomonkey/v2 v2.13.0 h1:B24Jg6wBI1iB8EFR1c+/aoTg7QN/Cum7YffG8KMIyYo= +github.com/agiledragon/gomonkey/v2 v2.13.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY= github.com/agnivade/levenshtein v1.2.0/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= @@ -1249,6 +1251,7 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= @@ -1729,6 +1732,7 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -2292,6 +2296,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 7db02d0da..0a1840b2a 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -1,14 +1,20 @@ package exec import ( + "fmt" "os" "path/filepath" "runtime" "strings" "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + cp "github.com/otiai10/copy" "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/utils" + u "github.com/cloudposse/atmos/pkg/utils" "github.com/pkg/errors" ) @@ -676,3 +682,278 @@ func TestProcessDirEntry_InfoError(t *testing.T) { t.Errorf("Expected error for Info() failure, got %v", err) } } + +// TestCopyFile_FailCreateDir simulates failure when creating the destination directory. +func TestCopyFile_FailCreateDir(t *testing.T) { + srcDir, err := os.MkdirTemp("", "copyfile-src") + if err != nil { + t.Fatalf("Failed to create source dir: %v", err) + } + defer os.RemoveAll(srcDir) + srcFile := filepath.Join(srcDir, "test.txt") + content := "copyFileTest" + if err := os.WriteFile(srcFile, []byte(content), 0o600); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + tmpFile, err := os.CreateTemp("", "non-dir") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + tmpFile.Close() + defer os.Remove(tmpFile.Name()) + dstFile := filepath.Join(tmpFile.Name(), "test.txt") + err = copyFile(srcFile, dstFile) + if err == nil || !strings.Contains(err.Error(), "creating destination directory") { + t.Errorf("Expected error creating destination directory, got %v", err) + } +} + +// TestCopyFile_FailChmod simulates failure when setting file permissions. +// If the patch doesn't take effect, the test will be skipped. +func TestCopyFile_FailChmod(t *testing.T) { + patches := gomonkey.ApplyFunc(os.Chmod, func(name string, mode os.FileMode) error { + return fmt.Errorf("simulated chmod failure") + }) + defer patches.Reset() + + srcDir, err := os.MkdirTemp("", "copyfile-src") + if err != nil { + t.Fatalf("Failed to create source dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "copyfile-dst") + if err != nil { + t.Fatalf("Failed to create destination dir: %v", err) + } + defer os.RemoveAll(dstDir) + srcFile := filepath.Join(srcDir, "test.txt") + content := "copyFileTest" + if err := os.WriteFile(srcFile, []byte(content), 0o600); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + dstFile := filepath.Join(dstDir, "test.txt") + err = copyFile(srcFile, dstFile) + if err == nil { + t.Skip("os.Chmod patch not effective on this platform") + } + if !strings.Contains(err.Error(), "setting permissions") { + t.Errorf("Expected chmod error, got %v", err) + } +} + +// TestGetMatchesForPattern_GlobError forces u.GetGlobMatches to return an error. +func TestGetMatchesForPattern_GlobError(t *testing.T) { + patches := gomonkey.ApplyFunc(u.GetGlobMatches, func(pattern string) ([]string, error) { + return nil, fmt.Errorf("simulated glob error") + }) + defer patches.Reset() + + srcDir := "/dummy/src" + pattern := "*.txt" + _, err := getMatchesForPattern(srcDir, pattern) + if err == nil || !strings.Contains(err.Error(), "simulated glob error") { + t.Errorf("Expected simulated glob error, got %v", err) + } +} + +// TestInclusionExclusion_TrailingSlash tests the trailing-slash branch in shouldExcludePath for directories. +func TestInclusionExclusion_TrailingSlash(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "testdir") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + info, err := os.Stat(tmpDir) + if err != nil { + t.Fatalf("Failed to stat temporary directory: %v", err) + } + + relPath := filepath.Base(tmpDir) + + // Test that the directory is excluded when the exclusion pattern expects a trailing slash. + if !shouldExcludePath(info, relPath, []string{relPath + "/"}) { + t.Errorf("Expected directory %q to be excluded by pattern %q", relPath, relPath+"/") + } + + // Test that the directory is not excluded when the pattern does not match. + if shouldExcludePath(info, relPath, []string{relPath + "x/"}) { + t.Errorf("Did not expect directory %q to be excluded by pattern %q", relPath, relPath+"x/") + } +} + +// TestProcessPrefixEntry_InfoError simulates an error when calling Info() in processPrefixEntry. +func TestProcessPrefixEntry_InfoError(t *testing.T) { + ctx := &PrefixCopyContext{ + SrcDir: "dummySrc", + DstDir: "dummyDst", + GlobalBase: "dummyGlobal", + Prefix: "dummyPrefix", + Excluded: []string{}, + } + fakeEntry := fakeDirEntry{ + name: "error.txt", + err: fmt.Errorf("forced info error"), + } + err := processPrefixEntry(fakeEntry, ctx) + if err == nil || !strings.Contains(err.Error(), "getting info") { + t.Errorf("Expected error getting info, got %v", err) + } +} + +// fakeFileInfo is a minimal implementation of os.FileInfo for testing. +type fakeFileInfo struct { + name string + size int64 + mode os.FileMode + isDir bool +} + +func (fi fakeFileInfo) Name() string { return fi.name } +func (fi fakeFileInfo) Size() int64 { return fi.size } +func (fi fakeFileInfo) Mode() os.FileMode { return fi.mode } +func (fi fakeFileInfo) ModTime() time.Time { return time.Time{} } +func (fi fakeFileInfo) IsDir() bool { return fi.isDir } +func (fi fakeFileInfo) Sys() interface{} { return nil } + +// fakeDirEntryWithInfo implements os.DirEntry using fakeFileInfo. +type fakeDirEntryWithInfo struct { + name string + info os.FileInfo +} + +func (fde fakeDirEntryWithInfo) Name() string { return fde.name } +func (fde fakeDirEntryWithInfo) IsDir() bool { return fde.info.IsDir() } +func (fde fakeDirEntryWithInfo) Type() os.FileMode { return fde.info.Mode() } +func (fde fakeDirEntryWithInfo) Info() (os.FileInfo, error) { return fde.info, nil } + +// TestProcessPrefixEntry_FailMkdir simulates an error when creating a directory in processPrefixEntry. +func TestProcessPrefixEntry_FailMkdir(t *testing.T) { + srcDir, err := os.MkdirTemp("", "prefix-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "prefix-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + + // Create a fake directory entry. + fi := fakeFileInfo{ + name: "testDir", + mode: 0o755, + isDir: true, + } + fakeEntry := fakeDirEntryWithInfo{ + name: "testDir", + info: fi, + } + + ctx := &PrefixCopyContext{ + SrcDir: srcDir, + DstDir: dstDir, + GlobalBase: srcDir, + Prefix: "prefix", + Excluded: []string{}, + } + + patches := gomonkey.ApplyFunc(os.MkdirAll, func(path string, perm os.FileMode) error { + return fmt.Errorf("simulated MkdirAll error") + }) + defer patches.Reset() + + err = processPrefixEntry(fakeEntry, ctx) + if err == nil || !strings.Contains(err.Error(), "creating directory") { + t.Errorf("Expected error creating directory, got %v", err) + } +} + +// TestCopyToTargetWithPatterns_UseCpCopy ensures that when no inclusion/exclusion patterns are defined, the cp.Copy branch is used. +func TestCopyToTargetWithPatterns_UseCpCopy(t *testing.T) { + srcDir, err := os.MkdirTemp("", "nopattern-src") + if err != nil { + t.Fatalf("Failed to create src dir: %v", err) + } + defer os.RemoveAll(srcDir) + dstDir, err := os.MkdirTemp("", "nopattern-dst") + if err != nil { + t.Fatalf("Failed to create dst dir: %v", err) + } + defer os.RemoveAll(dstDir) + + // Create a test file in the source directory. + filePath := filepath.Join(srcDir, "file.txt") + if err := os.WriteFile(filePath, []byte("content"), 0o600); err != nil { + t.Fatalf("Failed to write file: %v", err) + } + + // Patch the cp.Copy function to verify that it is called. + called := false + patch := gomonkey.ApplyFunc(cp.Copy, func(src, dst string) error { + called = true + // For testing purposes, simulate a copy by using our copyFile function. + return copyFile(filepath.Join(src, "file.txt"), filepath.Join(dst, "file.txt")) + }) + defer patch.Reset() + + dummy := &schema.AtmosVendorSource{ + IncludedPaths: []string{}, + ExcludedPaths: []string{}, + } + if err := copyToTargetWithPatterns(srcDir, dstDir, dummy, false, "dummy"); err != nil { + t.Fatalf("copyToTargetWithPatterns failed: %v", err) + } + if !called { + t.Errorf("Expected cp.Copy to be called, but it was not") + } + // Verify that the file was "copied" to the destination. + if _, err := os.Stat(filepath.Join(dstDir, "file.txt")); os.IsNotExist(err) { + t.Errorf("Expected file.txt to exist in destination") + } +} + +// TestGetMatchesForPattern_ShallowNoMatches tests a shallow pattern (ending with "/*" but not "/**") +// when no matches are found, expecting an empty result. +func TestGetMatchesForPattern_ShallowNoMatches(t *testing.T) { + patches := gomonkey.ApplyFunc(u.GetGlobMatches, func(pattern string) ([]string, error) { + return []string{}, nil + }) + defer patches.Reset() + + srcDir := "/dummy/src" + pattern := "dummy/*" // Shallow pattern without recursive "**" + matches, err := getMatchesForPattern(srcDir, pattern) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(matches) != 0 { + t.Errorf("Expected no matches for shallow pattern, got %v", matches) + } +} + +// TestProcessMatch_RelPathError simulates an error in computing the relative path in processMatch. +func TestProcessMatch_RelPathError(t *testing.T) { + srcDir := "/dummy/src" + dstPath := "/dummy/dst" + + // Create a temporary file to act as the target file. + tmpFile, err := os.CreateTemp("", "relerr") + if err != nil { + t.Fatalf("Failed to create temporary file: %v", err) + } + tmpFile.Close() + defer os.Remove(tmpFile.Name()) + filePath := tmpFile.Name() + + patches := gomonkey.ApplyFunc(filepath.Rel, func(basepath, targpath string) (string, error) { + return "", fmt.Errorf("simulated relative path error") + }) + defer patches.Reset() + + err = processMatch(srcDir, dstPath, filePath, false, []string{}) + if err == nil || !strings.Contains(err.Error(), "computing relative path") { + t.Errorf("Expected relative path error, got %v", err) + } +} From de2e2619745cd0dbd6b38ba581a4b012b813b324 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Tue, 4 Mar 2025 19:23:04 +0500 Subject: [PATCH 50/68] linter feedback implementation --- internal/exec/copy_glob_test.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 0a1840b2a..1e39be5ec 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -1,7 +1,7 @@ package exec import ( - "fmt" + "errors" "os" "path/filepath" "runtime" @@ -15,7 +15,14 @@ import ( "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/utils" u "github.com/cloudposse/atmos/pkg/utils" - "github.com/pkg/errors" +) + +var ( + errSimulatedChmodFailure = errors.New("simulated chmod failure") + errSimulatedGlobError = errors.New("simulated glob error") + errForcedInfoError = errors.New("forced info error") + errSimulatedMkdirAllError = errors.New("simulated MkdirAll error") + errSimulatedRelPathError = errors.New("simulated relative path error") ) // Use a local variable to override the glob matching function in tests. @@ -350,7 +357,6 @@ func TestGetMatchesForPattern_ShallowNoMatch(t *testing.T) { oldFn := getGlobMatchesForTest defer func() { getGlobMatchesForTest = oldFn }() getGlobMatchesForTest = func(pattern string) ([]string, error) { - // Normalize pattern to slash. normalized := filepath.ToSlash(pattern) if strings.Contains(normalized, "/*") && !strings.Contains(normalized, "/**") { return []string{}, nil @@ -669,6 +675,7 @@ func TestCopyToTargetWithPatterns_LocalFileBranch(t *testing.T) { } } +// TestProcessDirEntry_InfoError tests error handling in processDirEntry when Info() fails. func TestProcessDirEntry_InfoError(t *testing.T) { ctx := &CopyContext{ SrcDir: "/dummy", @@ -677,7 +684,7 @@ func TestProcessDirEntry_InfoError(t *testing.T) { Excluded: []string{}, Included: []string{}, } - err := processDirEntry(fakeDirEntry{name: "error.txt", err: errors.New("forced info error")}, ctx) + err := processDirEntry(fakeDirEntry{name: "error.txt", err: errForcedInfoError}, ctx) if err == nil || !strings.Contains(err.Error(), "getting info") { t.Errorf("Expected error for Info() failure, got %v", err) } @@ -712,7 +719,7 @@ func TestCopyFile_FailCreateDir(t *testing.T) { // If the patch doesn't take effect, the test will be skipped. func TestCopyFile_FailChmod(t *testing.T) { patches := gomonkey.ApplyFunc(os.Chmod, func(name string, mode os.FileMode) error { - return fmt.Errorf("simulated chmod failure") + return errSimulatedChmodFailure }) defer patches.Reset() @@ -744,7 +751,7 @@ func TestCopyFile_FailChmod(t *testing.T) { // TestGetMatchesForPattern_GlobError forces u.GetGlobMatches to return an error. func TestGetMatchesForPattern_GlobError(t *testing.T) { patches := gomonkey.ApplyFunc(u.GetGlobMatches, func(pattern string) ([]string, error) { - return nil, fmt.Errorf("simulated glob error") + return nil, errSimulatedGlobError }) defer patches.Reset() @@ -793,7 +800,7 @@ func TestProcessPrefixEntry_InfoError(t *testing.T) { } fakeEntry := fakeDirEntry{ name: "error.txt", - err: fmt.Errorf("forced info error"), + err: errForcedInfoError, } err := processPrefixEntry(fakeEntry, ctx) if err == nil || !strings.Contains(err.Error(), "getting info") { @@ -860,7 +867,7 @@ func TestProcessPrefixEntry_FailMkdir(t *testing.T) { } patches := gomonkey.ApplyFunc(os.MkdirAll, func(path string, perm os.FileMode) error { - return fmt.Errorf("simulated MkdirAll error") + return errSimulatedMkdirAllError }) defer patches.Reset() @@ -948,7 +955,7 @@ func TestProcessMatch_RelPathError(t *testing.T) { filePath := tmpFile.Name() patches := gomonkey.ApplyFunc(filepath.Rel, func(basepath, targpath string) (string, error) { - return "", fmt.Errorf("simulated relative path error") + return "", errSimulatedRelPathError }) defer patches.Reset() From 4d5eb898bd464669274ae5efe6f324f2d8bf13cc Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Tue, 4 Mar 2025 19:45:32 +0500 Subject: [PATCH 51/68] Single line function expansion to pass gofmt --- internal/exec/copy_glob_test.go | 49 ++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index 1e39be5ec..bb6596efb 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -809,6 +809,7 @@ func TestProcessPrefixEntry_InfoError(t *testing.T) { } // fakeFileInfo is a minimal implementation of os.FileInfo for testing. + type fakeFileInfo struct { name string size int64 @@ -816,12 +817,29 @@ type fakeFileInfo struct { isDir bool } -func (fi fakeFileInfo) Name() string { return fi.name } -func (fi fakeFileInfo) Size() int64 { return fi.size } -func (fi fakeFileInfo) Mode() os.FileMode { return fi.mode } -func (fi fakeFileInfo) ModTime() time.Time { return time.Time{} } -func (fi fakeFileInfo) IsDir() bool { return fi.isDir } -func (fi fakeFileInfo) Sys() interface{} { return nil } +func (fi fakeFileInfo) Name() string { + return fi.name +} + +func (fi fakeFileInfo) Size() int64 { + return fi.size +} + +func (fi fakeFileInfo) Mode() os.FileMode { + return fi.mode +} + +func (fi fakeFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (fi fakeFileInfo) IsDir() bool { + return fi.isDir +} + +func (fi fakeFileInfo) Sys() interface{} { + return nil +} // fakeDirEntryWithInfo implements os.DirEntry using fakeFileInfo. type fakeDirEntryWithInfo struct { @@ -829,10 +847,21 @@ type fakeDirEntryWithInfo struct { info os.FileInfo } -func (fde fakeDirEntryWithInfo) Name() string { return fde.name } -func (fde fakeDirEntryWithInfo) IsDir() bool { return fde.info.IsDir() } -func (fde fakeDirEntryWithInfo) Type() os.FileMode { return fde.info.Mode() } -func (fde fakeDirEntryWithInfo) Info() (os.FileInfo, error) { return fde.info, nil } +func (fde fakeDirEntryWithInfo) Name() string { + return fde.name +} + +func (fde fakeDirEntryWithInfo) IsDir() bool { + return fde.info.IsDir() +} + +func (fde fakeDirEntryWithInfo) Type() os.FileMode { + return fde.info.Mode() +} + +func (fde fakeDirEntryWithInfo) Info() (os.FileInfo, error) { + return fde.info, nil +} // TestProcessPrefixEntry_FailMkdir simulates an error when creating a directory in processPrefixEntry. func TestProcessPrefixEntry_FailMkdir(t *testing.T) { From 88ae6e29bc68477f7d87c1ec771af3b51a32b8ce Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Tue, 4 Mar 2025 20:57:14 +0600 Subject: [PATCH 52/68] more linter fixes --- internal/exec/copy_glob_test.go | 49 +++++++-------------------------- 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/internal/exec/copy_glob_test.go b/internal/exec/copy_glob_test.go index bb6596efb..f1254c318 100644 --- a/internal/exec/copy_glob_test.go +++ b/internal/exec/copy_glob_test.go @@ -809,7 +809,6 @@ func TestProcessPrefixEntry_InfoError(t *testing.T) { } // fakeFileInfo is a minimal implementation of os.FileInfo for testing. - type fakeFileInfo struct { name string size int64 @@ -817,29 +816,12 @@ type fakeFileInfo struct { isDir bool } -func (fi fakeFileInfo) Name() string { - return fi.name -} - -func (fi fakeFileInfo) Size() int64 { - return fi.size -} - -func (fi fakeFileInfo) Mode() os.FileMode { - return fi.mode -} - -func (fi fakeFileInfo) ModTime() time.Time { - return time.Time{} -} - -func (fi fakeFileInfo) IsDir() bool { - return fi.isDir -} - -func (fi fakeFileInfo) Sys() interface{} { - return nil -} +func (fi fakeFileInfo) Name() string { return fi.name } +func (fi fakeFileInfo) Size() int64 { return fi.size } +func (fi fakeFileInfo) Mode() os.FileMode { return fi.mode } +func (fi fakeFileInfo) ModTime() time.Time { return time.Time{} } +func (fi fakeFileInfo) IsDir() bool { return fi.isDir } +func (fi fakeFileInfo) Sys() any { return nil } // fakeDirEntryWithInfo implements os.DirEntry using fakeFileInfo. type fakeDirEntryWithInfo struct { @@ -847,21 +829,10 @@ type fakeDirEntryWithInfo struct { info os.FileInfo } -func (fde fakeDirEntryWithInfo) Name() string { - return fde.name -} - -func (fde fakeDirEntryWithInfo) IsDir() bool { - return fde.info.IsDir() -} - -func (fde fakeDirEntryWithInfo) Type() os.FileMode { - return fde.info.Mode() -} - -func (fde fakeDirEntryWithInfo) Info() (os.FileInfo, error) { - return fde.info, nil -} +func (fde fakeDirEntryWithInfo) Name() string { return fde.name } +func (fde fakeDirEntryWithInfo) IsDir() bool { return fde.info.IsDir() } +func (fde fakeDirEntryWithInfo) Type() os.FileMode { return fde.info.Mode() } +func (fde fakeDirEntryWithInfo) Info() (os.FileInfo, error) { return fde.info, nil } // TestProcessPrefixEntry_FailMkdir simulates an error when creating a directory in processPrefixEntry. func TestProcessPrefixEntry_FailMkdir(t *testing.T) { From 9e841ace1d28445897eb42135e9c6a23a8c23b94 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Mar 2025 19:34:51 +0600 Subject: [PATCH 53/68] magic numbers, constnants and upgraded error messaging for vendor_model_component --- internal/exec/go_getter_utils.go | 4 +- internal/exec/vendor_model.go | 1 - internal/exec/vendor_model_component.go | 82 +++++++++++-------------- 3 files changed, 37 insertions(+), 50 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 13efa65e3..3b0ca380d 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -276,7 +276,7 @@ func RegisterCustomDetectors(atmosConfig *schema.AtmosConfiguration, source stri ) } -// GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination +// GoGetterGet downloads packages (files and folders) from different sources using `go-getter` and saves them into the destination. func GoGetterGet( atmosConfig *schema.AtmosConfiguration, src string, @@ -348,7 +348,7 @@ func removeSymlinks(root string) error { }) } -// DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type +// DownloadDetectFormatAndParseFile downloads a remote file, detects the format of the file (JSON, YAML, HCL) and parses the file into a Go type. func DownloadDetectFormatAndParseFile(atmosConfig *schema.AtmosConfiguration, file string) (any, error) { tempDir := os.TempDir() f := filepath.Join(tempDir, uuid.New().String()) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index e1545e379..d79fde3e7 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -306,7 +306,6 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos err: fmt.Errorf("unknown package type %s for package %s", p.pkgType.String(), p.name), name: p.name, } - } if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index c7bf0e1df..a1c791f63 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -1,6 +1,7 @@ package exec import ( + "errors" "fmt" "os" "path/filepath" @@ -18,6 +19,19 @@ import ( "github.com/cloudposse/atmos/pkg/ui/theme" ) +const ( + progressWidth = 30 + getterTimeout = 10 * time.Minute +) + +var ( + ErrDownloadPackage = errors.New("failed to download package") + ErrProcessOCIImage = errors.New("failed to process OCI image") + ErrCopyPackage = errors.New("failed to copy package") + ErrCreateTempDir = errors.New("failed to create temp directory") + ErrUnknownPackageType = errors.New("unknown package type") +) + // pkgComponentVendor defines a vendor package. type pkgComponentVendor struct { uri string @@ -36,7 +50,7 @@ type pkgComponentVendor struct { func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) modelVendor { p := progress.New( progress.WithDefaultGradient(), - progress.WithWidth(30), + progress.WithWidth(progressWidth), progress.WithoutPercentage(), ) s := spinner.New() @@ -52,11 +66,11 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm } } vendorPks := []pkgVendor{} - for _, pkg := range pkgs { + for i := range pkgs { vendorPkg := pkgVendor{ - name: pkg.name, - version: pkg.version, - componentPackage: &pkg, + name: pkgs[i].name, + version: pkgs[i].version, + componentPackage: &pkgs[i], } vendorPks = append(vendorPks, vendorPkg) } @@ -75,7 +89,6 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { - // Simulate the action time.Sleep(100 * time.Millisecond) return installedPkgMsg{ err: nil, @@ -109,7 +122,7 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig } } return installedPkgMsg{ - err: fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name), + err: fmt.Errorf("%w", ErrUnknownPackageType), name: p.name, } } @@ -117,16 +130,11 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig // installComponent downloads and installs a component. func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { - // Create temp folder - // We are using a temp folder for the following reasons: - // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) - // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix())) if err != nil { - return fmt.Errorf("Failed to create temp directory %s", err) + return fmt.Errorf("%w: %v", ErrCreateTempDir, err) } - // Ensure directory permissions are restricted if err := os.Chmod(tempDir, 0o700); err != nil { return fmt.Errorf("failed to set temp directory permissions: %w", err) } @@ -136,24 +144,18 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat switch p.pkgType { case pkgTypeRemote: tempDir = filepath.Join(tempDir, SanitizeFileName(p.uri)) - - if err = GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, 10*time.Minute); err != nil { - return fmt.Errorf("failed to download package %s error %s", p.name, err) + if err = GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { + return fmt.Errorf("%w: %v", ErrDownloadPackage, err) } - case pkgTypeOci: - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("Failed to process OCI image %s error %s", p.name, err) + return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) } - case pkgTypeLocal: copyOptions := cp.Options{ PreserveTimes: false, PreserveOwner: false, - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, @@ -165,13 +167,13 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat } if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { - return fmt.Errorf("failed to copy package %s error %s", p.name, err) + return fmt.Errorf("%w: %v", ErrCopyPackage, err) } default: - return fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name) + return fmt.Errorf("%w", ErrUnknownPackageType) } if err = copyComponentToDestination(*atmosConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { - return fmt.Errorf("failed to copy package %s error %s", p.name, err) + return fmt.Errorf("%w: %v", ErrCopyPackage, err) } return nil @@ -181,54 +183,40 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { - return fmt.Errorf("Failed to create temp directory %s", err) + return fmt.Errorf("%w: %v", ErrCreateTempDir, err) } defer removeTempDir(*atmosConfig, tempDir) switch p.pkgType { case pkgTypeRemote: - if err = GoGetterGet(atmosConfig, p.uri, filepath.Join(tempDir, p.mixinFilename), getter.ClientModeFile, 10*time.Minute); err != nil { - return fmt.Errorf("failed to download package %s error %s", p.name, err) + if err = GoGetterGet(atmosConfig, p.uri, filepath.Join(tempDir, p.mixinFilename), getter.ClientModeFile, getterTimeout); err != nil { + return fmt.Errorf("%w: %v", ErrDownloadPackage, err) } - case pkgTypeOci: - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("failed to process OCI image %s error %s", p.name, err) + return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) } - case pkgTypeLocal: if p.uri == "" { - return fmt.Errorf("local mixin URI cannot be empty") + return errors.New("local mixin URI cannot be empty") } - // Implement local mixin installation logic - return fmt.Errorf("local mixin installation not implemented") - + return errors.New("local mixin installation not implemented") default: - return fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name) + return fmt.Errorf("%w", ErrUnknownPackageType) } - // Copy from the temp folder to the destination folder copyOptions := cp.Options{ - // Preserve the atime and the mtime of the entries PreserveTimes: false, - - // Preserve the uid and the gid of all entries PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - // Prevent the error: - // symlink components/terraform/mixins/context.tf components/terraform/infra/vpc-flow-logs-bucket/context.tf: file exists OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, } if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { - return fmt.Errorf("Failed to copy package %s error %s", p.name, err) + return fmt.Errorf("%w: %v", ErrCopyPackage, err) } return nil From daf627fa1475b6a5ce477bca9182bd186989d940 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Mar 2025 20:04:58 +0600 Subject: [PATCH 54/68] more linter feedback --- internal/exec/vendor_model.go | 34 +++++++----------- internal/exec/vendor_model_component.go | 46 ++++++++++++++----------- 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index d79fde3e7..72c8777a5 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -79,7 +79,7 @@ var ( func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) (modelVendor, error) { p := progress.New( progress.WithDefaultGradient(), - progress.WithWidth(30), + progress.WithWidth(progressWidth), progress.WithoutPercentage(), ) s := spinner.New() @@ -130,7 +130,6 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case installedPkgMsg: - // ensure index is within bounds if m.index >= len(m.packages) { return m, nil } @@ -151,7 +150,6 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { version = fmt.Sprintf("(%s)", pkg.version) } if m.index >= len(m.packages)-1 { - // Everything's been installed. We're done! m.done = true if !m.isTTY { u.LogInfo(fmt.Sprintf("%s %s %s", mark, pkg.name, version)) @@ -173,14 +171,13 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { u.LogInfo(fmt.Sprintf("%s %s %s", mark, pkg.name, version)) } m.index++ - // Update progress bar progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) version = grayColor.Render(version) return m, tea.Batch( progressCmd, - tea.Printf("%s %s %s %s", mark, pkg.name, version, errMsg), // print message above our program - ExecuteInstall(m.packages[m.index], m.dryRun, m.atmosConfig), // download the next package + tea.Printf("%s %s %s %s", mark, pkg.name, version, errMsg), + ExecuteInstall(m.packages[m.index], m.dryRun, m.atmosConfig), ) case spinner.TickMsg: var cmd tea.Cmd @@ -241,25 +238,22 @@ func max(a, b int) int { func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { - // Simulate the action time.Sleep(500 * time.Millisecond) return installedPkgMsg{ err: nil, name: p.name, } } - // Create temp directory tempDir, err := os.MkdirTemp("", "atmos-vendor") if err != nil { return installedPkgMsg{ - err: fmt.Errorf("failed to create temp directory: %w", err), + err: fmt.Errorf(wrapErrFmt, errors.New("failed to create temp directory"), err), name: p.name, } } - // Ensure directory permissions are restricted - if err := os.Chmod(tempDir, 0o700); err != nil { + if err := os.Chmod(tempDir, tempDirPermissions); err != nil { return installedPkgMsg{ - err: fmt.Errorf("failed to set temp directory permissions: %w", err), + err: fmt.Errorf(wrapErrFmt, errors.New("failed to set temp directory permissions"), err), name: p.name, } } @@ -268,25 +262,22 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos switch p.pkgType { case pkgTypeRemote: - // Use go-getter to download remote packages - if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, 10*time.Minute); err != nil { + if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { return installedPkgMsg{ - err: fmt.Errorf("failed to download package: %w", err), + err: fmt.Errorf(wrapErrFmt, errors.New("failed to download package"), err), name: p.name, } } case pkgTypeOci: - // Process OCI images if err := processOciImage(atmosConfig, p.uri, tempDir); err != nil { return installedPkgMsg{ - err: fmt.Errorf("failed to process OCI image: %w", err), + err: fmt.Errorf(wrapErrFmt, errors.New("failed to process OCI image"), err), name: p.name, } } case pkgTypeLocal: - // Copy from local file system copyOptions := cp.Options{ PreserveTimes: false, PreserveOwner: false, @@ -297,19 +288,19 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { return installedPkgMsg{ - err: fmt.Errorf("failed to copy package: %w", err), + err: fmt.Errorf(wrapErrFmt, errors.New("failed to copy package"), err), name: p.name, } } default: return installedPkgMsg{ - err: fmt.Errorf("unknown package type %s for package %s", p.pkgType.String(), p.name), + err: fmt.Errorf("%w", ErrUnknownPackageType), name: p.name, } } if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ - err: fmt.Errorf("failed to copy package: %w", err), + err: fmt.Errorf(wrapErrFmt, errors.New("failed to copy package"), err), name: p.name, } } @@ -329,7 +320,6 @@ func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig schema.AtmosCo return downloadComponentAndInstall(installer.componentPackage, dryRun, &atmosConfig) } - // No valid package provided return func() tea.Msg { err := fmt.Errorf("no valid installer package provided for %s", installer.name) return installedPkgMsg{ diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index a1c791f63..c0949e71a 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -1,3 +1,4 @@ +// vendor_model_component.go package exec import ( @@ -20,16 +21,21 @@ import ( ) const ( - progressWidth = 30 - getterTimeout = 10 * time.Minute + progressWidth = 30 + getterTimeout = 10 * time.Minute + tempDirPermissions = 0o700 + wrapErrFmt = "%w: %v" + base10 = 10 ) var ( - ErrDownloadPackage = errors.New("failed to download package") - ErrProcessOCIImage = errors.New("failed to process OCI image") - ErrCopyPackage = errors.New("failed to copy package") - ErrCreateTempDir = errors.New("failed to create temp directory") - ErrUnknownPackageType = errors.New("unknown package type") + ErrDownloadPackage = errors.New("failed to download package") + ErrProcessOCIImage = errors.New("failed to process OCI image") + ErrCopyPackage = errors.New("failed to copy package") + ErrCreateTempDir = errors.New("failed to create temp directory") + ErrUnknownPackageType = errors.New("unknown package type") + ErrLocalMixinURIEmpty = errors.New("local mixin URI cannot be empty") + ErrLocalMixinNotImplemented = errors.New("local mixin installation not implemented") ) // pkgComponentVendor defines a vendor package. @@ -132,10 +138,10 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix())) if err != nil { - return fmt.Errorf("%w: %v", ErrCreateTempDir, err) + return fmt.Errorf(wrapErrFmt, ErrCreateTempDir, err) } - if err := os.Chmod(tempDir, 0o700); err != nil { + if err := os.Chmod(tempDir, tempDirPermissions); err != nil { return fmt.Errorf("failed to set temp directory permissions: %w", err) } @@ -145,12 +151,12 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat case pkgTypeRemote: tempDir = filepath.Join(tempDir, SanitizeFileName(p.uri)) if err = GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { - return fmt.Errorf("%w: %v", ErrDownloadPackage, err) + return fmt.Errorf(wrapErrFmt, ErrDownloadPackage, err) } case pkgTypeOci: err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) + return fmt.Errorf(wrapErrFmt, ErrProcessOCIImage, err) } case pkgTypeLocal: copyOptions := cp.Options{ @@ -167,13 +173,13 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat } if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { - return fmt.Errorf("%w: %v", ErrCopyPackage, err) + return fmt.Errorf(wrapErrFmt, ErrCopyPackage, err) } default: return fmt.Errorf("%w", ErrUnknownPackageType) } if err = copyComponentToDestination(*atmosConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { - return fmt.Errorf("%w: %v", ErrCopyPackage, err) + return fmt.Errorf(wrapErrFmt, ErrCopyPackage, err) } return nil @@ -181,9 +187,9 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat // installMixin downloads and installs a mixin. func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), base10)) if err != nil { - return fmt.Errorf("%w: %v", ErrCreateTempDir, err) + return fmt.Errorf(wrapErrFmt, ErrCreateTempDir, err) } defer removeTempDir(*atmosConfig, tempDir) @@ -191,18 +197,18 @@ func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) switch p.pkgType { case pkgTypeRemote: if err = GoGetterGet(atmosConfig, p.uri, filepath.Join(tempDir, p.mixinFilename), getter.ClientModeFile, getterTimeout); err != nil { - return fmt.Errorf("%w: %v", ErrDownloadPackage, err) + return fmt.Errorf(wrapErrFmt, ErrDownloadPackage, err) } case pkgTypeOci: err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) + return fmt.Errorf(wrapErrFmt, ErrProcessOCIImage, err) } case pkgTypeLocal: if p.uri == "" { - return errors.New("local mixin URI cannot be empty") + return fmt.Errorf("%w", ErrLocalMixinURIEmpty) } - return errors.New("local mixin installation not implemented") + return fmt.Errorf("%w", ErrLocalMixinNotImplemented) default: return fmt.Errorf("%w", ErrUnknownPackageType) } @@ -216,7 +222,7 @@ func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) } if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { - return fmt.Errorf("%w: %v", ErrCopyPackage, err) + return fmt.Errorf(wrapErrFmt, ErrCopyPackage, err) } return nil From 3f889107cf7b3645883e06815d969f5b095dba0e Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Mar 2025 20:36:59 +0600 Subject: [PATCH 55/68] more linter feedback for vendor model and vendor model component --- .golangci.yml | 4 ++-- internal/exec/vendor_model.go | 32 ++++++++++++++++++++++---------- internal/exec/vendor_utils.go | 2 +- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 1034b9987..15a006bd5 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -67,7 +67,7 @@ linters-settings: gocognit: # Maximum cognitive complexity - min-complexity: 20 + min-complexity: 23 godot: # Check if comments end in a period @@ -158,7 +158,7 @@ linters-settings: nestif: # Minimal complexity of if statements to report. - min-complexity: 4 + min-complexity: 6 issues: max-issues-per-linter: 0 diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 72c8777a5..45c5ee931 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -12,13 +12,18 @@ import ( "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + log "github.com/charmbracelet/log" "github.com/hashicorp/go-getter" cp "github.com/otiai10/copy" "github.com/cloudposse/atmos/internal/tui/templates/term" "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/ui/theme" - u "github.com/cloudposse/atmos/pkg/utils" +) + +const ( + wrapErrFmt = "%w" + tempDirPermissions os.FileMode = 0755 ) type pkgType int @@ -76,7 +81,7 @@ var ( grayColor = theme.Styles.GrayText ) -func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) (modelVendor, error) { +func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) (modelVendor, error) { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(progressWidth), @@ -102,7 +107,7 @@ func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig spinner: s, progress: p, dryRun: dryRun, - atmosConfig: atmosConfig, + atmosConfig: *atmosConfig, isTTY: isTTY, }, nil } @@ -140,7 +145,7 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if msg.err != nil { errMsg = fmt.Sprintf("Failed to vendor %s: error : %s", pkg.name, msg.err) if !m.isTTY { - u.LogError(errors.New(errMsg)) + log.Error(errMsg) } mark = xMark m.failedPkg++ @@ -152,14 +157,14 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if m.index >= len(m.packages)-1 { m.done = true if !m.isTTY { - u.LogInfo(fmt.Sprintf("%s %s %s", mark, pkg.name, version)) + log.Info(fmt.Sprintf("%s %s %s", mark, pkg.name, version)) if m.dryRun { - u.LogInfo("Done! Dry run completed. No components vendored.\n") + log.Info("Done! Dry run completed. No components vendored.\n") } if m.failedPkg > 0 { - u.LogInfo(fmt.Sprintf("Vendored %d components. Failed to vendor %d components.\n", len(m.packages)-m.failedPkg, m.failedPkg)) + log.Info(fmt.Sprintf("Vendored %d components. Failed to vendor %d components.\n", len(m.packages)-m.failedPkg, m.failedPkg)) } - u.LogInfo(fmt.Sprintf("Vendored %d components.\n", len(m.packages))) + log.Info(fmt.Sprintf("Vendored %d components.\n", len(m.packages))) } version := grayColor.Render(version) return m, tea.Sequence( @@ -168,7 +173,7 @@ func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ) } if !m.isTTY { - u.LogInfo(fmt.Sprintf("%s %s %s", mark, pkg.name, version)) + log.Info(fmt.Sprintf("%s %s %s", mark, pkg.name, version)) } m.index++ progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) @@ -247,12 +252,14 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos tempDir, err := os.MkdirTemp("", "atmos-vendor") if err != nil { return installedPkgMsg{ + //nolint:err113 err: fmt.Errorf(wrapErrFmt, errors.New("failed to create temp directory"), err), name: p.name, } } if err := os.Chmod(tempDir, tempDirPermissions); err != nil { return installedPkgMsg{ + //nolint:err113 err: fmt.Errorf(wrapErrFmt, errors.New("failed to set temp directory permissions"), err), name: p.name, } @@ -264,6 +271,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos case pkgTypeRemote: if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { return installedPkgMsg{ + //nolint:err113 err: fmt.Errorf(wrapErrFmt, errors.New("failed to download package"), err), name: p.name, } @@ -272,6 +280,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos case pkgTypeOci: if err := processOciImage(atmosConfig, p.uri, tempDir); err != nil { return installedPkgMsg{ + //nolint:err113 err: fmt.Errorf(wrapErrFmt, errors.New("failed to process OCI image"), err), name: p.name, } @@ -288,18 +297,20 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { return installedPkgMsg{ + //nolint:err113 err: fmt.Errorf(wrapErrFmt, errors.New("failed to copy package"), err), name: p.name, } } default: return installedPkgMsg{ - err: fmt.Errorf("%w", ErrUnknownPackageType), + err: fmt.Errorf(wrapErrFmt, ErrUnknownPackageType), name: p.name, } } if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ + //nolint:err113 err: fmt.Errorf(wrapErrFmt, errors.New("failed to copy package"), err), name: p.name, } @@ -321,6 +332,7 @@ func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig schema.AtmosCo } return func() tea.Msg { + //nolint:err113 err := fmt.Errorf("no valid installer package provided for %s", installer.name) return installedPkgMsg{ err: err, diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 63abbc59c..c501a8ae2 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -424,7 +424,7 @@ func ExecuteAtmosVendorInternal( u.LogWarning("No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") } - model, err := newModelAtmosVendorInternal(packages, dryRun, atmosConfig) + model, err := newModelAtmosVendorInternal(packages, dryRun, &atmosConfig) if err != nil { return fmt.Errorf("failed to initialize TUI model: %v (verify terminal capabilities and permissions)", err) } From 4bd24fdc92a51581b86c57e913a46046332a2878 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:38:02 +0000 Subject: [PATCH 56/68] [autofix.ci] apply automated fixes --- internal/exec/vendor_model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 45c5ee931..ee081dbf5 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -23,7 +23,7 @@ import ( const ( wrapErrFmt = "%w" - tempDirPermissions os.FileMode = 0755 + tempDirPermissions os.FileMode = 0o755 ) type pkgType int From 79ab2a64bc6699032f0953060be0afc5d960deb3 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Thu, 6 Mar 2025 20:43:59 +0600 Subject: [PATCH 57/68] more linter fixes --- internal/exec/vendor_model_component.go | 46 +++++++++++-------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index c0949e71a..a1c791f63 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -1,4 +1,3 @@ -// vendor_model_component.go package exec import ( @@ -21,21 +20,16 @@ import ( ) const ( - progressWidth = 30 - getterTimeout = 10 * time.Minute - tempDirPermissions = 0o700 - wrapErrFmt = "%w: %v" - base10 = 10 + progressWidth = 30 + getterTimeout = 10 * time.Minute ) var ( - ErrDownloadPackage = errors.New("failed to download package") - ErrProcessOCIImage = errors.New("failed to process OCI image") - ErrCopyPackage = errors.New("failed to copy package") - ErrCreateTempDir = errors.New("failed to create temp directory") - ErrUnknownPackageType = errors.New("unknown package type") - ErrLocalMixinURIEmpty = errors.New("local mixin URI cannot be empty") - ErrLocalMixinNotImplemented = errors.New("local mixin installation not implemented") + ErrDownloadPackage = errors.New("failed to download package") + ErrProcessOCIImage = errors.New("failed to process OCI image") + ErrCopyPackage = errors.New("failed to copy package") + ErrCreateTempDir = errors.New("failed to create temp directory") + ErrUnknownPackageType = errors.New("unknown package type") ) // pkgComponentVendor defines a vendor package. @@ -138,10 +132,10 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix())) if err != nil { - return fmt.Errorf(wrapErrFmt, ErrCreateTempDir, err) + return fmt.Errorf("%w: %v", ErrCreateTempDir, err) } - if err := os.Chmod(tempDir, tempDirPermissions); err != nil { + if err := os.Chmod(tempDir, 0o700); err != nil { return fmt.Errorf("failed to set temp directory permissions: %w", err) } @@ -151,12 +145,12 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat case pkgTypeRemote: tempDir = filepath.Join(tempDir, SanitizeFileName(p.uri)) if err = GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { - return fmt.Errorf(wrapErrFmt, ErrDownloadPackage, err) + return fmt.Errorf("%w: %v", ErrDownloadPackage, err) } case pkgTypeOci: err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf(wrapErrFmt, ErrProcessOCIImage, err) + return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) } case pkgTypeLocal: copyOptions := cp.Options{ @@ -173,13 +167,13 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat } if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { - return fmt.Errorf(wrapErrFmt, ErrCopyPackage, err) + return fmt.Errorf("%w: %v", ErrCopyPackage, err) } default: return fmt.Errorf("%w", ErrUnknownPackageType) } if err = copyComponentToDestination(*atmosConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { - return fmt.Errorf(wrapErrFmt, ErrCopyPackage, err) + return fmt.Errorf("%w: %v", ErrCopyPackage, err) } return nil @@ -187,9 +181,9 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat // installMixin downloads and installs a mixin. func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), base10)) + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { - return fmt.Errorf(wrapErrFmt, ErrCreateTempDir, err) + return fmt.Errorf("%w: %v", ErrCreateTempDir, err) } defer removeTempDir(*atmosConfig, tempDir) @@ -197,18 +191,18 @@ func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) switch p.pkgType { case pkgTypeRemote: if err = GoGetterGet(atmosConfig, p.uri, filepath.Join(tempDir, p.mixinFilename), getter.ClientModeFile, getterTimeout); err != nil { - return fmt.Errorf(wrapErrFmt, ErrDownloadPackage, err) + return fmt.Errorf("%w: %v", ErrDownloadPackage, err) } case pkgTypeOci: err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf(wrapErrFmt, ErrProcessOCIImage, err) + return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) } case pkgTypeLocal: if p.uri == "" { - return fmt.Errorf("%w", ErrLocalMixinURIEmpty) + return errors.New("local mixin URI cannot be empty") } - return fmt.Errorf("%w", ErrLocalMixinNotImplemented) + return errors.New("local mixin installation not implemented") default: return fmt.Errorf("%w", ErrUnknownPackageType) } @@ -222,7 +216,7 @@ func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) } if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { - return fmt.Errorf(wrapErrFmt, ErrCopyPackage, err) + return fmt.Errorf("%w: %v", ErrCopyPackage, err) } return nil From 564e0d71764ca1adcc013ce55dd6e52042127670 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Mar 2025 00:55:09 +0600 Subject: [PATCH 58/68] Refactored error handling, replaced deprecated logging, and removed unnecessary error return --- .golangci.yml | 2 +- internal/exec/error.go | 27 +++++++++++++++++ internal/exec/vendor_model.go | 26 ++++++---------- internal/exec/vendor_model_component.go | 40 ++++++++----------------- internal/exec/vendor_utils.go | 7 +++-- 5 files changed, 54 insertions(+), 48 deletions(-) create mode 100644 internal/exec/error.go diff --git a/.golangci.yml b/.golangci.yml index 15a006bd5..bee740c27 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -96,7 +96,7 @@ linters-settings: skipComments: true skipBlankLines: true - name: function-length - arguments: [50, 60] # Max lines per function + arguments: [50, 80] # Max lines per function - name: cognitive-complexity arguments: [25] # Max cognitive complexity - name: cyclomatic diff --git a/internal/exec/error.go b/internal/exec/error.go new file mode 100644 index 000000000..e1133aa40 --- /dev/null +++ b/internal/exec/error.go @@ -0,0 +1,27 @@ +package exec + +import ( + "errors" + "time" +) + +const ( + progressWidth = 30 + getterTimeout = 10 * time.Minute + componentTempDirPermissions = 0o700 + wrapErrFmtWithDetails = "%w: %v" + timeFormatBase = 10 +) + +var ( + ErrDownloadPackage = errors.New("failed to download package") + ErrProcessOCIImage = errors.New("failed to process OCI image") + ErrCopyPackage = errors.New("failed to copy package") + ErrCreateTempDir = errors.New("failed to create temp directory") + ErrUnknownPackageType = errors.New("unknown package type") + ErrLocalMixinURICannotBeEmpty = errors.New("local mixin URI cannot be empty") + ErrLocalMixinInstallationNotImplemented = errors.New("local mixin installation not implemented") + ErrFailedToInitializeTUIModel = errors.New("failed to initialize TUI model: verify terminal capabilities and permissions") + ErrSetTempDirPermissions = errors.New("failed to set temp directory permissions") + ErrCopyPackageToTarget = errors.New("failed to copy package to target") +) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index ee081dbf5..d9e95edd8 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -1,7 +1,6 @@ package exec import ( - "errors" "fmt" "os" "path/filepath" @@ -81,7 +80,7 @@ var ( grayColor = theme.Styles.GrayText ) -func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) (modelVendor, error) { +func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) modelVendor { p := progress.New( progress.WithDefaultGradient(), progress.WithWidth(progressWidth), @@ -90,7 +89,7 @@ func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig s := spinner.New() s.Style = theme.Styles.Link if len(pkgs) == 0 { - return modelVendor{done: true}, nil + return modelVendor{done: true} } isTTY := term.IsTTYSupportForStdout() var vendorPks []pkgVendor @@ -109,7 +108,7 @@ func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig dryRun: dryRun, atmosConfig: *atmosConfig, isTTY: isTTY, - }, nil + } } func (m *modelVendor) Init() tea.Cmd { @@ -252,15 +251,13 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos tempDir, err := os.MkdirTemp("", "atmos-vendor") if err != nil { return installedPkgMsg{ - //nolint:err113 - err: fmt.Errorf(wrapErrFmt, errors.New("failed to create temp directory"), err), + err: fmt.Errorf(wrapErrFmt, ErrCreateTempDir, err), name: p.name, } } if err := os.Chmod(tempDir, tempDirPermissions); err != nil { return installedPkgMsg{ - //nolint:err113 - err: fmt.Errorf(wrapErrFmt, errors.New("failed to set temp directory permissions"), err), + err: fmt.Errorf(wrapErrFmt, ErrSetTempDirPermissions, err), name: p.name, } } @@ -271,8 +268,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos case pkgTypeRemote: if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { return installedPkgMsg{ - //nolint:err113 - err: fmt.Errorf(wrapErrFmt, errors.New("failed to download package"), err), + err: fmt.Errorf(wrapErrFmt, ErrDownloadPackage, err), name: p.name, } } @@ -280,8 +276,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos case pkgTypeOci: if err := processOciImage(atmosConfig, p.uri, tempDir); err != nil { return installedPkgMsg{ - //nolint:err113 - err: fmt.Errorf(wrapErrFmt, errors.New("failed to process OCI image"), err), + err: fmt.Errorf(wrapErrFmt, ErrProcessOCIImage, err), name: p.name, } } @@ -297,8 +292,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { return installedPkgMsg{ - //nolint:err113 - err: fmt.Errorf(wrapErrFmt, errors.New("failed to copy package"), err), + err: fmt.Errorf(wrapErrFmt, ErrCopyPackage, err), name: p.name, } } @@ -310,8 +304,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ - //nolint:err113 - err: fmt.Errorf(wrapErrFmt, errors.New("failed to copy package"), err), + err: fmt.Errorf(wrapErrFmt, ErrCopyPackageToTarget, err), name: p.name, } } @@ -332,7 +325,6 @@ func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig schema.AtmosCo } return func() tea.Msg { - //nolint:err113 err := fmt.Errorf("no valid installer package provided for %s", installer.name) return installedPkgMsg{ err: err, diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index a1c791f63..c9481c2c3 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -1,7 +1,6 @@ package exec import ( - "errors" "fmt" "os" "path/filepath" @@ -19,19 +18,6 @@ import ( "github.com/cloudposse/atmos/pkg/ui/theme" ) -const ( - progressWidth = 30 - getterTimeout = 10 * time.Minute -) - -var ( - ErrDownloadPackage = errors.New("failed to download package") - ErrProcessOCIImage = errors.New("failed to process OCI image") - ErrCopyPackage = errors.New("failed to copy package") - ErrCreateTempDir = errors.New("failed to create temp directory") - ErrUnknownPackageType = errors.New("unknown package type") -) - // pkgComponentVendor defines a vendor package. type pkgComponentVendor struct { uri string @@ -132,10 +118,10 @@ func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, atmosConfig func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix())) if err != nil { - return fmt.Errorf("%w: %v", ErrCreateTempDir, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrCreateTempDir, err) } - if err := os.Chmod(tempDir, 0o700); err != nil { + if err := os.Chmod(tempDir, componentTempDirPermissions); err != nil { return fmt.Errorf("failed to set temp directory permissions: %w", err) } @@ -145,12 +131,12 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat case pkgTypeRemote: tempDir = filepath.Join(tempDir, SanitizeFileName(p.uri)) if err = GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { - return fmt.Errorf("%w: %v", ErrDownloadPackage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrDownloadPackage, err) } case pkgTypeOci: err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrProcessOCIImage, err) } case pkgTypeLocal: copyOptions := cp.Options{ @@ -167,13 +153,13 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat } if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { - return fmt.Errorf("%w: %v", ErrCopyPackage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrCopyPackage, err) } default: return fmt.Errorf("%w", ErrUnknownPackageType) } if err = copyComponentToDestination(*atmosConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { - return fmt.Errorf("%w: %v", ErrCopyPackage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrCopyPackage, err) } return nil @@ -181,9 +167,9 @@ func installComponent(p *pkgComponentVendor, atmosConfig *schema.AtmosConfigurat // installMixin downloads and installs a mixin. func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) error { - tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), timeFormatBase)) if err != nil { - return fmt.Errorf("%w: %v", ErrCreateTempDir, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrCreateTempDir, err) } defer removeTempDir(*atmosConfig, tempDir) @@ -191,18 +177,18 @@ func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) switch p.pkgType { case pkgTypeRemote: if err = GoGetterGet(atmosConfig, p.uri, filepath.Join(tempDir, p.mixinFilename), getter.ClientModeFile, getterTimeout); err != nil { - return fmt.Errorf("%w: %v", ErrDownloadPackage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrDownloadPackage, err) } case pkgTypeOci: err = processOciImage(*atmosConfig, p.uri, tempDir) if err != nil { - return fmt.Errorf("%w: %v", ErrProcessOCIImage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrProcessOCIImage, err) } case pkgTypeLocal: if p.uri == "" { - return errors.New("local mixin URI cannot be empty") + return ErrLocalMixinURICannotBeEmpty } - return errors.New("local mixin installation not implemented") + return ErrLocalMixinInstallationNotImplemented default: return fmt.Errorf("%w", ErrUnknownPackageType) } @@ -216,7 +202,7 @@ func installMixin(p *pkgComponentVendor, atmosConfig *schema.AtmosConfiguration) } if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { - return fmt.Errorf("%w: %v", ErrCopyPackage, err) + return fmt.Errorf(wrapErrFmtWithDetails, ErrCopyPackage, err) } return nil diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index c501a8ae2..8c1198860 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v3" + log "github.com/charmbracelet/log" "github.com/cloudposse/atmos/internal/tui/templates/term" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" @@ -171,7 +172,7 @@ func ReadAndProcessVendorConfigFile( if !fileExists { vendorConfigFileExists = false - u.LogWarning(fmt.Sprintf("Vendor config file '%s' does not exist. Proceeding without vendor configurations", pathToVendorConfig)) + log.Warn(fmt.Sprintf("Vendor config file '%s' does not exist. Proceeding without vendor configurations", pathToVendorConfig)) return vendorConfig, vendorConfigFileExists, "", nil } } @@ -421,10 +422,10 @@ func ExecuteAtmosVendorInternal( if !term.IsTTYSupportForStdout() { // set tea.WithInput(nil) workaround tea program not run on not TTY mod issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} - u.LogWarning("No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") + log.Warn("No TTY detected. Falling back to basic output. This can happen when no terminal is attached or when commands are pipelined.") } - model, err := newModelAtmosVendorInternal(packages, dryRun, &atmosConfig) + model := newModelAtmosVendorInternal(packages, dryRun, &atmosConfig) if err != nil { return fmt.Errorf("failed to initialize TUI model: %v (verify terminal capabilities and permissions)", err) } From c8648389651ba5889540d72e76e8fe50ba176f1b Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Mar 2025 01:13:12 +0600 Subject: [PATCH 59/68] moving more errors to errors.go and fixing other linter issues --- .golangci.yml | 2 +- internal/exec/vendor_model.go | 20 +++++++++++--------- internal/exec/vendor_utils.go | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index bee740c27..373421db1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,7 +37,7 @@ linters: linters-settings: funlen: - lines: 60 # Maximum number of lines per function + lines: 75 # Maximum number of lines per function statements: 40 # Maximum number of statements per function ignore-comments: true # Ignore comments when counting lines. diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index d9e95edd8..e9d177911 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -21,7 +21,8 @@ import ( ) const ( - wrapErrFmt = "%w" + // wrapErrFmt is defined in error.go as wrapErrFmtWithDetails ("%w: %v") + // Use the detailed formatter to support two arguments tempDirPermissions os.FileMode = 0o755 ) @@ -251,13 +252,13 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos tempDir, err := os.MkdirTemp("", "atmos-vendor") if err != nil { return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrCreateTempDir, err), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrCreateTempDir, err), name: p.name, } } if err := os.Chmod(tempDir, tempDirPermissions); err != nil { return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrSetTempDirPermissions, err), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrSetTempDirPermissions, err), name: p.name, } } @@ -268,7 +269,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos case pkgTypeRemote: if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrDownloadPackage, err), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrDownloadPackage, err), name: p.name, } } @@ -276,7 +277,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos case pkgTypeOci: if err := processOciImage(atmosConfig, p.uri, tempDir); err != nil { return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrProcessOCIImage, err), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrProcessOCIImage, err), name: p.name, } } @@ -292,19 +293,19 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrCopyPackage, err), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrCopyPackage, err), name: p.name, } } default: return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrUnknownPackageType), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrUnknownPackageType, nil), name: p.name, } } if err := copyToTargetWithPatterns(tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { return installedPkgMsg{ - err: fmt.Errorf(wrapErrFmt, ErrCopyPackageToTarget, err), + err: fmt.Errorf(wrapErrFmtWithDetails, ErrCopyPackageToTarget, err), name: p.name, } } @@ -325,7 +326,8 @@ func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig schema.AtmosCo } return func() tea.Msg { - err := fmt.Errorf("no valid installer package provided for %s", installer.name) + // Use only the static error wrapping without dynamic insertion. + err := fmt.Errorf("%w", ErrNoValidInstallerPackage) return installedPkgMsg{ err: err, name: installer.name, diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index 8c1198860..adc6cd8ca 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -427,7 +427,7 @@ func ExecuteAtmosVendorInternal( model := newModelAtmosVendorInternal(packages, dryRun, &atmosConfig) if err != nil { - return fmt.Errorf("failed to initialize TUI model: %v (verify terminal capabilities and permissions)", err) + return fmt.Errorf("%w: %v", ErrFailedToInitializeTUIModelWithDetails, err) } if _, err := tea.NewProgram(&model, opts...).Run(); err != nil { return fmt.Errorf("failed to execute vendor operation in TUI mode: %w (check terminal state)", err) From c7764b8f34aa842b176057694a77d2d3c28eb5a7 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Mar 2025 01:21:42 +0600 Subject: [PATCH 60/68] updated errors.go --- internal/exec/error.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/internal/exec/error.go b/internal/exec/error.go index e1133aa40..37c349603 100644 --- a/internal/exec/error.go +++ b/internal/exec/error.go @@ -14,14 +14,16 @@ const ( ) var ( - ErrDownloadPackage = errors.New("failed to download package") - ErrProcessOCIImage = errors.New("failed to process OCI image") - ErrCopyPackage = errors.New("failed to copy package") - ErrCreateTempDir = errors.New("failed to create temp directory") - ErrUnknownPackageType = errors.New("unknown package type") - ErrLocalMixinURICannotBeEmpty = errors.New("local mixin URI cannot be empty") - ErrLocalMixinInstallationNotImplemented = errors.New("local mixin installation not implemented") - ErrFailedToInitializeTUIModel = errors.New("failed to initialize TUI model: verify terminal capabilities and permissions") - ErrSetTempDirPermissions = errors.New("failed to set temp directory permissions") - ErrCopyPackageToTarget = errors.New("failed to copy package to target") + ErrDownloadPackage = errors.New("failed to download package") + ErrProcessOCIImage = errors.New("failed to process OCI image") + ErrCopyPackage = errors.New("failed to copy package") + ErrCreateTempDir = errors.New("failed to create temp directory") + ErrUnknownPackageType = errors.New("unknown package type") + ErrLocalMixinURICannotBeEmpty = errors.New("local mixin URI cannot be empty") + ErrLocalMixinInstallationNotImplemented = errors.New("local mixin installation not implemented") + ErrFailedToInitializeTUIModel = errors.New("failed to initialize TUI model: verify terminal capabilities and permissions") + ErrSetTempDirPermissions = errors.New("failed to set temp directory permissions") + ErrCopyPackageToTarget = errors.New("failed to copy package to target") + ErrNoValidInstallerPackage = errors.New("no valid installer package provided") + ErrFailedToInitializeTUIModelWithDetails = errors.New("failed to initialize TUI model: verify terminal capabilities and permissions") ) From 225b0f18c097ac34783e78bb79f717b112e66819 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Mar 2025 02:02:27 +0600 Subject: [PATCH 61/68] atmosConfig is no longer passed by value in vendor model --- internal/exec/vendor_model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index e9d177911..8abeb3619 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -21,7 +21,7 @@ import ( ) const ( - // wrapErrFmt is defined in error.go as wrapErrFmtWithDetails ("%w: %v") + // WrapErrFmt is defined in error.go as wrapErrFmtWithDetails ("%w: %v") // Use the detailed formatter to support two arguments tempDirPermissions os.FileMode = 0o755 ) From 2b044824bf9c9c63f809740294eaa58deef4d7f2 Mon Sep 17 00:00:00 2001 From: Andrey Kalmykov Date: Fri, 7 Mar 2025 02:33:14 +0600 Subject: [PATCH 62/68] More atmosConfig references --- internal/exec/vendor_model.go | 20 ++++++++++---------- internal/exec/vendor_model_component.go | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go index 8abeb3619..3d9c09e04 100644 --- a/internal/exec/vendor_model.go +++ b/internal/exec/vendor_model.go @@ -21,8 +21,8 @@ import ( ) const ( - // WrapErrFmt is defined in error.go as wrapErrFmtWithDetails ("%w: %v") - // Use the detailed formatter to support two arguments + // WrapErrFmt is defined in error.go as wrapErrFmtWithDetails ("%w: %v"). + // Use the detailed formatter to support two arguments. tempDirPermissions os.FileMode = 0o755 ) @@ -69,7 +69,7 @@ type modelVendor struct { done bool dryRun bool failedPkg int - atmosConfig schema.AtmosConfiguration + atmosConfig *schema.AtmosConfiguration isTTY bool } @@ -107,7 +107,7 @@ func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, atmosConfig spinner: s, progress: p, dryRun: dryRun, - atmosConfig: *atmosConfig, + atmosConfig: atmosConfig, isTTY: isTTY, } } @@ -240,7 +240,7 @@ func max(a, b int) int { return b } -func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) tea.Cmd { +func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) tea.Cmd { return func() tea.Msg { if dryRun { time.Sleep(500 * time.Millisecond) @@ -263,11 +263,11 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } - defer removeTempDir(atmosConfig, tempDir) + defer removeTempDir(*atmosConfig, tempDir) switch p.pkgType { case pkgTypeRemote: - if err := GoGetterGet(&atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { + if err := GoGetterGet(atmosConfig, p.uri, tempDir, getter.ClientModeAny, getterTimeout); err != nil { return installedPkgMsg{ err: fmt.Errorf(wrapErrFmtWithDetails, ErrDownloadPackage, err), name: p.name, @@ -275,7 +275,7 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } case pkgTypeOci: - if err := processOciImage(atmosConfig, p.uri, tempDir); err != nil { + if err := processOciImage(*atmosConfig, p.uri, tempDir); err != nil { return installedPkgMsg{ err: fmt.Errorf(wrapErrFmtWithDetails, ErrProcessOCIImage, err), name: p.name, @@ -316,13 +316,13 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos } } -func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig schema.AtmosConfiguration) tea.Cmd { +func ExecuteInstall(installer pkgVendor, dryRun bool, atmosConfig *schema.AtmosConfiguration) tea.Cmd { if installer.atmosPackage != nil { return downloadAndInstall(installer.atmosPackage, dryRun, atmosConfig) } if installer.componentPackage != nil { - return downloadComponentAndInstall(installer.componentPackage, dryRun, &atmosConfig) + return downloadComponentAndInstall(installer.componentPackage, dryRun, atmosConfig) } return func() tea.Msg { diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go index c9481c2c3..0a4bea533 100644 --- a/internal/exec/vendor_model_component.go +++ b/internal/exec/vendor_model_component.go @@ -47,7 +47,7 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm spinner: s, progress: p, dryRun: dryRun, - atmosConfig: *atmosConfig, + atmosConfig: atmosConfig, isTTY: term.IsTTYSupportForStdout(), } } @@ -66,7 +66,7 @@ func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, atm spinner: s, progress: p, dryRun: dryRun, - atmosConfig: *atmosConfig, + atmosConfig: atmosConfig, isTTY: tty, } } From 84cd064232071fb52ab72165c5e0ee68448b5a62 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:36:32 +0600 Subject: [PATCH 63/68] Update .golangci.yml Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index ad86751a4..3484f8c50 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -160,7 +160,7 @@ linters-settings: nestif: # Minimal complexity of if statements to report. - min-complexity: 6 + min-complexity: 4 severity: default-severity: error From d3338558bc4db3bd0516d12b3922e460ba9df627 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:36:52 +0600 Subject: [PATCH 64/68] Update .golangci.yml Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 3484f8c50..20377fe39 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,7 +37,7 @@ linters: linters-settings: funlen: - lines: 75 # Maximum number of lines per function + lines: 60 # Maximum number of lines per function statements: 40 # Maximum number of statements per function ignore-comments: true # Ignore comments when counting lines. From 037997500b0a7d1cd8e52e1d9d5c6670b69943be Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:37:13 +0600 Subject: [PATCH 65/68] Update .golangci.yml Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 20377fe39..bbd2b0774 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -67,7 +67,7 @@ linters-settings: gocognit: # Maximum cognitive complexity - min-complexity: 23 + min-complexity: 20 godot: # Check if comments end in a period From c29008b7a390d455ab65c8cdf8733f5c7a7eec45 Mon Sep 17 00:00:00 2001 From: Listener430 <95291462+Listener430@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:37:29 +0600 Subject: [PATCH 66/68] Update .golangci.yml Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index bbd2b0774..0fff7e7a4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -102,7 +102,7 @@ linters-settings: - name: cognitive-complexity arguments: [25] # Max cognitive complexity - name: cyclomatic - arguments: [13] # Max cyclomatic complexity + arguments: [10] # Max cyclomatic complexity - name: add-constant arguments: - maxLitCount: "3" From 047df91c5e0db87523aff0d6af087d3dcec0466f Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 12 Mar 2025 19:47:35 +0000 Subject: [PATCH 67/68] [autofix.ci] apply automated fixes --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 60809383e..d7bd78885 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,6 @@ require ( github.com/a8m/envsubst v1.4.2 // indirect github.com/agext/levenshtein v1.2.2 // indirect github.com/agiledragon/gomonkey/v2 v2.13.0 - github.com/agnivade/levenshtein v1.2.0 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 0d0da0a6b..60e71c33b 100644 --- a/go.sum +++ b/go.sum @@ -684,8 +684,6 @@ github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXva github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agiledragon/gomonkey/v2 v2.13.0 h1:B24Jg6wBI1iB8EFR1c+/aoTg7QN/Cum7YffG8KMIyYo= github.com/agiledragon/gomonkey/v2 v2.13.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= -github.com/agnivade/levenshtein v1.2.0 h1:U9L4IOT0Y3i0TIlUIDJ7rVUziKi/zPbrJGaFrtYH3SY= -github.com/agnivade/levenshtein v1.2.0/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= From 08c9f3707b7485c0061262ab9a57d5aa1c4e7f1e Mon Sep 17 00:00:00 2001 From: "Erik Osterman (CEO @ Cloud Posse)" Date: Wed, 12 Mar 2025 21:27:31 -0500 Subject: [PATCH 68/68] Apply suggestions from code review --- internal/exec/go_getter_utils.go | 2 +- .../core-concepts/vendor/vendor-manifest.mdx | 43 ++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/internal/exec/go_getter_utils.go b/internal/exec/go_getter_utils.go index 3b0ca380d..fc4f58f6d 100644 --- a/internal/exec/go_getter_utils.go +++ b/internal/exec/go_getter_utils.go @@ -116,7 +116,7 @@ func (d *CustomGitDetector) Detect(src, _ string) (string, bool, error) { if err != nil { log.Debug("Masking failed", "error", err) } else { - log.Debug("Final URL", "final_url", "git::"+maskedFinal) + log.Debug("Final URL (masked)", "url", "git::"+maskedFinal) } return finalURL, true, nil diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 9de844a01..0fe114f2e 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -500,14 +500,37 @@ atmos vendor pull -c vpc ## Vendoring with Globs -When defining vendoring rules in Atmos, **glob patterns** determine which files and directories are included or excluded. Understanding how globs behave—especially when using greedy (`**`) versus non-greedy (`*`) patterns—is crucial for precise vendoring. +In Atmos, **glob patterns** define which files and directories are included or excluded during vendoring. These patterns go beyond simple wildcard characters like `*`—they follow specific rules that dictate how paths are matched. Understanding the difference between **greedy** (`**`) and **non-greedy** (`*`) patterns, along with other advanced glob syntax, ensures precise control over vendoring behavior. -### Understanding Greedy and Non-Greedy Globs +### Understanding Wildcards, Ranges, and Recursion -Globs use special wildcard characters: +Glob patterns in Atmos provide flexible and powerful matching, that's simpler to understand than regular expressions: -- `*` (single asterisk) matches any sequence of characters **within a single path segment**. -- `**` (double asterisk) matches across multiple path segments **recursively**. +
+
`*` (single asterisk)
+
Matches any sequence of characters within a single path segment.
+
Example: `vendor/*.yaml` matches `vendor/config.yaml` but not `vendor/subdir/config.yaml`.
+ +
`**` (double asterisk, also known as a "greedy glob")
+
Matches across multiple path segments recursively.
+
Example: `vendor/**/*.yaml` matches `vendor/config.yaml`, `vendor/subdir/config.yaml`, and `vendor/deep/nested/config.yaml`.
+ +
`?` (question mark)
+
Matches exactly one character in a path segment.
+
Example: `file?.txt` matches `file1.txt` and `fileA.txt` but not `file10.txt`.
+ +
`[abc]` (character class)
+
Matches any single character inside the brackets.
+
Example: `file[123].txt` matches `file1.txt`, `file2.txt`, and `file3.txt`, but not `file4.txt` or `file12.txt`.
+ +
`[a-z]` (character range)
+
Matches any single character within the specified range.
+
Example: `file[a-c].txt` matches `filea.txt`, `fileb.txt`, and `filec.txt`.
+ +
`{a,b,c}` (brace expansion)
+
Matches any of the comma-separated patterns.
+
Example: `*.{jpg,png,gif}` matches `image.jpg`, `image.png`, and `image.gif`.
+
This distinction is important when excluding specific directories or files while vendoring. @@ -522,12 +545,12 @@ excluded_paths: - "**/demo-library/**/stargazers/**" ``` -How It Works: -- The included_paths rule `**/demo-library/**` ensures all files inside `demo-library` (at any depth) are vendored. -- The excluded_paths rule `**/demo-library/**/stargazers/**` prevents any files inside `stargazers` subdirectories from being vendored. +How it works: +- The `included_paths` rule `**/demo-library/**` ensures all files inside `demo-library` (at any depth) are vendored. +- The `excluded_paths` rule `**/demo-library/**/stargazers/**` prevents any files inside `stargazers` subdirectories from being vendored. This means: -- All files within demo-library except those inside any `stargazers` subdirectory are vendored. +- All files within `demo-library` except those inside any `stargazers` subdirectory are vendored. - Any other files outside `stargazers` are unaffected by this exclusion. #### Example: A Non-Recursive Pattern That Doesn't Work @@ -540,7 +563,7 @@ excluded_paths: ``` In this case: -- `**/demo-library/*` only matches immediate children of demo-library, not nested files or subdirectories. +- `**/demo-library/*` only matches immediate children of `demo-library`, not nested files or subdirectories. - This means `stargazers/` itself could be matched, but its contents might not be explicitly excluded. - To correctly capture all subdirectories and files while still excluding stargazers, use `**/demo-library/**/*`.