diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 3ffb95dd29..f3b1a20c04 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -212,7 +212,7 @@ func setupAutoProviderCacheDir(ctx context.Context, l log.Logger, opts *options. // Check if OpenTofu is being used if tfImplementation != options.OpenTofuImpl { - return fmt.Errorf("auto provider cache dir requires OpenTofu, but detected %s", tfImplementation) + return errors.Errorf("auto provider cache dir requires OpenTofu, but detected %s", tfImplementation) } // Check OpenTofu version > 1.10 @@ -222,11 +222,11 @@ func setupAutoProviderCacheDir(ctx context.Context, l log.Logger, opts *options. requiredVersion, err := version.NewVersion(minTofuVersionForAutoProviderCacheDir) if err != nil { - return fmt.Errorf("failed to parse required version: %w", err) + return errors.Errorf("failed to parse required version: %w", err) } if terraformVersion.LessThan(requiredVersion) { - return fmt.Errorf("auto provider cache dir requires OpenTofu version >= 1.10, but found %s", terraformVersion) + return errors.Errorf("auto provider cache dir requires OpenTofu version >= 1.10, but found %s", terraformVersion) } // Set up the provider cache directory @@ -234,7 +234,7 @@ func setupAutoProviderCacheDir(ctx context.Context, l log.Logger, opts *options. if providerCacheDir == "" { cacheDir, err := util.GetCacheDir() if err != nil { - return fmt.Errorf("failed to get cache directory: %w", err) + return errors.Errorf("failed to get cache directory: %w", err) } providerCacheDir = filepath.Join(cacheDir, "providers") @@ -244,7 +244,7 @@ func setupAutoProviderCacheDir(ctx context.Context, l log.Logger, opts *options. if !filepath.IsAbs(providerCacheDir) { absPath, err := filepath.Abs(providerCacheDir) if err != nil { - return fmt.Errorf("failed to get absolute path for provider cache directory: %w", err) + return errors.Errorf("failed to get absolute path for provider cache directory: %w", err) } providerCacheDir = absPath @@ -254,7 +254,7 @@ func setupAutoProviderCacheDir(ctx context.Context, l log.Logger, opts *options. // Create the cache directory if it doesn't exist if err := os.MkdirAll(providerCacheDir, cacheDirMode); err != nil { - return fmt.Errorf("failed to create provider cache directory: %w", err) + return errors.Errorf("failed to create provider cache directory: %w", err) } // Initialize environment variables map if it's nil @@ -389,7 +389,7 @@ func initialSetup(cliCtx *cli.Context, l log.Logger, opts *options.TerragruntOpt if !doubleStarEnabled { opts.IncludeDirs, err = util.GlobCanonicalPath(l, opts.WorkingDir, opts.IncludeDirs...) if err != nil { - return fmt.Errorf("invalid include dirs: %w", err) + return errors.Errorf("invalid include dirs: %w", err) } } @@ -406,7 +406,7 @@ func initialSetup(cliCtx *cli.Context, l log.Logger, opts *options.TerragruntOpt if !doubleStarEnabled { opts.ExcludeDirs, err = util.GlobCanonicalPath(l, opts.WorkingDir, opts.ExcludeDirs...) if err != nil { - return fmt.Errorf("invalid exclude dirs: %w", err) + return errors.Errorf("invalid exclude dirs: %w", err) } } diff --git a/cli/commands/common/runall/runall.go b/cli/commands/common/runall/runall.go index 9d097c60dc..f838059686 100644 --- a/cli/commands/common/runall/runall.go +++ b/cli/commands/common/runall/runall.go @@ -120,7 +120,9 @@ func RunAllOnStack(ctx context.Context, l log.Logger, opts *options.TerragruntOp } } - return telemetry.TelemeterFromContext(ctx).Collect(ctx, "run_all_on_stack", map[string]any{ + var runErr error + + telemetryErr := telemetry.TelemeterFromContext(ctx).Collect(ctx, "run_all_on_stack", map[string]any{ "terraform_command": opts.TerraformCommand, "working_dir": opts.WorkingDir, }, func(ctx context.Context) error { @@ -141,11 +143,22 @@ func RunAllOnStack(ctx context.Context, l log.Logger, opts *options.TerragruntOp exitCode.Set(int(cli.ExitCodeGeneralError)) + // Save error to potentially return after telemetry completes + runErr = err + + // Return nil to allow telemetry and reporting to complete return nil } return nil }) + + // log telemetry error and continue execution + if telemetryErr != nil { + l.Warnf("Telemetry collection failed: %v", telemetryErr) + } + + return runErr } // shouldSkipSummary determines if summary output should be skipped for programmatic interactions. diff --git a/config/config.go b/config/config.go index 25d8b6c47a..283e408f71 100644 --- a/config/config.go +++ b/config/config.go @@ -1991,11 +1991,11 @@ func (cfg *TerragruntConfig) ErrorsConfig() (*options.ErrorsConfig, error) { // Validate retry settings if retryBlock.MaxAttempts < 1 { - return nil, fmt.Errorf("cannot have less than 1 max retry in errors.retry %q, but you specified %d", retryBlock.Label, retryBlock.MaxAttempts) + return nil, errors.Errorf("cannot have less than 1 max retry in errors.retry %q, but you specified %d", retryBlock.Label, retryBlock.MaxAttempts) } if retryBlock.SleepIntervalSec < 0 { - return nil, fmt.Errorf("cannot sleep for less than 0 seconds in errors.retry %q, but you specified %d", retryBlock.Label, retryBlock.SleepIntervalSec) + return nil, errors.Errorf("cannot sleep for less than 0 seconds in errors.retry %q, but you specified %d", retryBlock.Label, retryBlock.SleepIntervalSec) } compiledPatterns := make([]*options.ErrorsPattern, 0, len(retryBlock.RetryableErrors)) @@ -2003,7 +2003,7 @@ func (cfg *TerragruntConfig) ErrorsConfig() (*options.ErrorsConfig, error) { for _, pattern := range retryBlock.RetryableErrors { value, err := errorsPattern(pattern) if err != nil { - return nil, fmt.Errorf("invalid retry pattern %q in block %q: %w", + return nil, errors.Errorf("invalid retry pattern %q in block %q: %w", pattern, retryBlock.Label, err) } @@ -2042,7 +2042,7 @@ func (cfg *TerragruntConfig) ErrorsConfig() (*options.ErrorsConfig, error) { for _, pattern := range ignoreBlock.IgnorableErrors { value, err := errorsPattern(pattern) if err != nil { - return nil, fmt.Errorf("invalid ignore pattern %q in block %q: %w", + return nil, errors.Errorf("invalid ignore pattern %q in block %q: %w", pattern, ignoreBlock.Label, err) } diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 7e01911f1b..68a33ff8f4 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -429,34 +429,48 @@ func Parse( parserOptions []hclparse.Option, ) error { parseOpts := opts.Clone() - parseOpts.WorkingDir = c.Path() + + // Determine working directory and config filename, supporting file paths and stack kind + componentPath := c.Path() + + var workingDir, configFilename string + + // Defaults assume a directory path + workingDir = componentPath + configFilename = config.DefaultTerragruntConfigPath + + // If the path points directly to a file, split dir and filename + if util.FileExists(componentPath) && !util.IsDir(componentPath) { + workingDir = filepath.Dir(componentPath) + configFilename = filepath.Base(componentPath) + } else { + // Allow user-specified config filename when provided as a file path + if p := opts.TerragruntConfigPath; p != "" && !util.IsDir(p) { + configFilename = filepath.Base(p) + } + // Stacks always use the default stack filename + if c.Kind() == component.StackKind { + configFilename = config.DefaultStackFile + } + } + + parseOpts.WorkingDir = workingDir // Suppress logging to avoid cluttering the output. parseOpts.Writer = io.Discard parseOpts.ErrWriter = io.Discard parseOpts.SkipOutput = true - // If the user provided a specific terragrunt config path and it is not a directory, - // use its base name as the file to parse. This allows users to run terragrunt with - // a specific config file instead of the default terragrunt.hcl. - // Otherwise, use the default terragrunt.hcl filename. - filename := config.DefaultTerragruntConfigPath - if opts.TerragruntConfigPath != "" && !util.IsDir(opts.TerragruntConfigPath) { - filename = filepath.Base(opts.TerragruntConfigPath) - } - - // For stack configurations, always use the default stack config filename - if _, ok := c.(*component.Stack); ok { - filename = config.DefaultStackFile - } - - parseOpts.TerragruntConfigPath = filepath.Join(parseOpts.WorkingDir, filename) + parseOpts.TerragruntConfigPath = filepath.Join(parseOpts.WorkingDir, configFilename) parsingCtx := config.NewParsingContext(ctx, l, parseOpts).WithDecodeList( + config.TerraformSource, config.DependenciesBlock, config.DependencyBlock, + config.TerragruntFlags, config.FeatureFlagsBlock, config.ExcludeBlock, + config.ErrorsBlock, ).WithSkipOutputsResolution() // Apply custom parser options if provided via discovery @@ -505,10 +519,13 @@ func Parse( // Set a list with partial blocks used to do discovery parsingCtx = parsingCtx.WithDecodeList( + config.TerraformSource, config.DependenciesBlock, config.DependencyBlock, + config.TerragruntFlags, config.FeatureFlagsBlock, config.ExcludeBlock, + config.ErrorsBlock, ) //nolint: contextcheck diff --git a/internal/runner/common/unit.go b/internal/runner/common/unit.go index 56fddf1600..8b7f161d00 100644 --- a/internal/runner/common/unit.go +++ b/internal/runner/common/unit.go @@ -76,12 +76,7 @@ func (unit *Unit) FlushOutput() error { } if writer, ok := unit.TerragruntOptions.Writer.(*UnitWriter); ok { - key := unit.Path - if !filepath.IsAbs(key) { - if abs, err := filepath.Abs(key); err == nil { - key = abs - } - } + key := unit.AbsolutePath(unit.Logger) mu := getUnitOutputLock(key) @@ -166,6 +161,25 @@ func (unit *Unit) FindUnitInPath(targetDirs []string) bool { return slices.Contains(targetDirs, unit.Path) } +// AbsolutePath returns the absolute path of the unit. +// If path conversion fails, returns the original path and logs a warning. +func (unit *Unit) AbsolutePath(l log.Logger) string { + if filepath.IsAbs(unit.Path) { + return unit.Path + } + + absPath, err := filepath.Abs(unit.Path) + if err != nil { + if l != nil { + l.Warnf("Failed to get absolute path for %s: %v", unit.Path, err) + } + + return unit.Path + } + + return absPath +} + // getDependenciesForUnit Get the list of units this unit depends on func (unit *Unit) getDependenciesForUnit(unitsMap UnitsMap, terragruntConfigPaths []string) (Units, error) { dependencies := Units{} diff --git a/internal/runner/common/unit_resolver.go b/internal/runner/common/unit_resolver.go index 8df89a831f..3c44e6132e 100644 --- a/internal/runner/common/unit_resolver.go +++ b/internal/runner/common/unit_resolver.go @@ -1,22 +1,44 @@ +// Package common provides primitives to build and filter Terragrunt Terraform units and their dependencies. +// UnitResolver converts discovery results into executable units, resolves/wires dependencies, applies include/exclude rules and custom filters, and records telemetry. +// +// Usage +// +// 1. Create the resolver +// ctx := context.Background() +// resolver, err := NewUnitResolver(ctx, stack) +// if err != nil { /* handle error */ } +// +// 2. Optionally add filters (applied after dependency wiring) +// resolver = resolver.WithFilters( +// /* examples: FilterByGraph(...), FilterByPaths(...), custom UnitFilter funcs */ +// ) +// +// 3. Resolve from discovery output +// units, err := resolver.ResolveFromDiscovery(ctx, logger, discoveredComponents) +// if err != nil { /* handle error */ } +// +// 4. Iterate results +// for _, u := range units { +// if u.FlagExcluded { continue } +// // use u.Config, u.Dependencies, u.TerragruntOptions, etc. +// } +// +// Notes +// - Include/Exclude: CLI include/exclude patterns are honored when the "double-star" strict control is enabled. +// Globs are compiled relative to WorkingDir and matched against unit paths. +// - Telemetry: resolver stages are wrapped with telemetry to aid performance diagnostics. package common import ( "context" - "fmt" - "os" "path/filepath" - "slices" "github.com/gobwas/glob" - "github.com/gruntwork-io/go-commons/collections" - "github.com/gruntwork-io/terragrunt/cli/commands/run/creds" - "github.com/gruntwork-io/terragrunt/cli/commands/run/creds/providers/externalcmd" "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/internal/component" "github.com/gruntwork-io/terragrunt/internal/errors" - "github.com/gruntwork-io/terragrunt/internal/report" - "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/internal/strict/controls" "github.com/gruntwork-io/terragrunt/pkg/log" - "github.com/gruntwork-io/terragrunt/shell" "github.com/gruntwork-io/terragrunt/telemetry" "github.com/gruntwork-io/terragrunt/util" ) @@ -38,19 +60,21 @@ func NewUnitResolver(ctx context.Context, stack *Stack) (*UnitResolver, error) { doubleStarEnabled = false ) - if stack.TerragruntOptions.StrictControls.FilterByNames("double-star").SuppressWarning().Evaluate(ctx) != nil { + // Check if double-star strict control is enabled + if stack.TerragruntOptions.StrictControls.FilterByNames(controls.DoubleStar).SuppressWarning().Evaluate(ctx) != nil { var err error doubleStarEnabled = true + // Compile globs only when double-star is enabled includeGlobs, err = util.CompileGlobs(stack.TerragruntOptions.WorkingDir, stack.TerragruntOptions.IncludeDirs...) if err != nil { - return nil, fmt.Errorf("invalid include dirs: %w", err) + return nil, errors.Errorf("invalid include dirs: %w", err) } excludeGlobs, err = util.CompileGlobs(stack.TerragruntOptions.WorkingDir, stack.TerragruntOptions.ExcludeDirs...) if err != nil { - return nil, fmt.Errorf("invalid exclude dirs: %w", err) + return nil, errors.Errorf("invalid exclude dirs: %w", err) } } @@ -70,23 +94,50 @@ func (r *UnitResolver) WithFilters(filters ...UnitFilter) *UnitResolver { return r } -// ResolveTerraformModules goes through each of the given Terragrunt configuration files -// and resolve the unit that configuration file represents into a Unit struct. -// Return the list of these Unit structs. -func (r *UnitResolver) ResolveTerraformModules(ctx context.Context, l log.Logger, terragruntConfigPaths []string) (Units, error) { - canonicalTerragruntConfigPaths, err := util.CanonicalPaths(terragruntConfigPaths, ".") +// ResolveFromDiscovery builds units starting from discovery-parsed components, avoiding re-parsing +// for initially discovered units. It preserves the same filtering and dependency resolution pipeline. +func (r *UnitResolver) ResolveFromDiscovery(ctx context.Context, l log.Logger, discovered []component.Component) (Units, error) { + unitsMap, err := r.telemetryBuildUnitsFromDiscovery(ctx, l, discovered) if err != nil { return nil, err } - unitsMap, err := r.telemetryResolveUnits(ctx, l, canonicalTerragruntConfigPaths) + externalDependencies, err := r.telemetryResolveExternalDependencies(ctx, l, unitsMap) if err != nil { return nil, err } - externalDependencies, err := r.telemetryResolveExternalDependencies(ctx, l, unitsMap) - if err != nil { - return nil, err + // Build the canonical config paths list for cross-linking + // Include both discovered units and external dependencies + canonicalTerragruntConfigPaths := make([]string, 0, len(discovered)+len(externalDependencies)) + for _, c := range discovered { + if c.Kind() == component.StackKind { + continue + } + + fname := r.determineTerragruntConfigFilename() + configPath := filepath.Join(c.Path(), fname) + + canonicalPath, err := util.CanonicalPath(configPath, ".") + if err != nil { + return nil, errors.Errorf("canonicalizing terragrunt config path %q for unit %s: %w", configPath, c.Path(), err) + } + + canonicalTerragruntConfigPaths = append(canonicalTerragruntConfigPaths, canonicalPath) + } + + // Add external dependencies to canonical paths list + for _, extDep := range externalDependencies { + // Use the actual TerragruntConfigPath from the unit's options + // This handles non-default config file names correctly + if extDep.TerragruntOptions != nil && extDep.TerragruntOptions.TerragruntConfigPath != "" { + canonicalPath, err := util.CanonicalPath(extDep.TerragruntOptions.TerragruntConfigPath, ".") + if err != nil { + return nil, errors.Errorf("canonicalizing terragrunt config path %q for external dependency %s: %w", extDep.TerragruntOptions.TerragruntConfigPath, extDep.Path, err) + } + + canonicalTerragruntConfigPaths = append(canonicalTerragruntConfigPaths, canonicalPath) + } } crossLinkedUnits, err := r.telemetryCrossLinkDependencies(ctx, unitsMap, externalDependencies, canonicalTerragruntConfigPaths) @@ -132,16 +183,15 @@ func (r *UnitResolver) ResolveTerraformModules(ctx context.Context, l log.Logger return filteredUnits, nil } -// telemetryResolveUnits resolves Terraform units from the given Terragrunt configuration paths -func (r *UnitResolver) telemetryResolveUnits(ctx context.Context, l log.Logger, canonicalTerragruntConfigPaths []string) (UnitsMap, error) { +// telemetryBuildUnitsFromDiscovery wraps buildUnitsFromDiscovery in telemetry collection. +func (r *UnitResolver) telemetryBuildUnitsFromDiscovery(ctx context.Context, l log.Logger, discovered []component.Component) (UnitsMap, error) { var unitsMap UnitsMap - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "resolve_units", map[string]any{ + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "build_units_from_discovery", map[string]any{ "working_dir": r.Stack.TerragruntOptions.WorkingDir, + "unit_count": len(discovered), }, func(ctx context.Context) error { - howThesePathsWereFound := "Terragrunt config file found in a subdirectory of " + r.Stack.TerragruntOptions.WorkingDir - - result, err := r.resolveUnits(ctx, l, canonicalTerragruntConfigPaths, howThesePathsWereFound) + result, err := r.buildUnitsFromDiscovery(l, discovered) if err != nil { return err } @@ -154,881 +204,128 @@ func (r *UnitResolver) telemetryResolveUnits(ctx context.Context, l log.Logger, return unitsMap, err } -// telemetryResolveExternalDependencies resolves external dependencies for the given units -func (r *UnitResolver) telemetryResolveExternalDependencies(ctx context.Context, l log.Logger, unitsMap UnitsMap) (UnitsMap, error) { - var externalDependencies UnitsMap - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "resolve_external_dependencies_for_units", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(ctx context.Context) error { - result, err := r.resolveExternalDependenciesForUnits(ctx, l, unitsMap, UnitsMap{}, 0) - if err != nil { - return err - } - - externalDependencies = result - - return nil - }) - - return externalDependencies, err -} - -// telemetryCrossLinkDependencies cross-links dependencies between units -func (r *UnitResolver) telemetryCrossLinkDependencies(ctx context.Context, unitsMap, externalDependencies UnitsMap, canonicalTerragruntConfigPaths []string) (Units, error) { - var crossLinkedUnits Units - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "crosslink_dependencies", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(_ context.Context) error { - result, err := unitsMap.MergeMaps(externalDependencies).CrossLinkDependencies(canonicalTerragruntConfigPaths) - if err != nil { - return err - } - - crossLinkedUnits = result - - return nil - }) - - return crossLinkedUnits, err -} - -// telemetryFlagIncludedDirs flags directories that are included in the Terragrunt configuration -func (r *UnitResolver) telemetryFlagIncludedDirs(ctx context.Context, l log.Logger, crossLinkedUnits Units) (Units, error) { - var withUnitsIncluded Units - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_included_dirs", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(_ context.Context) error { - withUnitsIncluded = r.flagIncludedDirs(r.Stack.TerragruntOptions, l, crossLinkedUnits) - return nil - }) - - return withUnitsIncluded, err -} - -// telemetryFlagUnitsThatAreIncluded flags units that are included in the Terragrunt configuration -func (r *UnitResolver) telemetryFlagUnitsThatAreIncluded(ctx context.Context, withUnitsIncluded Units) (Units, error) { - var withUnitsThatAreIncludedByOthers Units - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_units_that_are_included", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(_ context.Context) error { - result, err := r.flagUnitsThatAreIncluded(r.Stack.TerragruntOptions, withUnitsIncluded) - if err != nil { - return err - } - - withUnitsThatAreIncludedByOthers = result - - return nil - }) - - return withUnitsThatAreIncludedByOthers, err -} - -// telemetryFlagExcludedUnits flags units that are excluded in the Terragrunt configuration -func (r *UnitResolver) telemetryFlagExcludedUnits(ctx context.Context, l log.Logger, withUnitsThatAreIncludedByOthers Units) (Units, error) { - var withExcludedUnits Units - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_excluded_units", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(_ context.Context) error { - result := r.flagExcludedUnits(l, r.Stack.TerragruntOptions, r.Stack.Report, withUnitsThatAreIncludedByOthers) - withExcludedUnits = result - - return nil - }) - - return withExcludedUnits, err -} - -// telemetryFlagUnitsThatRead flags units that read files in the Terragrunt configuration -func (r *UnitResolver) telemetryFlagUnitsThatRead(ctx context.Context, withExcludedUnits Units) (Units, error) { - var withUnitsRead Units - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_units_that_read", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(_ context.Context) error { - withUnitsRead = r.flagUnitsThatRead(r.Stack.TerragruntOptions, withExcludedUnits) - return nil - }) - - return withUnitsRead, err -} - -// telemetryFlagExcludedDirs flags directories that are excluded in the Terragrunt configuration -func (r *UnitResolver) telemetryFlagExcludedDirs(ctx context.Context, l log.Logger, withUnitsRead Units) (Units, error) { - var withUnitsExcluded Units - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_excluded_dirs", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(_ context.Context) error { - withUnitsExcluded = r.flagExcludedDirs(l, r.Stack.TerragruntOptions, r.Stack.Report, withUnitsRead) - return nil - }) - - return withUnitsExcluded, err -} - -// Go through each of the given Terragrunt configuration files and resolve the unit that configuration file represents -// into a Unit struct. Note that this method will NOT fill in the Dependencies field of the Unit -// struct (see the crosslinkDependencies method for that). Return a map from unit path to Unit struct. -func (r *UnitResolver) resolveUnits(ctx context.Context, l log.Logger, canonicalTerragruntConfigPaths []string, howTheseUnitsWereFound string) (UnitsMap, error) { - unitsMap := UnitsMap{} - - for _, terragruntConfigPath := range canonicalTerragruntConfigPaths { - if !util.FileExists(terragruntConfigPath) { - return nil, ProcessingUnitError{UnderlyingError: os.ErrNotExist, UnitPath: terragruntConfigPath, HowThisUnitWasFound: howTheseUnitsWereFound} - } - - var unit *Unit - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "resolve_terraform_unit", map[string]any{ - "config_path": terragruntConfigPath, - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - }, func(ctx context.Context) error { - m, err := r.resolveTerraformUnit(ctx, l, terragruntConfigPath, unitsMap, howTheseUnitsWereFound) - if err != nil { - return err - } - - unit = m - - return nil - }) - if err != nil { - return unitsMap, err - } - - if unit != nil { - unitsMap[unit.Path] = unit - - var dependencies UnitsMap - - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "resolve_dependencies_for_unit", map[string]any{ - "config_path": terragruntConfigPath, - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - "unit_path": unit.Path, - }, func(ctx context.Context) error { - deps, err := r.resolveDependenciesForUnit(ctx, l, unit, unitsMap, true) - if err != nil { - return err - } - - dependencies = deps - - return nil - }) - if err != nil { - return unitsMap, err - } - - unitsMap = collections.MergeMaps(unitsMap, dependencies) - } - } - - return unitsMap, nil -} - -// Create a Unit struct for the Terraform unit specified by the given Terragrunt configuration file path. -// Note that this method will NOT fill in the Dependencies field of the Unit struct (see the -// crosslinkDependencies method for that). -func (r *UnitResolver) resolveTerraformUnit(ctx context.Context, l log.Logger, terragruntConfigPath string, unitsMap UnitsMap, howThisUnitWasFound string) (*Unit, error) { - unitPath, err := r.resolveUnitPath(terragruntConfigPath) - if err != nil { - return nil, err - } - - if _, ok := unitsMap[unitPath]; ok { - return nil, nil - } - - l, opts, err := r.cloneOptionsWithConfigPath(l, terragruntConfigPath) - if err != nil { - return nil, err - } - - includeConfig := r.setupIncludeConfig(terragruntConfigPath, opts) - - excludeFn := func(l log.Logger, unitPath string) bool { - for globPath, glob := range r.excludeGlobs { - if glob.Match(unitPath) { - l.Debugf("Unit %s is excluded by glob %s", unitPath, globPath) - return true - } - } - - return false - } - if !r.doubleStarEnabled { - excludeFn = func(_ log.Logger, unitPath string) bool { - return collections.ListContainsElement(opts.ExcludeDirs, unitPath) - } - } - - if excludeFn(l, unitPath) { - return &Unit{Path: unitPath, Logger: l, TerragruntOptions: opts, FlagExcluded: true}, nil - } - - parseCtx := r.createParsingContext(ctx, l, opts) - - if err = r.acquireCredentials(ctx, l, opts); err != nil { - return nil, err - } - - //nolint:contextcheck - terragruntConfig, err := r.partialParseConfig(parseCtx, l, terragruntConfigPath, includeConfig, howThisUnitWasFound) - if err != nil { - return nil, err - } - - // Extract files read by this unit from the parsing context - var readFiles []string - if parseCtx.FilesRead != nil { - readFiles = *parseCtx.FilesRead - } - - terragruntSource, err := config.GetTerragruntSourceForModule(r.Stack.TerragruntOptions.Source, unitPath, terragruntConfig) - if err != nil { - return nil, err - } - - opts.Source = terragruntSource - - if err = r.setupDownloadDir(terragruntConfigPath, opts, l); err != nil { - return nil, err - } - - hasFiles, err := util.DirContainsTFFiles(filepath.Dir(terragruntConfigPath)) - if err != nil { - return nil, err - } - - if (terragruntConfig.Terraform == nil || terragruntConfig.Terraform.Source == nil || *terragruntConfig.Terraform.Source == "") && !hasFiles { - l.Debugf("Unit %s does not have an associated terraform configuration and will be skipped.", filepath.Dir(terragruntConfigPath)) - return nil, nil - } - - return &Unit{Path: unitPath, Logger: l, Config: *terragruntConfig, TerragruntOptions: opts, Reading: readFiles}, nil -} - -// resolveUnitPath converts a Terragrunt configuration file path to its corresponding unit path. -// Returns the canonical path of the directory containing the config file. -func (r *UnitResolver) resolveUnitPath(terragruntConfigPath string) (string, error) { - return util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".") -} - -// cloneOptionsWithConfigPath creates a copy of the Terragrunt options with a new config path. -// Returns the cloned logger, options, and any error that occurred during cloning. -func (r *UnitResolver) cloneOptionsWithConfigPath(l log.Logger, terragruntConfigPath string) (log.Logger, *options.TerragruntOptions, error) { - l, opts, err := r.Stack.TerragruntOptions.CloneWithConfigPath(l, terragruntConfigPath) - if err != nil { - return l, nil, err - } - - opts.OriginalTerragruntConfigPath = terragruntConfigPath - - return l, opts, nil -} - -// setupIncludeConfig creates an include configuration for Terragrunt config inheritance. -// Returns the include config if the path is processed, otherwise returns nil. -func (r *UnitResolver) setupIncludeConfig(terragruntConfigPath string, opts *options.TerragruntOptions) *config.IncludeConfig { - var includeConfig *config.IncludeConfig - if r.Stack.ChildTerragruntConfig != nil && r.Stack.ChildTerragruntConfig.ProcessedIncludes.ContainsPath(terragruntConfigPath) { - includeConfig = &config.IncludeConfig{ - Path: terragruntConfigPath, - } - opts.TerragruntConfigPath = r.Stack.TerragruntOptions.OriginalTerragruntConfigPath - } - - return includeConfig -} - -// createParsingContext initializes a parsing context for Terragrunt configuration files. -// Returns a configured parsing context with specific decode options for Terraform blocks. -func (r *UnitResolver) createParsingContext(ctx context.Context, l log.Logger, opts *options.TerragruntOptions) *config.ParsingContext { - parseOpts := opts.Clone() - parseOpts.SkipOutput = false - - return config.NewParsingContext(ctx, l, parseOpts). - WithParseOption(r.Stack.ParserOptions). - WithDecodeList( - config.TerraformSource, - config.DependenciesBlock, - config.DependencyBlock, - config.FeatureFlagsBlock, - config.ExcludeBlock, - config.ErrorsBlock, - ) -} - -// acquireCredentials obtains and updates environment credentials for Terraform providers. -// Returns an error if credential acquisition fails. -func (r *UnitResolver) acquireCredentials(ctx context.Context, l log.Logger, opts *options.TerragruntOptions) error { - credsGetter := creds.NewGetter() - return credsGetter.ObtainAndUpdateEnvIfNecessary(ctx, l, opts, externalcmd.NewProvider(l, opts)) -} - -// partialParseConfig parses a Terragrunt configuration file with limited block decoding. -// Returns the parsed configuration or an error if parsing fails. -func (r *UnitResolver) partialParseConfig(parseCtx *config.ParsingContext, l log.Logger, terragruntConfigPath string, includeConfig *config.IncludeConfig, howThisUnitWasFound string) (*config.TerragruntConfig, error) { - terragruntConfig, err := config.PartialParseConfigFile( - parseCtx, - l, - terragruntConfigPath, - includeConfig, - ) - if err != nil { - return nil, errors.New(ProcessingUnitError{ - UnderlyingError: err, - HowThisUnitWasFound: howThisUnitWasFound, - UnitPath: terragruntConfigPath, - }) - } - - return terragruntConfig, nil -} - -// setupDownloadDir configures the download directory for a Terragrunt unit. -// Returns an error if the download directory setup fails. -func (r *UnitResolver) setupDownloadDir(terragruntConfigPath string, opts *options.TerragruntOptions, l log.Logger) error { - _, defaultDownloadDir, err := options.DefaultWorkingAndDownloadDirs(r.Stack.TerragruntOptions.TerragruntConfigPath) - if err != nil { - return err - } - - if r.Stack.TerragruntOptions.DownloadDir == defaultDownloadDir { - _, downloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntConfigPath) - if err != nil { - return err - } - - l.Debugf("Setting download directory for unit %s to %s", filepath.Dir(opts.TerragruntConfigPath), downloadDir) - opts.DownloadDir = downloadDir - } - - return nil -} - -// resolveDependenciesForUnit looks through the dependencies of the given unit and resolve the dependency paths listed in the unit's config. -// If `skipExternal` is true, the func returns only dependencies that are inside of the current working directory, which means they are part of the environment the -// user is trying to run --all apply or run --all destroy. Note that this method will NOT fill in the Dependencies field of the Unit struct (see the crosslinkDependencies method for that). -func (r *UnitResolver) resolveDependenciesForUnit(ctx context.Context, l log.Logger, unit *Unit, unitsMap UnitsMap, skipExternal bool) (UnitsMap, error) { - if unit.Config.Dependencies == nil || len(unit.Config.Dependencies.Paths) == 0 { - return UnitsMap{}, nil - } - - externalTerragruntConfigPaths := []string{} - - for _, dependency := range unit.Config.Dependencies.Paths { - dependencyPath, err := util.CanonicalPath(dependency, unit.Path) - if err != nil { - return UnitsMap{}, err +// buildUnitsFromDiscovery constructs UnitsMap from discovery-parsed components without re-parsing, +// performing only the minimal parsing necessary to obtain missing fields (e.g., Terraform.source). +// +// This is the first stage of the unit resolution pipeline. It converts discovery components into +// Unit structs, preserving already-parsed configuration data to avoid redundant file I/O. +// +// The method: +// 1. Filters out non-units (e.g., stacks) +// 2. Skips units with parse errors from discovery +// 3. Determines the correct config file name (terragrunt.hcl or custom) +// 4. Resolves unit paths to canonical form +// 5. Checks if units should be excluded based on CLI flags (setting FlagExcluded=true) +// 6. Reuses parsed config from discovery (including TerraformSource and ErrorsBlock) +// 7. Sets up download directories for each unit +// 8. Skips units without Terraform source or TF files +// +// Units excluded at this stage have FlagExcluded=true and minimal configuration. +// They are still included in the UnitsMap for dependency resolution but won't be executed. +func (r *UnitResolver) buildUnitsFromDiscovery(l log.Logger, discovered []component.Component) (UnitsMap, error) { + units := make(UnitsMap) + + for _, c := range discovered { + // Only handle terraform units; skip stacks and anything else + if c.Kind() == component.StackKind { + continue } - if skipExternal && !util.HasPathPrefix(dependencyPath, r.Stack.TerragruntOptions.WorkingDir) { + dUnit, ok := c.(*component.Unit) + if !ok { continue } - terragruntConfigPath := config.GetDefaultConfigPath(dependencyPath) - - if _, alreadyContainsUnit := unitsMap[dependencyPath]; !alreadyContainsUnit { - externalTerragruntConfigPaths = append(externalTerragruntConfigPaths, terragruntConfigPath) + // Get the config that discovery already parsed + terragruntConfig := dUnit.Config() + if terragruntConfig == nil { + // Skip configurations that discovery could not parse + l.Warnf("Skipping unit at %s due to parse error", dUnit.Path()) + continue } - } - howThesePathsWereFound := fmt.Sprintf("dependency of unit at '%s'", unit.Path) - - result, err := r.resolveUnits(ctx, l, externalTerragruntConfigPaths, howThesePathsWereFound) - if err != nil { - return nil, err - } - - return result, nil -} - -// Look through the dependencies of the units in the given map and resolve the "external" dependency paths listed in -// each units config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths). -// These external dependencies are outside of the current working directory, which means they may not be part of the -// environment the user is trying to run --all apply or run --all destroy. Therefore, this method also confirms whether the user wants -// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in -// the Dependencies field of the Unit struct (see the crosslinkDependencies method for that). -func (r *UnitResolver) resolveExternalDependenciesForUnits(ctx context.Context, l log.Logger, unitsMap, unitsAlreadyProcessed UnitsMap, recursionLevel int) (UnitsMap, error) { - allExternalDependencies := UnitsMap{} - unitsToSkip := unitsMap.MergeMaps(unitsAlreadyProcessed) - - // Simple protection from circular dependencies causing a Stack Overflow due to infinite recursion - const maxLevelsOfRecursion = 20 - if recursionLevel > maxLevelsOfRecursion { - return allExternalDependencies, errors.New(InfiniteRecursionError{RecursionLevel: maxLevelsOfRecursion, Units: unitsToSkip}) - } - - sortedKeys := unitsMap.SortedKeys() - for _, key := range sortedKeys { - unit := unitsMap[key] - - externalDependencies, err := r.resolveDependenciesForUnit(ctx, l, unit, unitsToSkip, false) - if err != nil { - return externalDependencies, err + // Determine the actual config file path + terragruntConfigPath := dUnit.Path() + if util.IsDir(terragruntConfigPath) { + fname := r.determineTerragruntConfigFilename() + terragruntConfigPath = filepath.Join(dUnit.Path(), fname) } - l, unitOpts, err := r.Stack.TerragruntOptions.CloneWithConfigPath(l, config.GetDefaultConfigPath(unit.Path)) + unitPath, err := r.resolveUnitPath(terragruntConfigPath) if err != nil { return nil, err } - for _, externalDependency := range externalDependencies { - if _, alreadyFound := unitsToSkip[externalDependency.Path]; alreadyFound { - continue - } - - shouldApply := false - if !r.Stack.TerragruntOptions.IgnoreExternalDependencies { - shouldApply, err = r.confirmShouldApplyExternalDependency(ctx, unit, l, externalDependency, unitOpts) - if err != nil { - return externalDependencies, err - } - } - - externalDependency.AssumeAlreadyApplied = !shouldApply - // Mark external dependencies as excluded if they shouldn't be applied - // This ensures they are tracked in the report but not executed - if !shouldApply { - externalDependency.FlagExcluded = true - } - - allExternalDependencies[externalDependency.Path] = externalDependency - } - } - - if len(allExternalDependencies) > 0 { - recursiveDependencies, err := r.resolveExternalDependenciesForUnits(ctx, l, allExternalDependencies, unitsMap, recursionLevel+1) + // Prepare options with proper working dir + l, opts, err := r.Stack.TerragruntOptions.CloneWithConfigPath(l, terragruntConfigPath) if err != nil { - return allExternalDependencies, err + return nil, err } - return allExternalDependencies.MergeMaps(recursiveDependencies), nil - } - - return allExternalDependencies, nil -} - -// Confirm with the user whether they want Terragrunt to assume the given dependency of the given unit is already -// applied. If the user selects "yes", then Terragrunt will apply that unit as well. -// Note that we skip the prompt for `run --all destroy` calls. Given the destructive and irreversible nature of destroy, we don't -// want to provide any risk to the user of accidentally destroying an external dependency unless explicitly included -// with the --queue-include-external or --queue-include-dir flags. -func (r *UnitResolver) confirmShouldApplyExternalDependency(ctx context.Context, unit *Unit, l log.Logger, dependency *Unit, opts *options.TerragruntOptions) (bool, error) { - if opts.IncludeExternalDependencies { - l.Debugf("The --queue-include-external flag is set, so automatically including all external dependencies, and will run this command against unit %s, which is a dependency of unit %s.", dependency.Path, unit.Path) - return true, nil - } - - if opts.NonInteractive { - l.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run --all command, will not run this command against unit %s, which is a dependency of unit %s.", dependency.Path, unit.Path) - return false, nil - } - - stackCmd := opts.TerraformCommand - if stackCmd == "destroy" { - l.Debugf("run --all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run --all command, will not run this command against unit %s, which is a dependency of unit %s.", dependency.Path, unit.Path) - return false, nil - } - - l.Infof("Unit %s has external dependency %s", unit.Path, dependency.Path) - - return shell.PromptUserForYesNo(ctx, l, "Should Terragrunt apply the external dependency?", opts) -} - -// flagIncludedDirs includes all units by default. -// -// However, when anything that triggers ExcludeByDefault is set, the function will instead -// selectively include only the units that are in the list specified via the IncludeDirs option. -func (r *UnitResolver) flagIncludedDirs(opts *options.TerragruntOptions, l log.Logger, units Units) Units { - if !opts.ExcludeByDefault { - return units - } - - includeFn := func(l log.Logger, unit *Unit) bool { - for globPath, glob := range r.includeGlobs { - if glob.Match(unit.Path) { - l.Debugf("Unit %s is included by glob %s", unit.Path, globPath) - return true - } - } + opts.OriginalTerragruntConfigPath = terragruntConfigPath - return false - } - if !r.doubleStarEnabled { - includeFn = func(_ log.Logger, unit *Unit) bool { - if unit.FindUnitInPath(opts.IncludeDirs) { - return true - } else { - return false - } - } - } + // Exclusion check - create a temporary unit for matching + unitToExclude := &Unit{Path: unitPath, Logger: l, TerragruntOptions: opts, FlagExcluded: true} + excludeFn := r.createPathMatcherFunc("exclude", opts, l) - for _, unit := range units { - unit.FlagExcluded = true - if includeFn(l, unit) { - unit.FlagExcluded = false - } - } + if excludeFn(unitToExclude) { + units[unitPath] = unitToExclude - // Mark all affected dependencies as included before proceeding if not in strict include mode. - if !opts.StrictInclude { - for _, unit := range units { - if !unit.FlagExcluded { - for _, dependency := range unit.Dependencies { - dependency.FlagExcluded = false - } - } + continue } - } - - return units -} - -// flagUnitsThatAreIncluded iterates over a unit slice and flags all units that include at least one file in -// the specified include list on the TerragruntOptions ModulesThatInclude attribute. -func (r *UnitResolver) flagUnitsThatAreIncluded(opts *options.TerragruntOptions, units Units) (Units, error) { - unitsThatInclude := append(opts.ModulesThatInclude, opts.UnitsReading...) //nolint:gocritic - if len(unitsThatInclude) == 0 { - return units, nil - } - - unitsThatIncludeCanonicalPaths := []string{} - - for _, includePath := range unitsThatInclude { - canonicalPath, err := util.CanonicalPath(includePath, opts.WorkingDir) + // Determine effective source and setup download dir + terragruntSource, err := config.GetTerragruntSourceForModule(r.Stack.TerragruntOptions.Source, unitPath, terragruntConfig) if err != nil { return nil, err } - unitsThatIncludeCanonicalPaths = append(unitsThatIncludeCanonicalPaths, canonicalPath) - } + opts.Source = terragruntSource - for _, unit := range units { - if err := r.flagUnitIncludes(unit, unitsThatIncludeCanonicalPaths); err != nil { - return nil, err + // Update the config's source with the mapped source so that logging shows the correct URL + if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.Source != nil && terragruntSource != "" { + terragruntConfig.Terraform.Source = &terragruntSource } - if err := r.flagUnitDependencies(unit, unitsThatIncludeCanonicalPaths); err != nil { + if err = r.setupDownloadDir(terragruntConfigPath, opts, l); err != nil { return nil, err } - } - return units, nil -} + // Check for TF files in the directory or any of its subdirectories + dir := filepath.Dir(terragruntConfigPath) -// flagUnitIncludes marks a unit as included if any of its include paths match the canonical paths. -// Returns an error if path resolution fails during the comparison. -func (r *UnitResolver) flagUnitIncludes(unit *Unit, canonicalPaths []string) error { - for _, includeConfig := range unit.Config.ProcessedIncludes { - canonicalPath, err := util.CanonicalPath(includeConfig.Path, unit.Path) + hasFiles, err := util.DirContainsTFFiles(dir) if err != nil { - return err - } - - if util.ListContainsElement(canonicalPaths, canonicalPath) { - unit.FlagExcluded = false - } - } - - return nil -} - -// flagUnitDependencies processes dependencies of a unit and flags them based on include paths. -// Returns an error if dependency processing fails. -func (r *UnitResolver) flagUnitDependencies(unit *Unit, canonicalPaths []string) error { - for _, dependency := range unit.Dependencies { - if dependency.FlagExcluded { - continue - } - - if err := r.flagDependencyIncludes(dependency, unit.Path, canonicalPaths); err != nil { - return err - } - } - - return nil -} - -// flagDependencyIncludes marks a dependency as included if any of its include paths match the canonical paths. -// Returns an error if path resolution fails during the comparison. -func (r *UnitResolver) flagDependencyIncludes(dependency *Unit, unitPath string, canonicalPaths []string) error { - for _, includeConfig := range dependency.Config.ProcessedIncludes { - canonicalPath, err := util.CanonicalPath(includeConfig.Path, unitPath) - if err != nil { - return err - } - - if util.ListContainsElement(canonicalPaths, canonicalPath) { - dependency.FlagExcluded = false - } - } - - return nil -} - -// flagExcludedUnits iterates over a unit slice and flags all units that are excluded based on the exclude block. -func (r *UnitResolver) flagExcludedUnits(l log.Logger, opts *options.TerragruntOptions, reportInstance *report.Report, units Units) Units { - for _, unit := range units { - excludeConfig := unit.Config.Exclude - - if excludeConfig == nil { - continue + return nil, err } - if !excludeConfig.IsActionListed(opts.TerraformCommand) { + if (terragruntConfig.Terraform == nil || terragruntConfig.Terraform.Source == nil || *terragruntConfig.Terraform.Source == "") && !hasFiles { + l.Debugf("Unit %s does not have an associated terraform configuration and will be skipped.", filepath.Dir(terragruntConfigPath)) continue } - if excludeConfig.If { - // Check if unit was already excluded (e.g., by --queue-exclude-dir) - // If so, don't overwrite the existing exclusion reason - wasAlreadyExcluded := unit.FlagExcluded - l.Debugf("Unit %s is excluded by exclude block (wasAlreadyExcluded=%v)", unit.Path, wasAlreadyExcluded) - unit.FlagExcluded = true - - // Only update report if it's enabled AND the unit wasn't already excluded - // This ensures CLI flags like --queue-exclude-dir take precedence over exclude blocks - if reportInstance != nil && !wasAlreadyExcluded { - // Ensure path is absolute for reporting - unitPath := unit.Path - if !filepath.IsAbs(unitPath) { - var absErr error - - unitPath, absErr = filepath.Abs(unitPath) - if absErr != nil { - l.Warnf("Could not resolve absolute path for unit %s, using cleaned relative path: %v", unit.Path, absErr) - unitPath = filepath.Clean(unit.Path) - } - } - - // Only report if not already excluded - EndRun will handle this gracefully - // by returning early if the run already ended with ResultExcluded - run, err := reportInstance.EnsureRun(unitPath) - if err != nil { - l.Errorf("Error ensuring run for unit %s: %v", unitPath, err) - continue - } - - // EndRun will skip updating if already ended with ResultExcluded - if err := reportInstance.EndRun( - run.Path, - report.WithResult(report.ResultExcluded), - report.WithReason(report.ReasonExcludeBlock), - ); err != nil { - l.Errorf("Error ending run for unit %s: %v", unitPath, err) - continue - } - } - } - - if excludeConfig.ExcludeDependencies != nil && *excludeConfig.ExcludeDependencies { - l.Debugf("Excluding dependencies for unit %s by exclude block", unit.Path) - - for _, dependency := range unit.Dependencies { - // Check if dependency was already excluded - wasAlreadyExcluded := dependency.FlagExcluded - dependency.FlagExcluded = true - - // Only update report if it's enabled AND the dependency wasn't already excluded - // This ensures CLI exclusions take precedence over exclude blocks - if reportInstance != nil && !wasAlreadyExcluded { - // Ensure path is absolute for reporting - depPath := dependency.Path - if !filepath.IsAbs(depPath) { - var absErr error - - depPath, absErr = filepath.Abs(depPath) - if absErr != nil { - l.Errorf("Error getting absolute path for dependency %s: %v", dependency.Path, absErr) - // Revert exclusion since reporting couldn't proceed and this block changed the state - dependency.FlagExcluded = false - - continue - } - } - - run, err := reportInstance.EnsureRun(depPath) - if err != nil { - l.Errorf("Error ensuring run for dependency %s: %v", depPath, err) - continue - } - - if err := reportInstance.EndRun( - run.Path, - report.WithResult(report.ResultExcluded), - report.WithReason(report.ReasonExcludeBlock), - ); err != nil { - l.Errorf("Error ending run for dependency %s: %v", depPath, err) - continue - } - } - } - } + units[unitPath] = &Unit{Path: unitPath, Logger: l, Config: *terragruntConfig, TerragruntOptions: opts, Reading: dUnit.Reading()} } - return units -} - -// flagUnitsThatRead iterates over a unit slice and flags all units that read at least one file in the specified -// file list in the TerragruntOptions UnitsReading attribute. -func (r *UnitResolver) flagUnitsThatRead(opts *options.TerragruntOptions, units Units) Units { - // If no UnitsThatRead is specified, return the unit list instantly - if len(opts.UnitsReading) == 0 { - return units - } - - for _, path := range opts.UnitsReading { - if !filepath.IsAbs(path) { - path = filepath.Join(opts.WorkingDir, path) - path = filepath.Clean(path) - } - - for _, unit := range units { - if slices.Contains(unit.Reading, path) { - unit.FlagExcluded = false - } - } - } - - return units -} - -// flagExcludedDirs iterates over a unit slice and flags all entries as excluded listed in the queue-exclude-dir CLI flag. -func (r *UnitResolver) flagExcludedDirs(l log.Logger, opts *options.TerragruntOptions, reportInstance *report.Report, units Units) Units { - // If we don't have any excludes, we don't need to do anything. - if (len(r.excludeGlobs) == 0 && r.doubleStarEnabled) || len(opts.ExcludeDirs) == 0 { - return units - } - - excludeFn := func(l log.Logger, unit *Unit) bool { - for globPath, glob := range r.excludeGlobs { - if glob.Match(unit.Path) { - l.Debugf("Unit %s is excluded by glob %s", unit.Path, globPath) - return true - } - } - - return false - } - if !r.doubleStarEnabled { - excludeFn = func(l log.Logger, unit *Unit) bool { - return unit.FindUnitInPath(opts.ExcludeDirs) - } - } - - for _, unit := range units { - if excludeFn(l, unit) { - // Mark unit itself as excluded - l.Debugf("Unit %s is excluded", unit.Path) - unit.FlagExcluded = true - - // Only update report if it's enabled - if reportInstance != nil { - // Ensure path is absolute for reporting - unitPath := unit.Path - if !filepath.IsAbs(unitPath) { - var absErr error - - unitPath, absErr = filepath.Abs(unitPath) - if absErr != nil { - l.Errorf("Error getting absolute path for unit %s: %v", unit.Path, absErr) - continue - } - } - - // TODO: Make an upsert option for ends, - // so that I don't have to do this every time. - run, err := reportInstance.EnsureRun(unitPath) - if err != nil { - l.Errorf("Error ensuring run for unit %s: %v", unitPath, err) - continue - } - - if err := reportInstance.EndRun( - run.Path, - report.WithResult(report.ResultExcluded), - report.WithReason(report.ReasonExcludeDir), - ); err != nil { - l.Errorf("Error ending run for unit %s: %v", unitPath, err) - continue - } - } - } - - // Mark all affected dependencies as excluded - for _, dependency := range unit.Dependencies { - if excludeFn(l, dependency) { - dependency.FlagExcluded = true - - // Only update report if it's enabled - if reportInstance != nil { - // Ensure path is absolute for reporting - depPath := dependency.Path - if !filepath.IsAbs(depPath) { - var absErr error - - depPath, absErr = filepath.Abs(depPath) - if absErr != nil { - l.Errorf("Error getting absolute path for dependency %s: %v", dependency.Path, absErr) - continue - } - } - - run, err := reportInstance.EnsureRun(depPath) - if err != nil { - l.Errorf("Error ensuring run for dependency %s: %v", depPath, err) - continue - } - - if err := reportInstance.EndRun( - run.Path, - report.WithResult(report.ResultExcluded), - report.WithReason(report.ReasonExcludeDir), - ); err != nil { - l.Errorf("Error ending run for dependency %s: %v", depPath, err) - continue - } - } - } - } - } - - return units + return units, nil } -// telemetryApplyFilters applies all configured unit filters to the resolved units -func (r *UnitResolver) telemetryApplyFilters(ctx context.Context, units Units) (Units, error) { - if len(r.filters) == 0 { - return units, nil - } - - var filteredUnits Units +// telemetryResolveExternalDependencies resolves external dependencies for the given units +func (r *UnitResolver) telemetryResolveExternalDependencies(ctx context.Context, l log.Logger, unitsMap UnitsMap) (UnitsMap, error) { + var externalDependencies UnitsMap - err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "apply_unit_filters", map[string]any{ - "working_dir": r.Stack.TerragruntOptions.WorkingDir, - "filter_count": len(r.filters), + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "resolve_external_dependencies_for_units", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, }, func(ctx context.Context) error { - // Apply all filters in sequence - for _, filter := range r.filters { - if err := filter.Filter(ctx, units, r.Stack.TerragruntOptions); err != nil { - return err - } + result, err := r.resolveExternalDependenciesForUnits(ctx, l, unitsMap, UnitsMap{}, 0) + if err != nil { + return err } - filteredUnits = units + externalDependencies = result return nil }) - return filteredUnits, err + return externalDependencies, err } diff --git a/internal/runner/common/unit_resolver_dependencies.go b/internal/runner/common/unit_resolver_dependencies.go new file mode 100644 index 0000000000..cf528b4bcd --- /dev/null +++ b/internal/runner/common/unit_resolver_dependencies.go @@ -0,0 +1,161 @@ +package common + +import ( + "context" + "path/filepath" + + "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/internal/errors" + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/pkg/log" + "github.com/gruntwork-io/terragrunt/shell" + "github.com/gruntwork-io/terragrunt/telemetry" + "github.com/gruntwork-io/terragrunt/util" +) + +const maxLevelsOfRecursion = 20 + +// Look through the dependencies of the units in the given map and resolve the "external" dependency paths listed in +// each units config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths). +// These external dependencies are outside of the current working directory, which means they may not be part of the +// environment the user is trying to run --all apply or run --all destroy. Therefore, this method also confirms whether the user wants +// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in +// the Dependencies field of the Unit struct (see the crosslinkDependencies method for that). +func (r *UnitResolver) resolveExternalDependenciesForUnits(ctx context.Context, l log.Logger, unitsMap, unitsAlreadyProcessed UnitsMap, recursionLevel int) (UnitsMap, error) { + allExternalDependencies := UnitsMap{} + unitsToSkip := unitsMap.MergeMaps(unitsAlreadyProcessed) + + // Simple protection from circular dependencies causing a Stack Overflow due to infinite recursion + if recursionLevel > maxLevelsOfRecursion { + return allExternalDependencies, errors.New(InfiniteRecursionError{RecursionLevel: maxLevelsOfRecursion, Units: unitsToSkip}) + } + + sortedKeys := unitsMap.SortedKeys() + for _, key := range sortedKeys { + unit := unitsMap[key] + + // Check if this unit has dependencies that are considered "external" + if unit.Config.Dependencies == nil || len(unit.Config.Dependencies.Paths) == 0 { + continue + } + + l, unitOpts, err := r.Stack.TerragruntOptions.CloneWithConfigPath(l, config.GetDefaultConfigPath(unit.Path)) + if err != nil { + return nil, err + } + + // For each dependency, check if it's external (outside working dir) and already in unitsToSkip + for _, dependencyPath := range unit.Config.Dependencies.Paths { + canonicalPath, err := util.CanonicalPath(dependencyPath, unit.Path) + if err != nil { + return nil, err + } + + // Get the dependency unit from unitsToSkip first (it should be there from discovery) + externalDependency, found := unitsToSkip[canonicalPath] + if !found { + l.Debugf("Dependency %s of unit %s not found in unitsMap (may be excluded or outside discovery scope)", canonicalPath, unit.Path) + continue + } + + // Skip if not external (inside working directory) + // Convert both paths to absolute for proper comparison + absCanonicalPath, err := filepath.Abs(canonicalPath) + if err != nil { + return nil, err + } + + absWorkingDir, err := filepath.Abs(r.Stack.TerragruntOptions.WorkingDir) + if err != nil { + return nil, err + } + + if util.HasPathPrefix(absCanonicalPath, absWorkingDir) { + l.Debugf("Dependency %s is inside working directory, not treating as external", canonicalPath) + continue + } + + l.Debugf("Dependency %s is outside working directory, treating as external", canonicalPath) + + // Skip if already processed + if _, alreadyFound := allExternalDependencies[externalDependency.Path]; alreadyFound { + continue + } + + shouldApply := false + if !r.Stack.TerragruntOptions.IgnoreExternalDependencies { + shouldApply, err = r.confirmShouldApplyExternalDependency(ctx, unit, l, externalDependency, unitOpts) + if err != nil { + return nil, err + } + } + + externalDependency.AssumeAlreadyApplied = !shouldApply + // Mark external dependencies as excluded if they shouldn't be applied + // This ensures they are tracked in the report but not executed + if !shouldApply { + externalDependency.FlagExcluded = true + } + + allExternalDependencies[externalDependency.Path] = externalDependency + } + } + + if len(allExternalDependencies) > 0 { + recursiveDependencies, err := r.resolveExternalDependenciesForUnits(ctx, l, allExternalDependencies, unitsMap, recursionLevel+1) + if err != nil { + return allExternalDependencies, err + } + + return allExternalDependencies.MergeMaps(recursiveDependencies), nil + } + + return allExternalDependencies, nil +} + +// Confirm with the user whether they want Terragrunt to assume the given dependency of the given unit is already +// applied. If the user selects "yes", then Terragrunt will apply that unit as well. +// Note that we skip the prompt for `run --all destroy` calls. Given the destructive and irreversible nature of destroy, we don't +// want to provide any risk to the user of accidentally destroying an external dependency unless explicitly included +// with the --queue-include-external or --queue-include-dir flags. +func (r *UnitResolver) confirmShouldApplyExternalDependency(ctx context.Context, unit *Unit, l log.Logger, dependency *Unit, opts *options.TerragruntOptions) (bool, error) { + if opts.IncludeExternalDependencies { + l.Debugf("The --queue-include-external flag is set, so automatically including all external dependencies, and will run this command against unit %s, which is a dependency of unit %s.", dependency.Path, unit.Path) + return true, nil + } + + if opts.NonInteractive { + l.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run --all command, will not run this command against unit %s, which is a dependency of unit %s.", dependency.Path, unit.Path) + return false, nil + } + + stackCmd := opts.TerraformCommand + if stackCmd == "destroy" { + l.Debugf("run --all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run --all command, will not run this command against unit %s, which is a dependency of unit %s.", dependency.Path, unit.Path) + return false, nil + } + + l.Infof("Unit %s has external dependency %s", unit.Path, dependency.Path) + + return shell.PromptUserForYesNo(ctx, l, "Should Terragrunt apply the external dependency?", opts) +} + +// telemetryCrossLinkDependencies cross-links dependencies between units +func (r *UnitResolver) telemetryCrossLinkDependencies(ctx context.Context, unitsMap, externalDependencies UnitsMap, canonicalTerragruntConfigPaths []string) (Units, error) { + var crossLinkedUnits Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "crosslink_dependencies", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + }, func(_ context.Context) error { + result, err := unitsMap.MergeMaps(externalDependencies).CrossLinkDependencies(canonicalTerragruntConfigPaths) + if err != nil { + return err + } + + crossLinkedUnits = result + + return nil + }) + + return crossLinkedUnits, err +} diff --git a/internal/runner/common/unit_resolver_filtering.go b/internal/runner/common/unit_resolver_filtering.go new file mode 100644 index 0000000000..a45e86b29c --- /dev/null +++ b/internal/runner/common/unit_resolver_filtering.go @@ -0,0 +1,446 @@ +package common + +import ( + "context" + "path/filepath" + "slices" + + "github.com/gobwas/glob" + "github.com/gruntwork-io/terragrunt/internal/report" + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/pkg/log" + "github.com/gruntwork-io/terragrunt/telemetry" + "github.com/gruntwork-io/terragrunt/util" +) + +// reportUnitExclusion records a unit exclusion in the report with proper error handling. +// Handles path normalization, duplicate prevention, and error logging. +func (r *UnitResolver) reportUnitExclusion(l log.Logger, unitPath string, reason report.Reason) { + if r.Stack.Report == nil { + return + } + + // Ensure path is absolute for consistent reporting + absPath := unitPath + if !filepath.IsAbs(absPath) { + p, err := filepath.Abs(unitPath) + if err != nil { + l.Errorf("Error getting absolute path for unit %s: %v", unitPath, err) + return + } + + absPath = p + } + + absPath = util.CleanPath(absPath) + + run, err := r.Stack.Report.EnsureRun(absPath) + if err != nil { + l.Errorf("Error ensuring run for unit %s: %v", absPath, err) + return + } + + if err := r.Stack.Report.EndRun( + run.Path, + report.WithResult(report.ResultExcluded), + report.WithReason(reason), + ); err != nil { + l.Errorf("Error ending run for unit %s: %v", absPath, err) + return + } +} + +// createPathMatcherFunc builds a matcher for include/exclude patterns. +// Why: centralizes path matching used by CLI flags and config. +// Matching: glob when doubleStarEnabled; otherwise exact path prefix. +// Mode: "include" uses include globs/dirs; "exclude" uses exclude globs/dirs. +// Examples: "**/staging/**", "modules/*/test", "envs/prod". +// Returns: func(*Unit) bool that is true when the unit matches. +func (r *UnitResolver) createPathMatcherFunc(mode string, opts *options.TerragruntOptions, l log.Logger) func(*Unit) bool { + // Use glob matching when double-star is enabled, otherwise use exact path matching + if r.doubleStarEnabled { + var ( + globs map[string]glob.Glob + action string + ) + + if mode == "include" { + globs = r.includeGlobs + action = "included" + } else { + globs = r.excludeGlobs + action = "excluded" + } + + return func(unit *Unit) bool { + for globPath, globPattern := range globs { + if globPattern.Match(unit.Path) { + l.Debugf("Unit %s is %s by glob %s", unit.Path, action, globPath) + return true + } + } + + return false + } + } + + // Fallback to exact path matching when double-star is not enabled (backwards compatibility) + var ( + dirs []string + action string + ) + + if mode == "include" { + dirs = opts.IncludeDirs + action = "included" + } else { + dirs = opts.ExcludeDirs + action = "excluded" + } + + return func(unit *Unit) bool { + for _, dir := range dirs { + if util.HasPathPrefix(unit.Path, dir) { + l.Debugf("Unit %s is %s by exact path match %s", unit.Path, action, dir) + return true + } + } + + return false + } +} + +// telemetryFlagIncludedDirs flags directories that are included in the Terragrunt configuration +func (r *UnitResolver) telemetryFlagIncludedDirs(ctx context.Context, l log.Logger, crossLinkedUnits Units) (Units, error) { + var withUnitsIncluded Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_included_dirs", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + }, func(_ context.Context) error { + withUnitsIncluded = r.flagIncludedDirs(r.Stack.TerragruntOptions, l, crossLinkedUnits) + return nil + }) + + return withUnitsIncluded, err +} + +// flagIncludedDirs includes units when --terragrunt-include-dir is used (ExcludeByDefault). +// Why: invert default behavior to run only requested units; optionally include deps unless StrictInclude. +// Matching: glob when doubleStarEnabled; otherwise exact path prefix. +// Behavior: no-op when ExcludeByDefault is false. +// Examples: "**/prod/**", "apps/*/service-a", "envs/us-west-2". +func (r *UnitResolver) flagIncludedDirs(opts *options.TerragruntOptions, l log.Logger, units Units) Units { + if !opts.ExcludeByDefault { + return units + } + + includeFn := r.createPathMatcherFunc("include", opts, l) + + for _, unit := range units { + unit.FlagExcluded = true + if includeFn(unit) { + unit.FlagExcluded = false + } + } + + // Mark all affected dependencies as included before proceeding if not in strict include mode. + if !opts.StrictInclude { + for _, unit := range units { + if !unit.FlagExcluded { + for _, dependency := range unit.Dependencies { + dependency.FlagExcluded = false + } + } + } + } + + return units +} + +// telemetryFlagUnitsThatAreIncluded flags units that are included in the Terragrunt configuration +func (r *UnitResolver) telemetryFlagUnitsThatAreIncluded(ctx context.Context, withUnitsIncluded Units) (Units, error) { + var withUnitsThatAreIncludedByOthers Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_units_that_are_included", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + }, func(_ context.Context) error { + result, err := r.flagUnitsThatAreIncluded(r.Stack.TerragruntOptions, withUnitsIncluded) + if err != nil { + return err + } + + withUnitsThatAreIncludedByOthers = result + + return nil + }) + + return withUnitsThatAreIncludedByOthers, err +} + +// flagUnitsThatAreIncluded includes units that reference specific config files. +// Why: support --terragrunt-modules-that-include to target modules including given files. +// Behavior: canonicalize targets, check each unit and its dependencies' ProcessedIncludes; mark included when matched. +// Examples: "root.hcl", "region.hcl", "_common.hcl". +func (r *UnitResolver) flagUnitsThatAreIncluded(opts *options.TerragruntOptions, units Units) (Units, error) { + unitsThatInclude := append(opts.ModulesThatInclude, opts.UnitsReading...) //nolint:gocritic + + if len(unitsThatInclude) == 0 { + return units, nil + } + + unitsThatIncludeCanonicalPaths := []string{} + + for _, includePath := range unitsThatInclude { + canonicalPath, err := util.CanonicalPath(includePath, opts.WorkingDir) + if err != nil { + return nil, err + } + + unitsThatIncludeCanonicalPaths = append(unitsThatIncludeCanonicalPaths, canonicalPath) + } + + for _, unit := range units { + if err := r.flagUnitIncludes(unit, unitsThatIncludeCanonicalPaths); err != nil { + return nil, err + } + + if err := r.flagUnitDependencies(unit, unitsThatIncludeCanonicalPaths); err != nil { + return nil, err + } + } + + return units, nil +} + +// flagUnitIncludes marks a unit as included if any of its include paths match the canonical paths. +// Returns an error if path resolution fails during the comparison. +func (r *UnitResolver) flagUnitIncludes(unit *Unit, canonicalPaths []string) error { + for _, includeConfig := range unit.Config.ProcessedIncludes { + canonicalPath, err := util.CanonicalPath(includeConfig.Path, unit.Path) + if err != nil { + return err + } + + if util.ListContainsElement(canonicalPaths, canonicalPath) { + unit.FlagExcluded = false + } + } + + return nil +} + +// flagUnitDependencies processes dependencies of a unit and flags them based on include paths. +// Returns an error if dependency processing fails. +func (r *UnitResolver) flagUnitDependencies(unit *Unit, canonicalPaths []string) error { + for _, dependency := range unit.Dependencies { + if dependency.FlagExcluded { + continue + } + + if err := r.flagDependencyIncludes(dependency, unit.Path, canonicalPaths); err != nil { + return err + } + } + + return nil +} + +// flagDependencyIncludes marks a dependency as included if any of its include paths match the canonical paths. +// Returns an error if path resolution fails during the comparison. +func (r *UnitResolver) flagDependencyIncludes(dependency *Unit, unitPath string, canonicalPaths []string) error { + for _, includeConfig := range dependency.Config.ProcessedIncludes { + canonicalPath, err := util.CanonicalPath(includeConfig.Path, unitPath) + if err != nil { + return err + } + + if util.ListContainsElement(canonicalPaths, canonicalPath) { + dependency.FlagExcluded = false + } + } + + return nil +} + +// telemetryFlagUnitsThatRead flags units that read files in the Terragrunt configuration +func (r *UnitResolver) telemetryFlagUnitsThatRead(ctx context.Context, withExcludedUnits Units) (Units, error) { + var withUnitsRead Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_units_that_read", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + }, func(_ context.Context) error { + withUnitsRead = r.flagUnitsThatRead(r.Stack.TerragruntOptions, withExcludedUnits) + return nil + }) + + return withUnitsRead, err +} + +// flagUnitsThatRead iterates over a unit slice and flags all units that read at least one file in the specified +// file list in the TerragruntOptions UnitsReading attribute. +func (r *UnitResolver) flagUnitsThatRead(opts *options.TerragruntOptions, units Units) Units { + // If no UnitsThatRead is specified, return the unit list instantly + if len(opts.UnitsReading) == 0 { + return units + } + + for _, path := range opts.UnitsReading { + if !filepath.IsAbs(path) { + path = filepath.Join(opts.WorkingDir, path) + path = filepath.Clean(path) + } + + for _, unit := range units { + if slices.Contains(unit.Reading, path) { + unit.FlagExcluded = false + } + } + } + + return units +} + +// telemetryFlagExcludedDirs flags directories that are excluded in the Terragrunt configuration +func (r *UnitResolver) telemetryFlagExcludedDirs(ctx context.Context, l log.Logger, withUnitsRead Units) (Units, error) { + var withUnitsExcluded Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_excluded_dirs", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + }, func(_ context.Context) error { + withUnitsExcluded = r.flagExcludedDirs(l, r.Stack.TerragruntOptions, r.Stack.Report, withUnitsRead) + return nil + }) + + return withUnitsExcluded, err +} + +// flagExcludedDirs excludes units that match --terragrunt-exclude-dir patterns. +// Why: enforce explicit user exclusions with highest precedence and preserve exclusion reasons in reports. +// Matching: uses glob patterns when doubleStarEnabled; otherwise exact path prefix matching. +// Examples: +// - "**/staging/**" +// - "modules/*/test" +// - "envs/prod" +func (r *UnitResolver) flagExcludedDirs(l log.Logger, opts *options.TerragruntOptions, reportInstance *report.Report, units Units) Units { + // If we don't have any excludes, we don't need to do anything. + if (len(r.excludeGlobs) == 0 && r.doubleStarEnabled) || len(opts.ExcludeDirs) == 0 { + return units + } + + excludeFn := r.createPathMatcherFunc("exclude", opts, l) + + for _, unit := range units { + if excludeFn(unit) { + // Mark unit itself as excluded + unit.FlagExcluded = true + + // Only update report if it's enabled + if reportInstance != nil { + r.reportUnitExclusion(l, unit.Path, report.ReasonExcludeDir) + } + } + + // Mark all affected dependencies as excluded + for _, dependency := range unit.Dependencies { + if excludeFn(dependency) { + dependency.FlagExcluded = true + + // Only update report if it's enabled + if reportInstance != nil { + r.reportUnitExclusion(l, dependency.Path, report.ReasonExcludeDir) + } + } + } + } + + return units +} + +// telemetryFlagExcludedUnits flags units that are excluded in the Terragrunt configuration +func (r *UnitResolver) telemetryFlagExcludedUnits(ctx context.Context, l log.Logger, withUnitsThatAreIncludedByOthers Units) (Units, error) { + var withExcludedUnits Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "flag_excluded_units", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + }, func(_ context.Context) error { + result := r.flagExcludedUnits(l, r.Stack.TerragruntOptions, r.Stack.Report, withUnitsThatAreIncludedByOthers) + withExcludedUnits = result + + return nil + }) + + return withExcludedUnits, err +} + +// flagExcludedUnits iterates over a unit slice and flags all units that are excluded based on the exclude block. +func (r *UnitResolver) flagExcludedUnits(l log.Logger, opts *options.TerragruntOptions, reportInstance *report.Report, units Units) Units { + for _, unit := range units { + excludeConfig := unit.Config.Exclude + + if excludeConfig == nil { + continue + } + + if !excludeConfig.IsActionListed(opts.TerraformCommand) { + continue + } + + if excludeConfig.If { + // Check if unit was already excluded (e.g., by --queue-exclude-dir) + // If so, don't overwrite the existing exclusion reason + wasAlreadyExcluded := unit.FlagExcluded + unit.FlagExcluded = true + + // Only update report if it's enabled AND the unit wasn't already excluded + // This ensures CLI flags like --queue-exclude-dir take precedence over exclude blocks + if reportInstance != nil && !wasAlreadyExcluded { + r.reportUnitExclusion(l, unit.Path, report.ReasonExcludeBlock) + } + } + + if excludeConfig.ExcludeDependencies != nil && *excludeConfig.ExcludeDependencies { + l.Debugf("Excluding dependencies for unit %s by exclude block", unit.Path) + + for _, dependency := range unit.Dependencies { + // Check if dependency was already excluded + wasAlreadyExcluded := dependency.FlagExcluded + dependency.FlagExcluded = true + + // Only update report if it's enabled AND the dependency wasn't already excluded + // This ensures CLI exclusions take precedence over exclude blocks + if reportInstance != nil && !wasAlreadyExcluded { + r.reportUnitExclusion(l, dependency.Path, report.ReasonExcludeBlock) + } + } + } + } + + return units +} + +// telemetryApplyFilters applies all configured unit filters to the resolved units +func (r *UnitResolver) telemetryApplyFilters(ctx context.Context, units Units) (Units, error) { + if len(r.filters) == 0 { + return units, nil + } + + var filteredUnits Units + + err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "apply_unit_filters", map[string]any{ + "working_dir": r.Stack.TerragruntOptions.WorkingDir, + "filter_count": len(r.filters), + }, func(ctx context.Context) error { + // Apply all filters in sequence + for _, filter := range r.filters { + if err := filter.Filter(ctx, units, r.Stack.TerragruntOptions); err != nil { + return err + } + } + + filteredUnits = units + + return nil + }) + + return filteredUnits, err +} diff --git a/internal/runner/common/unit_resolver_helpers.go b/internal/runner/common/unit_resolver_helpers.go new file mode 100644 index 0000000000..2cfccca533 --- /dev/null +++ b/internal/runner/common/unit_resolver_helpers.go @@ -0,0 +1,48 @@ +package common + +import ( + "path/filepath" + + "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/options" + "github.com/gruntwork-io/terragrunt/pkg/log" + "github.com/gruntwork-io/terragrunt/util" +) + +// resolveUnitPath converts a Terragrunt configuration file path to its corresponding unit path. +// Returns the canonical path of the directory containing the config file. +func (r *UnitResolver) resolveUnitPath(terragruntConfigPath string) (string, error) { + return util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".") +} + +// setupDownloadDir sets the unit's download directory. +// If the stack uses the default dir, compute a per-unit dir; otherwise use the stack's setting. +func (r *UnitResolver) setupDownloadDir(terragruntConfigPath string, opts *options.TerragruntOptions, l log.Logger) error { + _, defaultDownloadDir, err := options.DefaultWorkingAndDownloadDirs(r.Stack.TerragruntOptions.TerragruntConfigPath) + if err != nil { + return err + } + + if r.Stack.TerragruntOptions.DownloadDir == defaultDownloadDir { + _, downloadDir, err := options.DefaultWorkingAndDownloadDirs(terragruntConfigPath) + if err != nil { + return err + } + + l.Debugf("Setting download directory for unit %s to %s", filepath.Dir(opts.TerragruntConfigPath), downloadDir) + opts.DownloadDir = downloadDir + } + + return nil +} + +// determineTerragruntConfigFilename returns the config filename to use. +// If a file path is explicitly set, it uses its basename; otherwise, "terragrunt.hcl". +func (r *UnitResolver) determineTerragruntConfigFilename() string { + fname := config.DefaultTerragruntConfigPath + if r.Stack.TerragruntOptions.TerragruntConfigPath != "" && !util.IsDir(r.Stack.TerragruntOptions.TerragruntConfigPath) { + fname = filepath.Base(r.Stack.TerragruntOptions.TerragruntConfigPath) + } + + return fname +} diff --git a/internal/runner/common/unit_resolver_test.go b/internal/runner/common/unit_resolver_test.go new file mode 100644 index 0000000000..7a80898420 --- /dev/null +++ b/internal/runner/common/unit_resolver_test.go @@ -0,0 +1,56 @@ +package common_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/internal/component" + "github.com/gruntwork-io/terragrunt/internal/runner/common" + "github.com/gruntwork-io/terragrunt/options" + thlogger "github.com/gruntwork-io/terragrunt/test/helpers/logger" +) + +func TestResolveFromDiscovery_UsesDiscoveryConfig(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + // Create a discovery unit with a pre-parsed config including Terraform.source + src := "./module" + tgCfg := &config.TerragruntConfig{ + Terraform: &config.TerraformConfig{Source: &src}, + } + + discUnit := component.NewUnit(tmpDir) + discUnit.WithConfig(tgCfg) + + discovered := component.Components{discUnit} + + // Prepare options and stack (ensure config file exists) + tgPath := filepath.Join(tmpDir, "terragrunt.hcl") + require.NoError(t, os.WriteFile(tgPath, []byte(""), 0o600)) + + opts, err := options.NewTerragruntOptionsForTest(tgPath) + require.NoError(t, err) + + // Quiet test logging with non-nil formatter + l := thlogger.CreateLogger() + + stack := &common.Stack{TerragruntOptions: opts} + resolver, err := common.NewUnitResolver(context.Background(), stack) + require.NoError(t, err) + + units, err := resolver.ResolveFromDiscovery(context.Background(), l, discovered) + require.NoError(t, err) + + require.Len(t, units, 1) + require.Equal(t, tmpDir, units[0].Path) + require.NotNil(t, units[0].Config.Terraform) + require.NotNil(t, units[0].Config.Terraform.Source) + require.Equal(t, src, *units[0].Config.Terraform.Source) +} diff --git a/internal/runner/common/unit_runner.go b/internal/runner/common/unit_runner.go index 2d1e063ae5..fa9fec3fac 100644 --- a/internal/runner/common/unit_runner.go +++ b/internal/runner/common/unit_runner.go @@ -93,16 +93,7 @@ func (runner *UnitRunner) runTerragrunt(ctx context.Context, opts *options.Terra // End the run with appropriate result (only if report is not nil) if r != nil { // Get the unit path (already computed above) - unitPath := runner.Unit.Path - if !filepath.IsAbs(unitPath) { - p, absErr := filepath.Abs(unitPath) - if absErr != nil { - runner.Unit.Logger.Errorf("Error getting absolute path for unit %s: %v", runner.Unit.Path, absErr) - } else { - unitPath = p - } - } - + unitPath := runner.Unit.AbsolutePath(runner.Unit.Logger) unitPath = util.CleanPath(unitPath) if runErr != nil { diff --git a/internal/runner/runnerpool/builder.go b/internal/runner/runnerpool/builder.go index e0b36e563e..4aa00c358f 100644 --- a/internal/runner/runnerpool/builder.go +++ b/internal/runner/runnerpool/builder.go @@ -28,6 +28,23 @@ func Build( workingDir = terragruntOptions.WorkingDir } + // Build config filenames list - include defaults plus any custom config file + configFilenames := append([]string{}, discovery.DefaultConfigFilenames...) + customConfigName := filepath.Base(terragruntOptions.TerragruntConfigPath) + // Only add custom config if it's different from defaults + isCustom := true + + for _, defaultName := range discovery.DefaultConfigFilenames { + if customConfigName == defaultName { + isCustom = false + break + } + } + + if isCustom && customConfigName != "" && customConfigName != "." { + configFilenames = append(configFilenames, customConfigName) + } + d := discovery. NewDiscovery(workingDir). WithOptions(opts...). @@ -37,36 +54,33 @@ func Build( WithParseExclude(). WithDiscoverDependencies(). WithSuppressParseErrors(). - WithConfigFilenames([]string{filepath.Base(terragruntOptions.TerragruntConfigPath)}). + WithConfigFilenames(configFilenames). WithDiscoveryContext(&component.DiscoveryContext{ Cmd: terragruntOptions.TerraformCliArgs.First(), Args: terragruntOptions.TerraformCliArgs.Tail(), }) - // Pass include directory filters + // Pass include directory filters to discovery + // Discovery will use glob matching to filter units appropriately if len(terragruntOptions.IncludeDirs) > 0 { d = d.WithIncludeDirs(terragruntOptions.IncludeDirs) } - // Don't pass ExcludeDirs to discovery - let all units be discovered - // and then filter/report them during unit resolution where we have access to the report - // if len(terragruntOptions.ExcludeDirs) > 0 { - // d = d.WithExcludeDirs(terragruntOptions.ExcludeDirs) - // } + // NOTE: We do NOT pass ExcludeDirs to discovery because excluded units need to be + // discovered and reported (for --report-file functionality). The unit resolver will + // handle exclusions after discovery, ensuring excluded units appear in reports. // Pass include behavior flags if terragruntOptions.StrictInclude { d = d.WithStrictInclude() } - // We intentionally do NOT set ExcludeByDefault during discovery, even if it's enabled in options. - // The filtering will happen later in the unit resolver after all modules have been discovered. - // This ensures that dependency resolution works correctly and units aren't prematurely excluded. + // Note: Discovery will use glob-based filtering for include patterns. + // Exclude patterns are handled by the unit resolver to ensure proper reporting. - // Pass dependency behavior flags - if terragruntOptions.IgnoreExternalDependencies { - d = d.WithIgnoreExternalDependencies() - } + // We do NOT use WithIgnoreExternalDependencies() even if IgnoreExternalDependencies is set. + // External dependencies need to be discovered so they can be included in the dependency graph. + // They will be marked as excluded (AssumeAlreadyApplied) in resolveExternalDependenciesForUnits. // Apply filter queries if the filter-flag experiment is enabled if terragruntOptions.Experiments.Evaluate(experiment.FilterFlag) && len(terragruntOptions.FilterQueries) > 0 { diff --git a/internal/runner/runnerpool/controller.go b/internal/runner/runnerpool/controller.go index 53bb94cb7d..9a5b94804f 100644 --- a/internal/runner/runnerpool/controller.go +++ b/internal/runner/runnerpool/controller.go @@ -112,7 +112,7 @@ func (dr *Controller) Run(ctx context.Context, l log.Logger) error { for _, e := range readyEntries { // log debug which entry is running - l.Debugf("Runner Pool Controller: running %s", e.Component.Path) + l.Debugf("Runner Pool Controller: running %s", e.Component.Path()) dr.q.SetEntryStatus(e, queue.StatusRunning) sem <- struct{}{} @@ -133,7 +133,7 @@ func (dr *Controller) Run(ctx context.Context, l log.Logger) error { unit := dr.unitsMap[ent.Component.Path()] if unit == nil { err := errors.Errorf("unit for path %s not found in discovered units", ent.Component.Path()) - l.Errorf("Runner Pool Controller: unit for path %s not found in discovered units, skipping execution", ent.Component.Path) + l.Errorf("Runner Pool Controller: unit for path %s not found in discovered units, skipping execution", ent.Component.Path()) dr.q.FailEntry(ent) results.Store(ent.Component.Path(), err) @@ -144,13 +144,13 @@ func (dr *Controller) Run(ctx context.Context, l log.Logger) error { results.Store(ent.Component.Path(), err) if err != nil { - l.Debugf("Runner Pool Controller: %s failed", ent.Component.Path) + l.Debugf("Runner Pool Controller: %s failed", ent.Component.Path()) dr.q.FailEntry(ent) return } - l.Debugf("Runner Pool Controller: %s succeeded", ent.Component.Path) + l.Debugf("Runner Pool Controller: %s succeeded", ent.Component.Path()) dr.q.SetEntryStatus(ent, queue.StatusSucceeded) }(e) } diff --git a/internal/runner/runnerpool/runner.go b/internal/runner/runnerpool/runner.go index 353c1a0c25..748844ae62 100644 --- a/internal/runner/runnerpool/runner.go +++ b/internal/runner/runnerpool/runner.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -13,6 +14,7 @@ import ( "strings" "github.com/gruntwork-io/go-commons/collections" + "github.com/gruntwork-io/terragrunt/internal/cli" "github.com/gruntwork-io/terragrunt/internal/runner/common" "github.com/gruntwork-io/terragrunt/tf" @@ -22,11 +24,13 @@ import ( "github.com/gruntwork-io/terragrunt/config" "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/internal/component" + tgerrors "github.com/gruntwork-io/terragrunt/internal/errors" "github.com/gruntwork-io/terragrunt/internal/queue" "github.com/gruntwork-io/terragrunt/internal/report" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/pkg/log" "github.com/gruntwork-io/terragrunt/telemetry" + "github.com/hashicorp/hcl/v2" ) // Runner implements the Stack interface for runner pool execution. @@ -50,7 +54,16 @@ func NewRunnerPoolStack( discovered component.Components, opts ...common.Option, ) (common.StackRunner, error) { - if len(discovered) == 0 { + // Filter out Stack components - we only want Unit components + // Stack components (terragrunt.stack.hcl files) are for stack generation, not execution + nonStackComponents := make(component.Components, 0, len(discovered)) + for _, c := range discovered { + if c.Kind() != component.StackKind { + nonStackComponents = append(nonStackComponents, c) + } + } + + if len(nonStackComponents) == 0 { l.Warnf("No units discovered. Creating an empty runner.") stack := common.Stack{ @@ -87,38 +100,6 @@ func NewRunnerPoolStack( // the report is available during unit resolution for tracking exclusions runner = runner.WithOptions(opts...) - // Collect all terragrunt.hcl paths for resolution. - unitPaths := make([]string, 0, len(discovered)) - - for _, c := range discovered { - unit, ok := c.(*component.Unit) - if !ok { - continue - } - - if unit.Config() == nil { - // Skip configurations that could not be parsed - l.Warnf("Skipping unit at %s due to parse error", c.Path()) - continue - } - - // Determine per-unit config filename - // - // TODO: Refactor this out later. - var fname string - if c.Kind() == component.StackKind { - fname = config.DefaultStackFile - } else { - fname = config.DefaultTerragruntConfigPath - if terragruntOptions.TerragruntConfigPath != "" && !util.IsDir(terragruntOptions.TerragruntConfigPath) { - fname = filepath.Base(terragruntOptions.TerragruntConfigPath) - } - } - - terragruntConfigPath := filepath.Join(unit.Path(), fname) - unitPaths = append(unitPaths, terragruntConfigPath) - } - // Resolve units (this applies to include/exclude logic and sets FlagExcluded accordingly). unitResolver, err := common.NewUnitResolver(ctx, runner.Stack) if err != nil { @@ -130,17 +111,16 @@ func NewRunnerPoolStack( unitResolver = unitResolver.WithFilters(runner.unitFilters...) } - unitsMap, err := unitResolver.ResolveTerraformModules(ctx, l, unitPaths) + // Use discovery-based resolution (no legacy fallback needed since discovery parses all required blocks) + // Use nonStackComponents which has Stack components filtered out + unitsMap, err := unitResolver.ResolveFromDiscovery(ctx, l, nonStackComponents) if err != nil { return nil, err } runner.Stack.Units = unitsMap - // Handle prevent_destroy logic for destroy operations - // If running destroy, exclude units with prevent_destroy=true and their dependencies if isDestroyCommand(terragruntOptions) { - l.Debugf("Detected destroy command, applying prevent_destroy exclusions") applyPreventDestroyExclusions(l, unitsMap) } @@ -158,6 +138,54 @@ func NewRunnerPoolStack( return runner.WithOptions(opts...), nil } +// Limit recursive descent when inspecting nested errors +const maxConfigurationErrorDepth = 100 + +// isConfigurationError checks if an error is a configuration/validation error +// that should always cause command failure regardless of fail-fast setting. +func isConfigurationError(err error) bool { + return isConfigurationErrorDepth(err, 0) +} + +func isConfigurationErrorDepth(err error, depth int) bool { + if err == nil { + return false + } + + if depth >= maxConfigurationErrorDepth { + return false + } + + // Check for specific configuration error types + if tgerrors.IsError(err, config.ConflictingRunCmdCacheOptionsError{}) { + return true + } + + // Inspect HCL diagnostics (structured errors) for run_cmd cache-option conflicts + for _, unwrapped := range tgerrors.UnwrapErrors(err) { + var diags hcl.Diagnostics + if errors.As(unwrapped, &diags) { + for _, d := range diags { + if d != nil && d.Severity == hcl.DiagError && d.Summary == "Error in function call" { + return true + } + } + } + } + + // Check wrapped errors in MultiError + var multiErr *tgerrors.MultiError + if errors.As(err, &multiErr) { + for _, wrappedErr := range multiErr.WrappedErrors() { + if isConfigurationErrorDepth(wrappedErr, depth+1) { + return true + } + } + } + + return false +} + // Run executes the stack according to TerragruntOptions and returns the first // error (or a joined error) once execution is finished. func (r *Runner) Run(ctx context.Context, l log.Logger, opts *options.TerragruntOptions) error { @@ -327,6 +355,28 @@ func (r *Runner) Run(ctx context.Context, l log.Logger, opts *options.Terragrunt } } + // Handle errors based on fail-fast mode and error type + // Configuration errors always fail regardless of --fail-fast + // Execution errors are suppressed when --fail-fast is not set + if err != nil { + if isConfigurationError(err) || opts.FailFast { + // Configuration errors or fail-fast mode: propagate error + return err + } + + // Execution errors without fail-fast: log but don't fail + l.Errorf("Run failed: %v", err) + + // Set detailed exit code if context has one + exitCode := tf.DetailedExitCodeFromContext(ctx) + if exitCode != nil { + exitCode.Set(int(cli.ExitCodeGeneralError)) + } + + // Return nil to indicate success (no --fail-fast) but errors were logged + return nil + } + return err } diff --git a/internal/runner/runnerpool/runner_test.go b/internal/runner/runnerpool/runner_test.go new file mode 100644 index 0000000000..fcec8ff11a --- /dev/null +++ b/internal/runner/runnerpool/runner_test.go @@ -0,0 +1,47 @@ +package runnerpool_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/gruntwork-io/terragrunt/config" + "github.com/gruntwork-io/terragrunt/internal/component" + "github.com/gruntwork-io/terragrunt/internal/runner/common" + "github.com/gruntwork-io/terragrunt/options" + thlogger "github.com/gruntwork-io/terragrunt/test/helpers/logger" +) + +func TestDiscoveryResolverMatchesLegacyPaths(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + // Create a trivial tf file so the resolver doesn't skip the unit + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "main.tf"), []byte(""), 0o600)) + tgPath := filepath.Join(tmpDir, "terragrunt.hcl") + require.NoError(t, os.WriteFile(tgPath, []byte(""), 0o600)) + + // Discovery produces a component with or without config; using empty config is fine here + discUnit := component.NewUnit(tmpDir).WithConfig(&config.TerragruntConfig{}) + discovered := component.Components{discUnit} + + // Stack and resolver + opts, err := options.NewTerragruntOptionsForTest(tgPath) + require.NoError(t, err) + + stack := &common.Stack{TerragruntOptions: opts} + resolver, err := common.NewUnitResolver(context.Background(), stack) + require.NoError(t, err) + + l := thlogger.CreateLogger() + + // Verify discovery-based resolution works correctly + fromDiscovery, err := resolver.ResolveFromDiscovery(context.Background(), l, discovered) + require.NoError(t, err) + require.Len(t, fromDiscovery, 1) + require.Equal(t, tmpDir, fromDiscovery[0].Path) +} diff --git a/test/fixtures/auth-provider-parallel/auth-provider.sh b/test/fixtures/auth-provider-parallel/auth-provider.sh new file mode 100755 index 0000000000..c4de5a7210 --- /dev/null +++ b/test/fixtures/auth-provider-parallel/auth-provider.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Mock auth provider that uses file-based coordination to verify parallel execution + +set -e + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOCK_DIR="${SCRIPT_DIR}/.auth-locks" +mkdir -p "$LOCK_DIR" + +# Get a unique ID for this invocation +# Use POSIX-compatible timestamp (seconds) + PID + RANDOM to ensure uniqueness +# This works on Linux, macOS, and BSD without requiring nanosecond precision +INVOCATION_ID="auth-$$-$(date +%s)-$RANDOM" + +# Log to stderr so it shows up in terragrunt output +echo "Auth start ${INVOCATION_ID}" >&2 + +# Create a lock file to indicate we've started +touch "${LOCK_DIR}/start-${INVOCATION_ID}" + +# Wait for other auth commands to also start (up to 500ms) +# This ensures we test the parallel execution scenario +WAIT_COUNT=0 +MAX_WAIT=50 # 50 * 10ms = 500ms max wait + +while [ $WAIT_COUNT -lt $MAX_WAIT ]; do + # Count how many auth commands have started + STARTED=$(ls -1 "${LOCK_DIR}"/start-* 2>/dev/null | wc -l | tr -d ' \t') + + # If we see at least 2 others started (3 total), we know it's parallel + if [ "$STARTED" -ge 2 ]; then + echo "Auth concurrent ${INVOCATION_ID} detected=$STARTED" >&2 + break + fi + + # Sleep a bit and check again + sleep 0.01 + WAIT_COUNT=$((WAIT_COUNT + 1)) +done + +# Simulate some auth work +sleep 0.1 + +# Return fake credentials as JSON +cat <&2 diff --git a/test/fixtures/auth-provider-parallel/unit-a/main.tf b/test/fixtures/auth-provider-parallel/unit-a/main.tf new file mode 100644 index 0000000000..761057be9e --- /dev/null +++ b/test/fixtures/auth-provider-parallel/unit-a/main.tf @@ -0,0 +1,3 @@ +output "unit_name" { + value = "unit-a" +} diff --git a/test/fixtures/auth-provider-parallel/unit-a/terragrunt.hcl b/test/fixtures/auth-provider-parallel/unit-a/terragrunt.hcl new file mode 100644 index 0000000000..fa47c95034 --- /dev/null +++ b/test/fixtures/auth-provider-parallel/unit-a/terragrunt.hcl @@ -0,0 +1 @@ +# Unit A - no dependencies diff --git a/test/fixtures/auth-provider-parallel/unit-b/main.tf b/test/fixtures/auth-provider-parallel/unit-b/main.tf new file mode 100644 index 0000000000..69d00eb4a8 --- /dev/null +++ b/test/fixtures/auth-provider-parallel/unit-b/main.tf @@ -0,0 +1,3 @@ +output "unit_name" { + value = "unit-b" +} diff --git a/test/fixtures/auth-provider-parallel/unit-b/terragrunt.hcl b/test/fixtures/auth-provider-parallel/unit-b/terragrunt.hcl new file mode 100644 index 0000000000..522b033164 --- /dev/null +++ b/test/fixtures/auth-provider-parallel/unit-b/terragrunt.hcl @@ -0,0 +1 @@ +# Unit B - no dependencies diff --git a/test/fixtures/auth-provider-parallel/unit-c/main.tf b/test/fixtures/auth-provider-parallel/unit-c/main.tf new file mode 100644 index 0000000000..534270c812 --- /dev/null +++ b/test/fixtures/auth-provider-parallel/unit-c/main.tf @@ -0,0 +1,3 @@ +output "unit_name" { + value = "unit-c" +} diff --git a/test/fixtures/auth-provider-parallel/unit-c/terragrunt.hcl b/test/fixtures/auth-provider-parallel/unit-c/terragrunt.hcl new file mode 100644 index 0000000000..de12d0b3d0 --- /dev/null +++ b/test/fixtures/auth-provider-parallel/unit-c/terragrunt.hcl @@ -0,0 +1 @@ +# Unit C - no dependencies diff --git a/test/integration_download_test.go b/test/integration_download_test.go index 991cd8a757..2c6db987bd 100644 --- a/test/integration_download_test.go +++ b/test/integration_download_test.go @@ -359,11 +359,6 @@ I'm not sure we're getting good value from the time taken on tests like this. func TestIncludeDirs(t *testing.T) { t.Parallel() - tmpDir := helpers.CopyEnvironment(t, testFixtureLocalWithIncludeDir) - workingDir := util.JoinPath(tmpDir, testFixtureLocalWithIncludeDir) - workingDir, err := filepath.EvalSymlinks(workingDir) - require.NoError(t, err) - // Populate module paths. unitNames := []string{ "integration-env/aws/module-aws-a", @@ -374,72 +369,89 @@ func TestIncludeDirs(t *testing.T) { } testCases := []struct { + name string includeArgs string includedUnitOutputs []string }{ { + name: "no-match", includeArgs: "--queue-include-dir xyz", includedUnitOutputs: []string{}, }, { + name: "wildcard-aws", includeArgs: "--queue-include-dir */aws", includedUnitOutputs: []string{"Module GCE B", "Module GCE C", "Module GCE E"}, }, { + name: "production-and-gce-c", includeArgs: "--queue-include-dir production-env --queue-include-dir **/module-gce-c", includedUnitOutputs: []string{"Module GCE B", "Module AWS A"}, }, { + name: "specific-modules", includeArgs: "--queue-include-dir integration-env/gce/module-gce-b --queue-include-dir integration-env/gce/module-gce-c --queue-include-dir **/module-aws*", includedUnitOutputs: []string{"Module GCE E"}, }, } - unitPaths := make(map[string]string, len(unitNames)) - for _, unitName := range unitNames { - unitPaths[unitName] = util.JoinPath(testFixtureLocalWithIncludeDir, unitName) - } - for _, tc := range testCases { - applyAllStdout := bytes.Buffer{} - applyAllStderr := bytes.Buffer{} - - // Apply modules according to test cases - err := helpers.RunTerragruntCommand( - t, - fmt.Sprintf( - "terragrunt run --all apply --non-interactive --log-level trace --working-dir %s %s", - workingDir, tc.includeArgs, - ), - &applyAllStdout, - &applyAllStderr, - ) - require.NoError(t, err) + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Each subtest gets its own clean environment + // Copy the entire fixtures/download directory to ensure relative source paths work + tmpDir := helpers.CopyEnvironment(t, "fixtures/download") + workingDir := util.JoinPath(tmpDir, testFixtureLocalWithIncludeDir) + workingDir, err := filepath.EvalSymlinks(workingDir) + require.NoError(t, err) - helpers.LogBufferContentsLineByLine(t, applyAllStdout, "run --all apply stdout") - helpers.LogBufferContentsLineByLine(t, applyAllStderr, "run --all apply stderr") + unitPaths := make(map[string]string, len(unitNames)) + for _, unitName := range unitNames { + unitPaths[unitName] = util.JoinPath(workingDir, unitName) + } - // Check that the included module output is present - for _, modulePath := range unitPaths { - showStdout := bytes.Buffer{} - showStderr := bytes.Buffer{} + applyAllStdout := bytes.Buffer{} + applyAllStderr := bytes.Buffer{} + // Apply modules according to test case err = helpers.RunTerragruntCommand( t, - "terragrunt show --non-interactive --log-level trace --working-dir "+modulePath, - &showStdout, - &showStderr, + fmt.Sprintf( + "terragrunt run --all apply --non-interactive --log-level trace --working-dir %s %s", + workingDir, tc.includeArgs, + ), + &applyAllStdout, + &applyAllStderr, ) - helpers.LogBufferContentsLineByLine(t, showStdout, "show stdout for "+modulePath) - helpers.LogBufferContentsLineByLine(t, showStderr, "show stderr for "+modulePath) - require.NoError(t, err) - output := showStdout.String() - for _, includedUnitOutput := range tc.includedUnitOutputs { - assert.NotContains(t, output, includedUnitOutput) + helpers.LogBufferContentsLineByLine(t, applyAllStdout, "run --all apply stdout") + helpers.LogBufferContentsLineByLine(t, applyAllStderr, "run --all apply stderr") + + // Check that the included module output is present + for _, modulePath := range unitPaths { + showStdout := bytes.Buffer{} + showStderr := bytes.Buffer{} + + err = helpers.RunTerragruntCommand( + t, + "terragrunt show --non-interactive --log-level trace --working-dir "+modulePath, + &showStdout, + &showStderr, + ) + helpers.LogBufferContentsLineByLine(t, showStdout, "show stdout for "+modulePath) + helpers.LogBufferContentsLineByLine(t, showStderr, "show stderr for "+modulePath) + + require.NoError(t, err) + + output := showStdout.String() + for _, includedUnitOutput := range tc.includedUnitOutputs { + assert.NotContains(t, output, includedUnitOutput) + } } - } + }) } } diff --git a/test/integration_runner_pool_test.go b/test/integration_runner_pool_test.go index db1163c719..6bc00c7e62 100644 --- a/test/integration_runner_pool_test.go +++ b/test/integration_runner_pool_test.go @@ -1,7 +1,9 @@ package test_test import ( + "path/filepath" "regexp" + "strconv" "strings" "testing" @@ -16,6 +18,7 @@ const ( testFixtureMixedConfig = "fixtures/mixed-config" testFixtureFailFast = "fixtures/fail-fast" testFixtureRunnerPoolRemoteSource = "fixtures/runner-pool-remote-source" + testFixtureAuthProviderParallel = "fixtures/auth-provider-parallel" ) func TestRunnerPoolDiscovery(t *testing.T) { @@ -182,3 +185,58 @@ func TestRunnerPoolSourceMap(t *testing.T) { // Verify that source map values are used require.Contains(t, stderr, "configurations from git::ssh://git@github.com/gruntwork-io/terragrunt.git?ref=v0.85.0") } + +// TestAuthProviderParallelExecution verifies that --auth-provider-cmd is executed in parallel +// for multiple units during the resolution phase. +// +// The test works by: +// 1. Running terragrunt with --auth-provider-cmd pointing to a script that: +// - Creates lock files to coordinate between concurrent invocations +// - Detects when multiple auth commands are running simultaneously +// - Logs "Auth concurrent" when it detects parallel execution +// 2. Parsing the output to find "Auth concurrent" messages +// 3. Verifying that at least one auth command detected concurrent execution +// (which is deterministic proof of parallelism) +func TestAuthProviderParallelExecution(t *testing.T) { + t.Parallel() + + helpers.CleanupTerraformFolder(t, testFixtureAuthProviderParallel) + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureAuthProviderParallel) + testPath := util.JoinPath(tmpEnvPath, testFixtureAuthProviderParallel) + + authProviderScript := filepath.Join(testPath, "auth-provider.sh") + + _, stderr, err := helpers.RunTerragruntCommandWithOutput( + t, + "terragrunt run --all --non-interactive --auth-provider-cmd "+authProviderScript+" --working-dir "+testPath+" -- validate", + ) + require.NoError(t, err) + + startCount := strings.Count(stderr, "Auth start") + endCount := strings.Count(stderr, "Auth end") + + reConcurrent := regexp.MustCompile(`Auth concurrent.*detected=(\d+)`) + matches := reConcurrent.FindAllStringSubmatch(stderr, -1) + + maxConcurrent := 0 + + for _, match := range matches { + detected, convErr := strconv.Atoi(match[1]) + require.NoError(t, convErr, "Invalid detected count in stderr: %q", match[0]) + + if detected > maxConcurrent { + maxConcurrent = detected + } + + t.Logf("Auth command detected %d concurrent executions", detected) + } + + require.GreaterOrEqual(t, startCount, 2, "Expected at least 2 auth start events") + require.GreaterOrEqual(t, endCount, 2, "Expected at least 2 auth end events") + assert.GreaterOrEqual(t, len(matches), 1, + "Expected at least one auth command to detect concurrent execution. "+ + "This would prove parallel execution. If this fails, auth commands may be running sequentially.") + assert.GreaterOrEqual(t, maxConcurrent, 2, + "Expected auth commands to detect at least 2 concurrent executions. "+ + "Detected max concurrent: %d. This proves parallel execution.", maxConcurrent) +} diff --git a/util/file.go b/util/file.go index e3d5ab2ee5..00eaf7d6f5 100644 --- a/util/file.go +++ b/util/file.go @@ -111,13 +111,13 @@ func CompileGlobs(basePath string, globPaths ...string) (map[string]glob.Glob, e for _, globPath := range globPaths { canGlobPath, err := CanonicalPath(globPath, basePath) if err != nil { - errs = append(errs, fmt.Errorf("failed to canonicalize glob path %q: %w", globPath, err)) + errs = append(errs, errors.Errorf("failed to canonicalize glob path %q: %w", globPath, err)) continue } compiledGlob, err := glob.Compile(canGlobPath, '/') if err != nil { - errs = append(errs, fmt.Errorf("invalid glob pattern %q: %w", globPath, err)) + errs = append(errs, errors.Errorf("invalid glob pattern %q: %w", globPath, err)) continue } @@ -125,7 +125,7 @@ func CompileGlobs(basePath string, globPaths ...string) (map[string]glob.Glob, e } if len(errs) > 0 { - return compiledGlobs, fmt.Errorf("failed to compile some glob patterns: %w", errors.Join(errs...)) + return compiledGlobs, errors.Errorf("failed to compile some glob patterns: %w", errors.Join(errs...)) } return compiledGlobs, nil @@ -1055,7 +1055,7 @@ func WalkWithSymlinks(root string, externalWalkFn filepath.WalkFunc) error { // Convert the current physical path to a logical path relative to the walk root rel, err := filepath.Rel(pair.physical, currentPath) if err != nil { - return fmt.Errorf("failed to get relative path between %s and %s: %w", pair.physical, currentPath, err) + return errors.Errorf("failed to get relative path between %s and %s: %w", pair.physical, currentPath, err) } logicalPath := filepath.Join(pair.logical, rel) @@ -1099,7 +1099,7 @@ func WalkWithSymlinks(root string, externalWalkFn filepath.WalkFunc) error { realRoot, err := filepath.EvalSymlinks(root) if err != nil { - return fmt.Errorf("failed to get evaluate sym links for %s: %w", root, err) + return errors.Errorf("failed to get evaluate sym links for %s: %w", root, err) } // Start the walk from the root directory @@ -1112,7 +1112,7 @@ func WalkWithSymlinks(root string, externalWalkFn filepath.WalkFunc) error { func evalRealPathAndInfo(currentPath string) (string, os.FileInfo, error) { realPath, err := filepath.EvalSymlinks(currentPath) if err != nil { - return "", nil, fmt.Errorf("failed to get evaluate sym links for %s: %w", currentPath, err) + return "", nil, errors.Errorf("failed to get evaluate sym links for %s: %w", currentPath, err) } // Get info about the symlink target @@ -1128,12 +1128,12 @@ func evalRealPathAndInfo(currentPath string) (string, os.FileInfo, error) { func evalRealPathForWalkDir(currentPath string) (string, bool, error) { realPath, err := filepath.EvalSymlinks(currentPath) if err != nil { - return "", false, fmt.Errorf("failed to evaluate symlinks for %s: %w", currentPath, err) + return "", false, errors.Errorf("failed to evaluate symlinks for %s: %w", currentPath, err) } realInfo, err := os.Stat(realPath) if err != nil { - return "", false, fmt.Errorf("failed to describe file %s: %w", realPath, err) + return "", false, errors.Errorf("failed to describe file %s: %w", realPath, err) } return realPath, realInfo.IsDir(), nil @@ -1174,7 +1174,7 @@ func WalkDirWithSymlinks(root string, externalWalkFn fs.WalkDirFunc) error { // Convert the current physical path to a logical path relative to the walk root rel, err := filepath.Rel(pair.physical, currentPath) if err != nil { - return fmt.Errorf("failed to get relative path between %s and %s: %w", pair.physical, currentPath, err) + return errors.Errorf("failed to get relative path between %s and %s: %w", pair.physical, currentPath, err) } logicalPath := filepath.Join(pair.logical, rel) @@ -1218,7 +1218,7 @@ func WalkDirWithSymlinks(root string, externalWalkFn fs.WalkDirFunc) error { realRoot, err := filepath.EvalSymlinks(root) if err != nil { - return fmt.Errorf("failed to evaluate symlinks for %s: %w", root, err) + return errors.Errorf("failed to evaluate symlinks for %s: %w", root, err) } // Start the walk from the root directory