From 580998cff1d26f610f49a346b8989fe1066e9632 Mon Sep 17 00:00:00 2001 From: Yousif Akbar <11247449+yhakbar@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:34:44 -0500 Subject: [PATCH] feat: Adding units-reading (#3550) * WIP - feat: Adding units-reading Working on adding more hooks and tests. * fix: Fixing typo in spelling of `functions` * fix: Moving function calls to `locals` so that they are read before DAG build * feat: Adding more function calls * fix: Adding `mark_as_read` * fix: Fixing tests * fix: Cleaning up test * chore: Documenting reading functionality * fix: Markdown linting * fix: Avoid logging extra when multiple flags are set * fix: Fixing tests * Update docs/_docs/04_reference/cli-options.md --- cli/app.go | 15 ++ cli/commands/flags.go | 59 ++++--- config/config_helpers.go | 54 +++++- configstack/module.go | 167 ++++++++++-------- configstack/stack.go | 43 +++-- docs/_docs/04_reference/built-in-functions.md | 28 +++ docs/_docs/04_reference/cli-options.md | 67 +++++++ options/options.go | 54 ++++++ .../units-reading/reading-from-tf/main.tf | 4 + .../reading-from-tf/terragrunt.hcl | 7 + .../reading-hcl-and-tfvars/main.tf | 9 + .../reading-hcl-and-tfvars/terragrunt.hcl | 9 + .../units-reading/reading-hcl/main.tf | 4 + .../units-reading/reading-hcl/terragrunt.hcl | 7 + .../units-reading/reading-json/main.tf | 4 + .../units-reading/reading-json/terragrunt.hcl | 7 + .../units-reading/reading-sops/main.tf | 4 + .../units-reading/reading-sops/terragrunt.hcl | 7 + .../units-reading/reading-tfvars/main.tf | 4 + .../reading-tfvars/terragrunt.hcl | 7 + test/fixtures/units-reading/secrets.txt | 21 +++ test/fixtures/units-reading/shared.hcl | 3 + test/fixtures/units-reading/shared.json | 3 + test/fixtures/units-reading/shared.tfvars | 1 + test/fixtures/units-reading/test_pgp_key.asc | 87 +++++++++ test/integartion_units_reading_test.go | 162 +++++++++++++++++ test/integration_test.go | 4 +- 27 files changed, 727 insertions(+), 114 deletions(-) create mode 100644 test/fixtures/units-reading/reading-from-tf/main.tf create mode 100644 test/fixtures/units-reading/reading-from-tf/terragrunt.hcl create mode 100644 test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf create mode 100644 test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl create mode 100644 test/fixtures/units-reading/reading-hcl/main.tf create mode 100644 test/fixtures/units-reading/reading-hcl/terragrunt.hcl create mode 100644 test/fixtures/units-reading/reading-json/main.tf create mode 100644 test/fixtures/units-reading/reading-json/terragrunt.hcl create mode 100644 test/fixtures/units-reading/reading-sops/main.tf create mode 100644 test/fixtures/units-reading/reading-sops/terragrunt.hcl create mode 100644 test/fixtures/units-reading/reading-tfvars/main.tf create mode 100644 test/fixtures/units-reading/reading-tfvars/terragrunt.hcl create mode 100644 test/fixtures/units-reading/secrets.txt create mode 100644 test/fixtures/units-reading/shared.hcl create mode 100644 test/fixtures/units-reading/shared.json create mode 100644 test/fixtures/units-reading/shared.tfvars create mode 100644 test/fixtures/units-reading/test_pgp_key.asc create mode 100644 test/integartion_units_reading_test.go diff --git a/cli/app.go b/cli/app.go index 5c5ef060ca..5fcb4e53b9 100644 --- a/cli/app.go +++ b/cli/app.go @@ -343,6 +343,21 @@ func initialSetup(cliCtx *cli.Context, opts *options.TerragruntOptions) error { opts.ExcludeByDefault = true } + if !opts.ExcludeByDefault && len(opts.ModulesThatInclude) > 0 { + opts.Logger.Debugf("Modules that include set. Excluding by default.") + opts.ExcludeByDefault = true + } + + if !opts.ExcludeByDefault && len(opts.UnitsReading) > 0 { + opts.Logger.Debugf("Units that read set. Excluding by default.") + opts.ExcludeByDefault = true + } + + if !opts.ExcludeByDefault && opts.StrictInclude { + opts.Logger.Debugf("Strict include set. Excluding by default.") + opts.ExcludeByDefault = true + } + opts.IncludeDirs, err = util.GlobCanonicalPath(opts.WorkingDir, opts.IncludeDirs...) if err != nil { return err diff --git a/cli/commands/flags.go b/cli/commands/flags.go index 2b2fe48b5d..56fc855673 100644 --- a/cli/commands/flags.go +++ b/cli/commands/flags.go @@ -67,30 +67,6 @@ const ( TerragruntIAMWebIdentityTokenFlagName = "terragrunt-iam-web-identity-token" TerragruntIAMWebIdentityTokenEnvName = "TERRAGRUNT_IAM_ASSUME_ROLE_WEB_IDENTITY_TOKEN" - TerragruntIgnoreDependencyErrorsFlagName = "terragrunt-ignore-dependency-errors" - TerragruntIgnoreDependencyErrorsEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS" - - TerragruntIgnoreDependencyOrderFlagName = "terragrunt-ignore-dependency-order" - TerragruntIgnoreDependencyOrderEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ORDER" - - TerragruntIgnoreExternalDependenciesFlagName = "terragrunt-ignore-external-dependencies" - TerragruntIgnoreExternalDependenciesEnvName = "TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES" - - TerragruntIncludeExternalDependenciesFlagName = "terragrunt-include-external-dependencies" - TerragruntIncludeExternalDependenciesEnvName = "TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES" - - TerragruntExcludesFileFlagName = "terragrunt-excludes-file" - TerragruntExcludesFileEnvName = "TERRAGRUNT_EXCLUDES_FILE" - - TerragruntExcludeDirFlagName = "terragrunt-exclude-dir" - TerragruntExcludeDirEnvName = "TERRAGRUNT_EXCLUDE_DIR" - - TerragruntIncludeDirFlagName = "terragrunt-include-dir" - TerragruntIncludeDirEnvName = "TERRAGRUNT_INCLUDE_DIR" - - TerragruntStrictIncludeFlagName = "terragrunt-strict-include" - TerragruntStrictIncludeEnvName = "TERRAGRUNT_STRICT_INCLUDE" - TerragruntParallelismFlagName = "terragrunt-parallelism" TerragruntParallelismEnvName = "TERRAGRUNT_PARALLELISM" @@ -130,6 +106,35 @@ const ( TerragruntNoDestroyDependenciesCheckFlagEnvName = "TERRAGRUNT_NO_DESTROY_DEPENDENCIES_CHECK" TerragruntNoDestroyDependenciesCheckFlagName = "terragrunt-no-destroy-dependencies-check" + // Queue related flags + + TerragruntIgnoreDependencyErrorsFlagName = "terragrunt-ignore-dependency-errors" + TerragruntIgnoreDependencyErrorsEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS" + + TerragruntIgnoreDependencyOrderFlagName = "terragrunt-ignore-dependency-order" + TerragruntIgnoreDependencyOrderEnvName = "TERRAGRUNT_IGNORE_DEPENDENCY_ORDER" + + TerragruntIgnoreExternalDependenciesFlagName = "terragrunt-ignore-external-dependencies" + TerragruntIgnoreExternalDependenciesEnvName = "TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES" + + TerragruntIncludeExternalDependenciesFlagName = "terragrunt-include-external-dependencies" + TerragruntIncludeExternalDependenciesEnvName = "TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES" + + TerragruntExcludesFileFlagName = "terragrunt-excludes-file" + TerragruntExcludesFileEnvName = "TERRAGRUNT_EXCLUDES_FILE" + + TerragruntExcludeDirFlagName = "terragrunt-exclude-dir" + TerragruntExcludeDirEnvName = "TERRAGRUNT_EXCLUDE_DIR" + + TerragruntIncludeDirFlagName = "terragrunt-include-dir" + TerragruntIncludeDirEnvName = "TERRAGRUNT_INCLUDE_DIR" + + TerragruntStrictIncludeFlagName = "terragrunt-strict-include" + TerragruntStrictIncludeEnvName = "TERRAGRUNT_STRICT_INCLUDE" + + TerragruntUnitsReadingFlagName = "terragrunt-queue-include-units-reading" + TerragruntUnitsReadingEnvName = "TERRAGRUNT_QUEUE_INCLUDE_UNITS_READING" + // Logs related flags/envs TerragruntLogLevelFlagName = "terragrunt-log-level" @@ -453,6 +458,12 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags { Destination: &opts.ModulesThatInclude, Usage: "If flag is set, 'run-all' will only run the command against Terragrunt modules that include the specified file.", }, + &cli.SliceFlag[string]{ + Name: TerragruntUnitsReadingFlagName, + EnvVar: TerragruntUnitsReadingEnvName, + Destination: &opts.UnitsReading, + Usage: "If flag is set, 'run-all' will only run the command against Terragrunt units that read the specified file via an HCL function.", + }, &cli.BoolFlag{ Name: TerragruntFailOnStateBucketCreationFlagName, EnvVar: TerragruntFailOnStateBucketCreationEnvName, diff --git a/config/config_helpers.go b/config/config_helpers.go index 08f0bafc6f..666217b265 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -68,6 +68,7 @@ const ( FuncNameEndsWith = "endswith" FuncNameStrContains = "strcontains" FuncNameTimeCmp = "timecmp" + FuncNameMarkAsRead = "mark_as_read" sopsCacheName = "sopsCache" ) @@ -167,6 +168,7 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E FuncNameGetDefaultRetryableErrors: wrapVoidToStringSliceAsFuncImpl(ctx, getDefaultRetryableErrors), FuncNameReadTfvarsFile: wrapStringSliceToStringAsFuncImpl(ctx, readTFVarsFile), FuncNameGetWorkingDir: wrapVoidToStringAsFuncImpl(ctx, getWorkingDir), + FuncNameMarkAsRead: wrapStringSliceToStringAsFuncImpl(ctx, markAsRead), // Map with HCL functions introduced in Terraform after v0.15.3, since upgrade to a later version is not supported // https://github.com/gruntwork-io/terragrunt/blob/master/go.mod#L22 @@ -627,12 +629,25 @@ func ParseTerragruntConfig(ctx *ParsingContext, configPath string, defaultVal *c targetConfig := getCleanedTargetConfigPath(configPath, ctx.TerragruntOptions.TerragruntConfigPath) targetConfigFileExists := util.FileExists(targetConfig) + if !targetConfigFileExists && defaultVal == nil { return cty.NilVal, errors.New(TerragruntConfigNotFoundError{Path: targetConfig}) - } else if !targetConfigFileExists { + } + + if !targetConfigFileExists { return *defaultVal, nil } + path, err := util.CanonicalPath(targetConfig, ctx.TerragruntOptions.WorkingDir) + if err != nil { + return cty.NilVal, errors.New(err) + } + + ctx.TerragruntOptions.AppendReadFile( + path, + ctx.TerragruntOptions.WorkingDir, + ) + // We update the ctx of terragruntOptions to the config being read in. opts, err := ctx.TerragruntOptions.Clone(targetConfig) if err != nil { @@ -812,6 +827,11 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) { return "", errors.New(err) } + ctx.TerragruntOptions.AppendReadFile( + canonicalSourceFile, + ctx.TerragruntOptions.WorkingDir, + ) + // Set environment variables from the TerragruntOptions.Env map. // This is especially useful for integrations with things like the `terragrunt-auth-provider` flag, // which can set environment variables that are used for decryption. @@ -1008,6 +1028,11 @@ func readTFVarsFile(ctx *ParsingContext, args []string) (string, error) { return "", errors.New(TFVarFileNotFoundError{File: varFile}) } + ctx.TerragruntOptions.AppendReadFile( + varFile, + ctx.TerragruntOptions.WorkingDir, + ) + fileContents, err := os.ReadFile(varFile) if err != nil { return "", errors.New(fmt.Errorf("could not read file %q: %w", varFile, err)) @@ -1036,6 +1061,33 @@ func readTFVarsFile(ctx *ParsingContext, args []string) (string, error) { return string(data), nil } +// markAsRead marks a file as explicitly read. This is useful for detection via TerragruntUnitsReading flag. +func markAsRead(ctx *ParsingContext, args []string) (string, error) { + if len(args) != 1 { + return "", errors.New(WrongNumberOfParamsError{Func: "mark_as_read", Expected: "1", Actual: len(args)}) + } + + file := args[0] + + path, err := util.CanonicalPath(file, ctx.TerragruntOptions.WorkingDir) + if err != nil { + return "", errors.New(err) + } + + ctx.TerragruntOptions.AppendReadFile( + path, + ctx.TerragruntOptions.WorkingDir, + ) + + return file, nil +} + +// warnWhenFileNotMarkedAsRead warns when a file is not being marked as read, even though a user might expect it to be. +// Situations where this is the case include: +// - A user specifies a file in the UnitsReading flag and that file is being read while parsing the inputs attribute. +// +// When the file is not marked as read, the function will return true, otherwise false. + // ParseAndDecodeVarFile uses the HCL2 file to parse the given varfile string into an HCL file body, and then decode it // into the provided output. func ParseAndDecodeVarFile(opts *options.TerragruntOptions, varFile string, fileContents []byte, out interface{}) error { diff --git a/configstack/module.go b/configstack/module.go index 2bf4f7176a..ba845d5a64 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -96,11 +96,11 @@ func (module *TerraformModule) checkForCyclesUsingDepthFirstSearch(visitedPaths } // planFile - return plan file location, if output folder is set -func (module *TerraformModule) planFile(terragruntOptions *options.TerragruntOptions) string { +func (module *TerraformModule) planFile(opts *options.TerragruntOptions) string { var planFile string // set plan file location if output folder is set - planFile = module.outputFile(terragruntOptions) + planFile = module.outputFile(opts) planCommand := module.TerragruntOptions.TerraformCommand == terraform.CommandNamePlan || module.TerragruntOptions.TerraformCommand == terraform.CommandNameShow @@ -158,26 +158,26 @@ func (module *TerraformModule) findModuleInPath(targetDirs []string) bool { // Note that we skip the prompt for `run-all destroy` calls. Given the destructive and irreversible nature of destroy, we don't // want to provide any risk to the user of accidentally destroying an external dependency unless explicitly included // with the --terragrunt-include-external-dependencies or --terragrunt-include-dir flags. -func (module *TerraformModule) confirmShouldApplyExternalDependency(ctx context.Context, dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) { - if terragruntOptions.IncludeExternalDependencies { - terragruntOptions.Logger.Debugf("The --terragrunt-include-external-dependencies flag is set, so automatically including all external dependencies, and will run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) +func (module *TerraformModule) confirmShouldApplyExternalDependency(ctx context.Context, dependency *TerraformModule, opts *options.TerragruntOptions) (bool, error) { + if opts.IncludeExternalDependencies { + opts.Logger.Debugf("The --terragrunt-include-external-dependencies flag is set, so automatically including all external dependencies, and will run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) return true, nil } - if terragruntOptions.NonInteractive { - terragruntOptions.Logger.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + if opts.NonInteractive { + opts.Logger.Debugf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with a run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) return false, nil } - stackCmd := terragruntOptions.TerraformCommand + stackCmd := opts.TerraformCommand if stackCmd == "destroy" { - terragruntOptions.Logger.Debugf("run-all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) + opts.Logger.Debugf("run-all command called with destroy. To avoid accidentally having destructive effects on external dependencies with run-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path) return false, nil } - terragruntOptions.Logger.Infof("Module %s has external dependency %s", module.Path, dependency.Path) + opts.Logger.Infof("Module %s has external dependency %s", module.Path, dependency.Path) - return shell.PromptUserForYesNo(ctx, "Should Terragrunt apply the external dependency?", terragruntOptions) + return shell.PromptUserForYesNo(ctx, "Should Terragrunt apply the external dependency?", opts) } // Get the list of modules this module depends on @@ -220,15 +220,15 @@ type TerraformModules []*TerraformModule // FindWhereWorkingDirIsIncluded - find where working directory is included, flow: // 1. Find root git top level directory and build list of modules -// 2. Iterate over includes from terragruntOptions if git top level directory detection failed +// 2. Iterate over includes from opts if git top level directory detection failed // 3. Filter found module only items which has in dependencies working directory -func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) TerraformModules { +func FindWhereWorkingDirIsIncluded(ctx context.Context, opts *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) TerraformModules { var ( pathsToCheck []string matchedModulesMap = make(TerraformModulesMap) ) - if gitTopLevelDir, err := shell.GitTopLevelDir(ctx, terragruntOptions, terragruntOptions.WorkingDir); err == nil { + if gitTopLevelDir, err := shell.GitTopLevelDir(ctx, opts, opts.WorkingDir); err == nil { pathsToCheck = append(pathsToCheck, gitTopLevelDir) } else { // detection failed, trying to use include directories as source for stacks @@ -247,14 +247,14 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *optio cfgOptions, err := options.NewTerragruntOptionsWithConfigPath(dir) if err != nil { - terragruntOptions.Logger.Debugf("Failed to build terragrunt options from %s %v", dir, err) + opts.Logger.Debugf("Failed to build terragrunt options from %s %v", dir, err) continue } - cfgOptions.Env = terragruntOptions.Env - cfgOptions.LogLevel = terragruntOptions.LogLevel - cfgOptions.OriginalTerragruntConfigPath = terragruntOptions.OriginalTerragruntConfigPath - cfgOptions.TerraformCommand = terragruntOptions.TerraformCommand + cfgOptions.Env = opts.Env + cfgOptions.LogLevel = opts.LogLevel + cfgOptions.OriginalTerragruntConfigPath = opts.OriginalTerragruntConfigPath + cfgOptions.TerraformCommand = opts.TerraformCommand cfgOptions.NonInteractive = true cfgOptions.Logger.SetOptions(log.WithHooks(NewForceLogLevelHook(log.DebugLevel))) @@ -263,13 +263,13 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *optio if err != nil { // log error as debug since in some cases stack building may fail because parent files can be designed // to work with relative paths from downstream modules - terragruntOptions.Logger.Debugf("Failed to build module stack %v", err) + opts.Logger.Debugf("Failed to build module stack %v", err) continue } dependentModules := stack.ListStackDependentModules() - deps, found := dependentModules[terragruntOptions.WorkingDir] + deps, found := dependentModules[opts.WorkingDir] if found { for _, module := range stack.Modules { for _, dep := range deps { @@ -295,19 +295,19 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, terragruntOptions *optio // for a directed graph. It can be used to dump a .dot file. // This is a similar implementation to terraform's digraph https://github.com/hashicorp/terraform/blob/master/digraph/graphviz.go // adding some styling to modules that are excluded from the execution in *-all commands -func (modules TerraformModules) WriteDot(w io.Writer, terragruntOptions *options.TerragruntOptions) error { +func (modules TerraformModules) WriteDot(w io.Writer, opts *options.TerragruntOptions) error { if _, err := w.Write([]byte("digraph {\n")); err != nil { return errors.New(err) } defer func(w io.Writer, p []byte) { _, err := w.Write(p) if err != nil { - terragruntOptions.Logger.Warnf("Failed to close graphviz output: %v", err) + opts.Logger.Warnf("Failed to close graphviz output: %v", err) } }(w, []byte("}\n")) // all paths are relative to the TerragruntConfigPath - prefix := filepath.Dir(terragruntOptions.TerragruntConfigPath) + "/" + prefix := filepath.Dir(opts.TerragruntConfigPath) + "/" for _, source := range modules { // apply a different coloring for excluded nodes @@ -407,41 +407,17 @@ func (modules TerraformModules) CheckForCycles() error { return nil } -// flagExcludedDirs iterates over a module slice and flags all entries as excluded, which should be ignored via the terragrunt-exclude-dir CLI flag. -func (modules TerraformModules) flagExcludedDirs(terragruntOptions *options.TerragruntOptions) TerraformModules { - for _, module := range modules { - if module.findModuleInPath(terragruntOptions.ExcludeDirs) { - // Mark module itself as excluded - module.FlagExcluded = true - } - - // Mark all affected dependencies as excluded - for _, dependency := range module.Dependencies { - if dependency.findModuleInPath(terragruntOptions.ExcludeDirs) { - dependency.FlagExcluded = true - } - } - } - - return modules -} - -// flagIncludedDirs iterates over a module slice and flags all entries not in the list specified via the terragrunt-include-dir CLI flag as excluded. -func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.TerragruntOptions) TerraformModules { - // If we're not excluding by default, we should include everything by default. - // This can happen when a user doesn't set include flags. - if !terragruntOptions.ExcludeByDefault { - // If we aren't given any include directories, but are given the strict include flag, - // return no modules. - if terragruntOptions.StrictInclude { - return TerraformModules{} - } - +// flagIncludedDirs includes all units by default. +// +// However, when anything that triggers ExcludeByDefault is set, the function will instead +// selectively include only the units that are in the list specified via the IncludeDirs option. +func (modules TerraformModules) flagIncludedDirs(opts *options.TerragruntOptions) TerraformModules { + if !opts.ExcludeByDefault { return modules } for _, module := range modules { - if module.findModuleInPath(terragruntOptions.IncludeDirs) { + if module.findModuleInPath(opts.IncludeDirs) { module.FlagExcluded = false } else { module.FlagExcluded = true @@ -449,7 +425,7 @@ func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.Terr } // Mark all affected dependencies as included before proceeding if not in strict include mode. - if !terragruntOptions.StrictInclude { + if !opts.StrictInclude { for _, module := range modules { if !module.FlagExcluded { for _, dependency := range module.Dependencies { @@ -462,36 +438,26 @@ func (modules TerraformModules) flagIncludedDirs(terragruntOptions *options.Terr return modules } -// flagModulesThatDontInclude iterates over a module slice and flags all modules that don't include at least one file in -// the specified include list on the TerragruntOptions ModulesThatInclude attribute. Flagged modules will be filtered -// out of the set. -func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *options.TerragruntOptions) (TerraformModules, error) { +// flagUnitsThatAreIncluded iterates over a module slice and flags all modules that include at least one file in +// the specified include list on the TerragruntOptions ModulesThatInclude attribute. +func (modules TerraformModules) flagUnitsThatAreIncluded(opts *options.TerragruntOptions) (TerraformModules, error) { // If no ModulesThatInclude is specified return the modules list instantly - if len(terragruntOptions.ModulesThatInclude) == 0 { + if len(opts.ModulesThatInclude) == 0 { return modules, nil } - modulesThatIncludeCanonicalPath := []string{} + modulesThatIncludeCanonicalPaths := []string{} - for _, includePath := range terragruntOptions.ModulesThatInclude { - canonicalPath, err := util.CanonicalPath(includePath, terragruntOptions.WorkingDir) + for _, includePath := range opts.ModulesThatInclude { + canonicalPath, err := util.CanonicalPath(includePath, opts.WorkingDir) if err != nil { return nil, err } - modulesThatIncludeCanonicalPath = append(modulesThatIncludeCanonicalPath, canonicalPath) + modulesThatIncludeCanonicalPaths = append(modulesThatIncludeCanonicalPaths, canonicalPath) } for _, module := range modules { - // Ignore modules that are already excluded because this feature is a filter for excluding the subset, not - // including modules that have already been excluded through other means. - if module.FlagExcluded { - continue - } - - // Mark modules that don't include any of the specified paths as excluded. To do this, we first flag the module - // as excluded, and if it includes any path in the set, we set the exclude flag back to false. - module.FlagExcluded = true for _, includeConfig := range module.Config.ProcessedIncludes { // resolve include config to canonical path to compare with modulesThatIncludeCanonicalPath // https://github.com/gruntwork-io/terragrunt/issues/1944 @@ -500,7 +466,7 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op return nil, err } - if util.ListContainsElement(modulesThatIncludeCanonicalPath, canonicalPath) { + if util.ListContainsElement(modulesThatIncludeCanonicalPaths, canonicalPath) { module.FlagExcluded = false } } @@ -512,14 +478,13 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op continue } - dependency.FlagExcluded = true for _, includeConfig := range dependency.Config.ProcessedIncludes { canonicalPath, err := util.CanonicalPath(includeConfig.Path, module.Path) if err != nil { return nil, err } - if util.ListContainsElement(modulesThatIncludeCanonicalPath, canonicalPath) { + if util.ListContainsElement(modulesThatIncludeCanonicalPaths, canonicalPath) { dependency.FlagExcluded = false } } @@ -529,6 +494,54 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op return modules, nil } +// flagUnitsThatRead iterates over a module slice and flags all modules that read at least one file in the specified +// file list in the TerragruntOptions UnitsReading attribute. +func (modules TerraformModules) flagUnitsThatRead(opts *options.TerragruntOptions) (TerraformModules, error) { + // If no UnitsThatRead is specified return the modules list instantly + if len(opts.UnitsReading) == 0 { + return modules, nil + } + + for _, readPath := range opts.UnitsReading { + path, err := util.CanonicalPath(readPath, opts.WorkingDir) + if err != nil { + return nil, err + } + + for _, module := range modules { + if opts.DidReadFile(path, module.Path) { + module.FlagExcluded = false + } + } + } + + return modules, nil +} + +// flagExcludedDirs iterates over a module slice and flags all entries as excluded listed in the terragrunt-exclude-dir CLI flag. +func (modules TerraformModules) flagExcludedDirs(opts *options.TerragruntOptions) TerraformModules { + // If we don't have any excludes, we don't need to do anything. + if len(opts.ExcludeDirs) == 0 { + return modules + } + + for _, module := range modules { + if module.findModuleInPath(opts.ExcludeDirs) { + // Mark module itself as excluded + module.FlagExcluded = true + } + + // Mark all affected dependencies as excluded + for _, dependency := range module.Dependencies { + if dependency.findModuleInPath(opts.ExcludeDirs) { + dependency.FlagExcluded = true + } + } + } + + return modules +} + var existingModules = cache.NewCache[*TerraformModulesMap](existingModulesCacheName) type TerraformModulesMap map[string]*TerraformModule diff --git a/configstack/stack.go b/configstack/stack.go index b037e4ea93..0f99a72e2f 100644 --- a/configstack/stack.go +++ b/configstack/stack.go @@ -399,12 +399,12 @@ func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfi return nil, err } - var includedModules TerraformModules + var withUnitsIncluded TerraformModules err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_included_dirs", map[string]interface{}{ "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - includedModules = crossLinkedModules.flagIncludedDirs(stack.terragruntOptions) + withUnitsIncluded = crossLinkedModules.flagIncludedDirs(stack.terragruntOptions) return nil }) @@ -412,12 +412,18 @@ func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfi return nil, err } - var includedModulesWithExcluded TerraformModules + var withUnitsThatAreIncludedByOthers TerraformModules - err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_excluded_dirs", map[string]interface{}{ + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_units_that_are_included", map[string]interface{}{ "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - includedModulesWithExcluded = includedModules.flagExcludedDirs(stack.terragruntOptions) + result, err := withUnitsIncluded.flagUnitsThatAreIncluded(stack.terragruntOptions) + if err != nil { + return err + } + + withUnitsThatAreIncludedByOthers = result + return nil }) @@ -425,25 +431,39 @@ func (stack *Stack) ResolveTerraformModules(ctx context.Context, terragruntConfi return nil, err } - var finalModules TerraformModules + var withUnitsRead TerraformModules - err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_modules_that_dont_include", map[string]interface{}{ + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_units_that_read", map[string]interface{}{ "working_dir": stack.terragruntOptions.WorkingDir, }, func(childCtx context.Context) error { - result, err := includedModulesWithExcluded.flagModulesThatDontInclude(stack.terragruntOptions) + result, err := withUnitsThatAreIncludedByOthers.flagUnitsThatRead(stack.terragruntOptions) if err != nil { return err } - finalModules = result + withUnitsRead = result return nil }) + if err != nil { return nil, err } - return finalModules, nil + var withModulesExcluded TerraformModules + + err = telemetry.Telemetry(ctx, stack.terragruntOptions, "flag_excluded_dirs", map[string]interface{}{ + "working_dir": stack.terragruntOptions.WorkingDir, + }, func(childCtx context.Context) error { + withModulesExcluded = withUnitsRead.flagExcludedDirs(stack.terragruntOptions) + return nil + }) + + if err != nil { + return nil, err + } + + return withModulesExcluded, nil } // Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents @@ -584,6 +604,9 @@ func (stack *Stack) resolveTerraformModule(ctx context.Context, terragruntConfig }) } + // Hack to persist readFiles. Need to discuss with team to see if there is a better way to handle this. + stack.terragruntOptions.CloneReadFiles(opts.ReadFiles) + terragruntSource, err := config.GetTerragruntSourceForModule(stack.terragruntOptions.Source, modulePath, terragruntConfig) if err != nil { return nil, err diff --git a/docs/_docs/04_reference/built-in-functions.md b/docs/_docs/04_reference/built-in-functions.md index c829cbfe5a..6d09017fb5 100644 --- a/docs/_docs/04_reference/built-in-functions.md +++ b/docs/_docs/04_reference/built-in-functions.md @@ -40,6 +40,7 @@ Terragrunt allows you to use built-in functions anywhere in `terragrunt.hcl`, ju - [sops\_decrypt\_file](#sops_decrypt_file) - [get\_terragrunt\_source\_cli\_flag](#get_terragrunt_source_cli_flag) - [read\_tfvars\_file](#read_tfvars_file) +- [mark\_as\_read](#mark_as_read) ## OpenTofu/Terraform built-in functions @@ -867,3 +868,30 @@ remote_state { } } ``` + +## mark_as_read + +`mark_as_read(file_path)` marks a file as read, so that it can be picked up for inclusion by the [queue-include-units-reading](./cli-options.md#queue-include-units-reading) flag. + +This is useful for situations when you want to mark a file as read, but are not reading it using a native Terragrunt HCL function. + +For example: + +```hcl +locals { + filename = mark_as_read("file-read-by-tofu.txt") +} + +inputs = { + filename = local.filename +} +``` + +By using `mark_as_read` on `file-read-by-tofu.txt`, you can ensure that the `terragrunt.hcl` file passing in the `file-read-by-tofu.txt` file as an input will be included in +any `run-all` run where the flag `--queue-include-units-reading file-read-by-tofu.txt` is set. + +The same technique can be used to mark a file as read when reading a file using code in `run_cmd`, etc. + +**NOTE**: Due to the way that Terragrunt parses configurations during a `run-all`, functions will only properly mark files as read +if they are used in the `locals` block. Reading a file directly in the `inputs` block will not mark the file as read, as the `inputs` +block is not evaluated until *after* the queue has been populated with units to run. diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index f25460fd07..2b5a50a1ad 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -1291,6 +1291,73 @@ passed in, the set will be the union of modules that includes at least one of th NOTE: When using relative paths, the paths are relative to the working directory. This is either the current working directory, or any path passed in to [terragrunt-working-dir](#terragrunt-working-dir). +### terragrunt-queue-include-units-reading + +**CLI Arg**: `--terragrunt-queue-include-units-reading`
+**Environment Variable**: `TERRAGRUNT_QUEUE_INCLUDE_UNITS_READING`
+**Commands**: + +- [run-all](#run-all) +- [plan-all (DEPRECATED: use run-all)](#plan-all-deprecated-use-run-all) +- [apply-all (DEPRECATED: use run-all)](#apply-all-deprecated-use-run-all) +- [output-all (DEPRECATED: use run-all)](#output-all-deprecated-use-run-all) +- [destroy-all (DEPRECATED: use run-all)](#destroy-all-deprecated-use-run-all) +- [validate-all (DEPRECATED: use run-all)](#validate-all-deprecated-use-run-all) + +This flag works very similarly to the `--terragrunt-modules-that-include` flag, but instead of looking for included configurations, +it instead looks for configurations that read a given file. + +When passed in, the `*-all` commands will include all units (modules) that read a given file into the queue. This is useful +when you want to trigger an update on all units that read a given file using HCL functions in their configurations. + +Consider the following folder structure: + +```tree +. +├── reading-shared-hcl +│   └── terragrunt.hcl +├── also-reading-shared-hcl +│   └── terragrunt.hcl +├── not-reading-shared-hcl +│   └── terragrunt.hcl +└── shared.hcl +``` + +Suppose that `reading-shared-hcl` and `also-reading-shared-hcl` both read `shared.hcl` in their configurations, like so: + +```hcl +locals { + shared = read_terragrunt_config(find_in_parent_folders("shared.hcl")) +} +``` + +If you run the command `run-all init --terragrunt-queue-include-units-reading shared.hcl` from the root folder, both +`reading-shared-hcl` and `also-reading-shared-hcl` will be run; not `not-reading-shared-hcl`. + +This is because the `read_terragrunt_config` HCL function has a special hook that allows Terragrunt to track that it has +read the file `shared.hcl`. This hook is used by all native HCL functions that Terragrunt supports which read files. + +Note, however, that there are certain scenarios where Terragrunt may not be able to track that a file has been read this way. + +For example, you may be using a bash script to read a file via `run_cmd`, or reading the file via OpenTofu code. To support these +use-cases, the [mark_as_read](./built-in-functions.md#mark_as_read) function can be used to manually mark a file as read. + +That would look something like this: + +```hcl +locals { + filename = mark_as_read("file-read-by-tofu.txt") +} + +inputs = { + filename = local.filename +} +``` + +**⚠️**: Due to the way that Terragrunt parses configurations during a `run-all`, functions will only properly mark files as read +if they are used in the `locals` block. Reading a file directly in the `inputs` block will not mark the file as read, as the `inputs` +block is not evaluated until _after_ the queue has been populated with units to run. + ### terragrunt-fetch-dependency-output-from-state **CLI Arg**: `--terragrunt-fetch-dependency-output-from-state`
diff --git a/options/options.go b/options/options.go index 4fc5ca93c7..2e98de2d65 100644 --- a/options/options.go +++ b/options/options.go @@ -248,6 +248,10 @@ type TerragruntOptions struct { // in this list. ModulesThatInclude []string + // When used with `run-all`, restrict the units in the stack to only those that read at least one of the files + // in this list. + UnitsReading []string + // A command that can be used to run Terragrunt with the given options. This is useful for running Terragrunt // multiple times (e.g. when spinning up a stack of Terraform modules). The actual command is normally defined // in the cli package, which depends on almost all other packages, so we declare it here so that other @@ -356,6 +360,10 @@ type TerragruntOptions struct { // FeatureFlags is a map of feature flags to enable. FeatureFlags map[string]string + + // ReadFiles is a map of files to the Units + // that read them using HCL functions in the unit. + ReadFiles map[string][]string } // TerragruntOptionsFunc is a functional option type used to pass options in certain integration tests @@ -587,6 +595,8 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp IncludeDirs: opts.IncludeDirs, ExcludeByDefault: opts.ExcludeByDefault, ModulesThatInclude: opts.ModulesThatInclude, + UnitsReading: opts.UnitsReading, + ReadFiles: opts.ReadFiles, Parallelism: opts.Parallelism, StrictInclude: opts.StrictInclude, RunTerragrunt: opts.RunTerragrunt, @@ -721,6 +731,50 @@ func (opts *TerragruntOptions) DataDir() string { return util.JoinPath(opts.WorkingDir, tfDataDir) } +// AppendReadFile appends to the list of files read by a given unit. +func (opts *TerragruntOptions) AppendReadFile(file, unit string) { + if opts.ReadFiles == nil { + opts.ReadFiles = map[string][]string{} + } + + for _, u := range opts.ReadFiles[file] { + if u == unit { + return + } + } + + opts.Logger.Debugf("Tracking that file %s was read by %s.", file, unit) + opts.ReadFiles[file] = append(opts.ReadFiles[file], unit) +} + +// DidReadFile checks if a given file was read by a given unit. +func (opts *TerragruntOptions) DidReadFile(file, unit string) bool { + if opts.ReadFiles == nil { + return false + } + + for _, u := range opts.ReadFiles[file] { + if u == unit { + return true + } + } + + return false +} + +// CloneReadFiles creates a copy of the ReadFiles map. +func (opts *TerragruntOptions) CloneReadFiles(readFiles map[string][]string) { + if readFiles == nil { + return + } + + for file, units := range readFiles { + for _, unit := range units { + opts.AppendReadFile(file, unit) + } + } +} + // identifyDefaultWrappedExecutable returns default path used for wrapped executable. func identifyDefaultWrappedExecutable() string { if util.IsCommandExecutable(TofuDefaultPath, "-version") { diff --git a/test/fixtures/units-reading/reading-from-tf/main.tf b/test/fixtures/units-reading/reading-from-tf/main.tf new file mode 100644 index 0000000000..7ee83502d5 --- /dev/null +++ b/test/fixtures/units-reading/reading-from-tf/main.tf @@ -0,0 +1,4 @@ +variable "filename" {} +output "shared" { + value = jsondecode(file(var.filename)) +} diff --git a/test/fixtures/units-reading/reading-from-tf/terragrunt.hcl b/test/fixtures/units-reading/reading-from-tf/terragrunt.hcl new file mode 100644 index 0000000000..abd30a9359 --- /dev/null +++ b/test/fixtures/units-reading/reading-from-tf/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + filename = mark_as_read(find_in_parent_folders("shared.json")) +} + +inputs = { + filename = local.filename +} diff --git a/test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf b/test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf new file mode 100644 index 0000000000..674790dc79 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl-and-tfvars/main.tf @@ -0,0 +1,9 @@ +variable "shared_hcl" {} +output "shared_hcl" { + value = var.shared_hcl +} + +variable "shared_tfvars" {} +output "shared_tfvars" { + value = var.shared_tfvars +} diff --git a/test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl b/test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl new file mode 100644 index 0000000000..fd8a3eced6 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl-and-tfvars/terragrunt.hcl @@ -0,0 +1,9 @@ +locals { + shared_hcl = read_terragrunt_config(find_in_parent_folders("shared.hcl")).locals + shared_tfvars = read_tfvars_file(find_in_parent_folders("shared.tfvars")) +} + +inputs = { + shared_hcl = local.shared_hcl + shared_tfvars = local.shared_tfvars +} diff --git a/test/fixtures/units-reading/reading-hcl/main.tf b/test/fixtures/units-reading/reading-hcl/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-hcl/terragrunt.hcl b/test/fixtures/units-reading/reading-hcl/terragrunt.hcl new file mode 100644 index 0000000000..06fab2f1c6 --- /dev/null +++ b/test/fixtures/units-reading/reading-hcl/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = read_terragrunt_config(find_in_parent_folders("shared.hcl")).locals +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/reading-json/main.tf b/test/fixtures/units-reading/reading-json/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-json/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-json/terragrunt.hcl b/test/fixtures/units-reading/reading-json/terragrunt.hcl new file mode 100644 index 0000000000..11d8df1783 --- /dev/null +++ b/test/fixtures/units-reading/reading-json/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = jsondecode(file(mark_as_read(find_in_parent_folders("shared.json")))) +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/reading-sops/main.tf b/test/fixtures/units-reading/reading-sops/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-sops/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-sops/terragrunt.hcl b/test/fixtures/units-reading/reading-sops/terragrunt.hcl new file mode 100644 index 0000000000..3e8310fa14 --- /dev/null +++ b/test/fixtures/units-reading/reading-sops/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = sops_decrypt_file(find_in_parent_folders("secrets.txt")) +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/reading-tfvars/main.tf b/test/fixtures/units-reading/reading-tfvars/main.tf new file mode 100644 index 0000000000..05d8c59b82 --- /dev/null +++ b/test/fixtures/units-reading/reading-tfvars/main.tf @@ -0,0 +1,4 @@ +variable "shared" {} +output "shared" { + value = var.shared +} diff --git a/test/fixtures/units-reading/reading-tfvars/terragrunt.hcl b/test/fixtures/units-reading/reading-tfvars/terragrunt.hcl new file mode 100644 index 0000000000..4c6441f83f --- /dev/null +++ b/test/fixtures/units-reading/reading-tfvars/terragrunt.hcl @@ -0,0 +1,7 @@ +locals { + shared = read_tfvars_file(find_in_parent_folders("shared.tfvars")) +} + +inputs = { + shared = local.shared +} diff --git a/test/fixtures/units-reading/secrets.txt b/test/fixtures/units-reading/secrets.txt new file mode 100644 index 0000000000..37426edfcd --- /dev/null +++ b/test/fixtures/units-reading/secrets.txt @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:w2jDRJR9BeIMSKE4+qnKWhfM,iv:08ACLYrUGtWriOV/ua4X6NZt57VmiTmAcnxB5V+8AUc=,tag:cVdkIO4EXAmyV3y7n/zbiA==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-12-17T18:38:13Z", + "mac": "ENC[AES256_GCM,data:8lPZmY8YgA0DqPRxLC9hVoRUXmbzaXgUBv3MHTm4iK44/6URIgJBUnPFPUbwIN7xbIgXd+QPQEMvfsmifqXorynGEwt2WtMKCPANg+2Ctf2KMmj7fGpe3HIlRhQiixip7/xzrIMbSdIRMS098D42JTvOIFNbWVQhByfN64AnDJY=,iv:wtouC/mWjhFwiJKDS6+5LqnQMcAeejElXLaL3H15jbY=,tag:6Bmemr2BMgShaMO3v4uiXw==,type:str]", + "pgp": [ + { + "created_at": "2021-12-17T18:38:12Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA0sXzMgpEabgAQf+KHsPp4Pp8YNtG7ChRpZO2qB/bFncWtAF9evO+RjAEahb\nM+hzxkB5KDUSMYs0aeWeOrOqYPrjPPJxCspZtQhy8/qrC064kA7gq2PWhYAqGcKP\ntnPI8D0SYDZBgoyHRqFuuD5TZio8swE89SxphftL0W3KkHay7WKQHj/cFqNoISNl\nn0XeCgbacIwo5WxWz1qNFvaeo0rFFFhIhbfaegx/SWwUi1y6WK7sB0QobMRwXHj+\nORiUWVvx/fCIMCaerPN/SjIA/DgzbZ3DWaixYXpW85Ipz7myu/zUQcWnWcGXnMRQ\nERMYc6GyyLHwjZN1XuvXdPXvAt6vvaH4w5U9kW2l19JeAZXkcM14ivDoGwY1oLcX\n4d2/MAS7vM7SgmcPBGmpNsJJgkWTgoc8qeFtu9u3e4e9pR4+dcJCbGQLQ5RiyM2Z\nsyHjL6em/j4JLdtbM16orP6Q3oEPelphG7sxbDXBeA==\n=6u1S\n-----END PGP MESSAGE-----\n", + "fp": "3EF98802EEDCAF0C688B81F419546E0C123C664E" + } + ], + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file diff --git a/test/fixtures/units-reading/shared.hcl b/test/fixtures/units-reading/shared.hcl new file mode 100644 index 0000000000..d5f16bab7d --- /dev/null +++ b/test/fixtures/units-reading/shared.hcl @@ -0,0 +1,3 @@ +locals { + shared_hcl = "value" +} diff --git a/test/fixtures/units-reading/shared.json b/test/fixtures/units-reading/shared.json new file mode 100644 index 0000000000..ead8846178 --- /dev/null +++ b/test/fixtures/units-reading/shared.json @@ -0,0 +1,3 @@ +{ + "key": "value" +} diff --git a/test/fixtures/units-reading/shared.tfvars b/test/fixtures/units-reading/shared.tfvars new file mode 100644 index 0000000000..0ffcc1d995 --- /dev/null +++ b/test/fixtures/units-reading/shared.tfvars @@ -0,0 +1 @@ +shared_tfvars = "value" diff --git a/test/fixtures/units-reading/test_pgp_key.asc b/test/fixtures/units-reading/test_pgp_key.asc new file mode 100644 index 0000000000..e42619f652 --- /dev/null +++ b/test/fixtures/units-reading/test_pgp_key.asc @@ -0,0 +1,87 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQOYBF69y60BCAC0Ton9jmk5/ZV3yKi2o0rY22G41dh9dLSoxTRJNKsKeSJePHK0 +qtrHriPjnHxFLXrXfYiv6FkdkPv9dhmKjMf1op/RRw0uTDZ6CsQd5C8nT9LBPiB6 +JsO89+GCwoVsPQGGVgqq7PWSbA8fJkAPBNvdQvxJnSbm4EJTcaU7XZmmY/eZnhPt +pho66X3PIQSx8Xm9Dc7jHo/7bBCoEV15OdFYi44imDW9sCdCpYTVzMF2ZRtXGylQ +U6P5Iu3SEMTnn4jwPpyKllT4qRFdqMQ4GvqE47uRb+k/FDXP8UC6h/5GNHfbQtEO +CMngw5vJciLEUWW1e97baBPiccLXNaBXiecXABEBAAEAB/44Mkpb1qkBRAHz5XyA +AADx+d5JR41D/L3Z5CzZrCqrBvopONnfaWjq12GkLT+mJ/ijdSLHALnVtzxy0P5A +6oPgESGSjnyTM3m/K9/YGSiBLiXXyM9CgxZ6rR+CK7J9+72f6u1EPLqFOly0Lq3E +gJUuLxSGtQ5M4xSJAWCoUhHzg6rUP978RQXc4AgfkfXaq/ILGyOOa7VSz4NVs3cZ ++WtPqlFhy5AsoMoLvXrwqLhVa9QsCiW0dYScRPD9q29wRHnKDN2nSxTuHbWu55r+ +krevzk5gPOTdj61vz6/xD/rqNFakZDjFfZIu+srqjnLMVEPYudrH0buIAF9RFYPp +8x+BBADTH3Q5+b+n770aFBDldIjD+biEnjnw6A396RNX2YIPqWoz0b7lQSYBM+DI +TTm7OFmacH2DSYHxx/e2GGOOglbc2czPlWGyegSfZblwcPPT47uVY6ZWIXAsszF6 +opL/ahQ336FEl8Ws8vEjUoTXWFzF50gfmdC2xPYYmoxymmZ1IQQA2qIvbPausqO7 +tHk9Bco1MLLzP5JIAvuLUNHOj9H+bUXh5btkm2gZRIGglScljF6HT180bJ4gt07n +H1QNpyWKuH0h0YSTDlBjLPQvElQAvxMuvT7RAWagXQo8ew+2FHbrH9Bhlys1wcG3 ++h1S2G8M6TyY0dv98TvqHV9RG+YCHTcEAMQTpJBn+yelK8LXyVrw9OmH0qmQ9TOF +uJihLBSd7nen+l4a3yDpISk6hrb/q4AjpzMJ8fK17AEuH9OxUkO8vgtxefvnoDTe +GYwuhszZRbWLYVDHZEkEjiGbCiqZw5530tHShA/IdBc5LMd9fwsag4Bru0cKhyCN +oe1FvVcABEHXR+O0EFRlcnJhZ3J1bnQgVGVzdHOJAUwEEwEKADYWIQQ++YgC7tyv +DGiLgfQZVG4MEjxmTgUCXr3LrQIbAwQLCQgHBBUKCQgFFgIDAQACHgECF4AACgkQ +GVRuDBI8Zk76SwgAkfEmyr9qOtNXpD4m8SnCrg8EJRX584hJyAu08/uB+QQV7Xzj +Azns4oVGGpZYEB7sUd22ejw8BGe6OI4puL43fnpDcFj4UA7FAIzQTKNztWFci2Ho +L0IAwcdzSlJb6qREMJacit8G4Aayn/bqZSNF4cu0XipRlOtK28FXP1PQThlNtSJL +yPpYRyU6s2CLSeRRqbi1MJ4KgKcnfTvv/4+VYQ6y7/rsDnXDA0nLgzYKMqtNdBPY +r4u7kZsGcWP8w9xQDWNJtOqj8NiRSAtnglqiYTHuEQsRjYGjH+vHqnGfw7dcK4+3 +Rlc7ZPGON2NCmJ6OajedIxZR6yNiDwj1AfuXxZ0DmARevcutAQgAtV1FnBgLKuqg +BVs09UbyrFuVJIx6M4e0IFac4bVzyfGJKjmxHx2n7D8slTZs+lcBNDy+CseAsAuc +ixv/jVAsl6saip5nUIAMmHkPkPM+BVAuqBRWoYWGFw3TL83MaYAgBkSkurx+Koxv +w2bTumxrb9xsyiy7Bn7kaIyZU/nN8Xb0xf6QC+y7ckzaI6ZT4utrtJGXya4Cu1rp +qUUYs/4ssWjZS7KuRwu8ijNwq/Zl0920IXH/sX/gUhL2AO5sgExC+52yjTTvhlIb +ee8tjMB4JT6Hq1RS9JGkNqcUOvJ99WlUoOtZUJU9xAN4eyPUy+r7v9wxcwoguanC +LWaKSNe9hQARAQABAAf8CToRuLNtHLBWFspWmI8/o67Ubh5qzc9UGycSO9YVlMsM +fZD00WPwBJq22RqCkyk0/vmDePHrD/RvRjvU6wAOsg1QQDLMh8cT8k01Elya+v3i +zxtFyFmOLQMU4O7j547PUkemEnf/eokC20U9H/60Z+WmGpJfAMwY27bMytMVJqPJ +gL1BNo9l0HGSTkU7mMqq48MsozTJvmCYbrVeKhnHdpAFHHSNQ9hGzgpZkpTisDk7 +qDIHhF1Nv7IRZgX4OGUbj1hBk63ao1TclhbB8d7gitvTODbMxFOF9crm7ttWtq/C +CIBM9X5ilufEnuN2eV4LxLHeghQOGsJhl/FB8gov+wQAx/Ja/2WFzHoEc5evJtNU +ifKpaIAp4Qj673dx21Vzr43qnuNNxLuG53FvgRpfXhVRyzWmNnJByQ3NRVUv4J1j +GXymlPPPzmAbML8874zShMnMcd1+UCZ+0dgFeB6CetVySExO0p+qUW+fioP5jrJk +spNxtY01RE5AUSWUeQN3sdcEAOg1T1cUIEq/dgPLDQE3mPQta7fJeiDddheNgQRp +xKHxDKSpWd1RtmiUxG0cT3z68M8aRcL/X+q731WGadNhM+yp4xg6wVTpL8USdLZr +qqiuvMYqowryZOdvPUP8OE1lQwtWizCFOoNL+yJyKVzt7+Z0CrkE8s3li68sLMeg +z5gDBACflcuTWLMNt3buo/31YrNWLDRxDMdKpNZ0Tpj+Kxda9+GjRGWdMZHwOsIO +WhGnftxtbKSWu2+PabFBchiwLC1r4WHMFy/fxFFhufJtYI/c58kRd+9I0vw6/JQx +WvGunELyeTnNu3u+uvagUSchBhNln5hZZOtpaBaDhNngm5DXM0g+iQE2BBgBCgAg +FiEEPvmIAu7crwxoi4H0GVRuDBI8Zk4FAl69y60CGwwACgkQGVRuDBI8Zk4gogf/ +SyXhch/Ep9ESW+Zx7T+ImfQAVPBmka0NNCEvSF5i/ghTm700Iv/sLcRqhSJDiKLb +1imeqVN6khmr6+d21uCqqgiv/6X4w6zmxH9h4uPMv3H6WnHi40kDUawR7hFctNLX +twArH+xCX5MJgzhqAP5Yzgsk5XLwzgjCuXz06RqtmZVW4ofOP4GWBT24Gg4TyUF9 +Ibl1/QLKeYOarD4a/PcEfHhJyvnAmD68uIFr1gABJJneW8vvO4OPjLcptZMIZ1Nf +xqUuLBAJ71OI3RSSowpI3qCpFhH2j5vICM7jX2gIL9PQEGGg2ljNJoorJIEjYZzS +0S9IMjn4Mj7CYtAZarnIQw== +=fO3n +-----END PGP PRIVATE KEY BLOCK----- +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBF69y60BCAC0Ton9jmk5/ZV3yKi2o0rY22G41dh9dLSoxTRJNKsKeSJePHK0 +qtrHriPjnHxFLXrXfYiv6FkdkPv9dhmKjMf1op/RRw0uTDZ6CsQd5C8nT9LBPiB6 +JsO89+GCwoVsPQGGVgqq7PWSbA8fJkAPBNvdQvxJnSbm4EJTcaU7XZmmY/eZnhPt +pho66X3PIQSx8Xm9Dc7jHo/7bBCoEV15OdFYi44imDW9sCdCpYTVzMF2ZRtXGylQ +U6P5Iu3SEMTnn4jwPpyKllT4qRFdqMQ4GvqE47uRb+k/FDXP8UC6h/5GNHfbQtEO +CMngw5vJciLEUWW1e97baBPiccLXNaBXiecXABEBAAG0EFRlcnJhZ3J1bnQgVGVz +dHOJAUwEEwEKADYWIQQ++YgC7tyvDGiLgfQZVG4MEjxmTgUCXr3LrQIbAwQLCQgH +BBUKCQgFFgIDAQACHgECF4AACgkQGVRuDBI8Zk76SwgAkfEmyr9qOtNXpD4m8SnC +rg8EJRX584hJyAu08/uB+QQV7XzjAzns4oVGGpZYEB7sUd22ejw8BGe6OI4puL43 +fnpDcFj4UA7FAIzQTKNztWFci2HoL0IAwcdzSlJb6qREMJacit8G4Aayn/bqZSNF +4cu0XipRlOtK28FXP1PQThlNtSJLyPpYRyU6s2CLSeRRqbi1MJ4KgKcnfTvv/4+V +YQ6y7/rsDnXDA0nLgzYKMqtNdBPYr4u7kZsGcWP8w9xQDWNJtOqj8NiRSAtnglqi +YTHuEQsRjYGjH+vHqnGfw7dcK4+3Rlc7ZPGON2NCmJ6OajedIxZR6yNiDwj1AfuX +xbkBDQRevcutAQgAtV1FnBgLKuqgBVs09UbyrFuVJIx6M4e0IFac4bVzyfGJKjmx +Hx2n7D8slTZs+lcBNDy+CseAsAucixv/jVAsl6saip5nUIAMmHkPkPM+BVAuqBRW +oYWGFw3TL83MaYAgBkSkurx+Koxvw2bTumxrb9xsyiy7Bn7kaIyZU/nN8Xb0xf6Q +C+y7ckzaI6ZT4utrtJGXya4Cu1rpqUUYs/4ssWjZS7KuRwu8ijNwq/Zl0920IXH/ +sX/gUhL2AO5sgExC+52yjTTvhlIbee8tjMB4JT6Hq1RS9JGkNqcUOvJ99WlUoOtZ +UJU9xAN4eyPUy+r7v9wxcwoguanCLWaKSNe9hQARAQABiQE2BBgBCgAgFiEEPvmI +Au7crwxoi4H0GVRuDBI8Zk4FAl69y60CGwwACgkQGVRuDBI8Zk4gogf/SyXhch/E +p9ESW+Zx7T+ImfQAVPBmka0NNCEvSF5i/ghTm700Iv/sLcRqhSJDiKLb1imeqVN6 +khmr6+d21uCqqgiv/6X4w6zmxH9h4uPMv3H6WnHi40kDUawR7hFctNLXtwArH+xC +X5MJgzhqAP5Yzgsk5XLwzgjCuXz06RqtmZVW4ofOP4GWBT24Gg4TyUF9Ibl1/QLK +eYOarD4a/PcEfHhJyvnAmD68uIFr1gABJJneW8vvO4OPjLcptZMIZ1NfxqUuLBAJ +71OI3RSSowpI3qCpFhH2j5vICM7jX2gIL9PQEGGg2ljNJoorJIEjYZzS0S9IMjn4 +Mj7CYtAZarnIQw== +=dO7W +-----END PGP PUBLIC KEY BLOCK----- diff --git a/test/integartion_units_reading_test.go b/test/integartion_units_reading_test.go new file mode 100644 index 0000000000..245100af40 --- /dev/null +++ b/test/integartion_units_reading_test.go @@ -0,0 +1,162 @@ +package test_test + +import ( + "regexp" + "strings" + "testing" + + "github.com/gruntwork-io/terragrunt/test/helpers" + "github.com/gruntwork-io/terragrunt/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testFixtureUnitsReading = "fixtures/units-reading/" +) + +func TestUnitsReading(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testFixtureUnitsReading) + + tc := []struct { + name string + unitsReading []string + unitsExcluding []string + unitsIncluding []string + expectedUnits []string + }{ + { + name: "empty", + unitsReading: []string{}, + expectedUnits: []string{ + "reading-from-tf", + "reading-hcl", + "reading-hcl-and-tfvars", + "reading-json", + "reading-sops", + "reading-tfvars", + }, + }, + { + name: "reading_hcl", + unitsReading: []string{ + "shared.hcl", + }, + expectedUnits: []string{ + "reading-hcl", + "reading-hcl-and-tfvars", + }, + }, + { + name: "reading_tfvars", + unitsReading: []string{ + "shared.tfvars", + }, + expectedUnits: []string{ + "reading-tfvars", + "reading-hcl-and-tfvars", + }, + }, + { + name: "reading_json", + unitsReading: []string{ + "shared.json", + }, + expectedUnits: []string{ + "reading-from-tf", + "reading-json", + }, + }, + { + name: "reading_sops", + unitsReading: []string{ + "secrets.txt", + }, + expectedUnits: []string{ + "reading-sops", + }, + }, + { + name: "reading_from_hcl_with_exclude", + unitsReading: []string{ + "shared.hcl", + }, + unitsExcluding: []string{ + "reading-hcl-and-tfvars", + }, + expectedUnits: []string{ + "reading-hcl", + }, + }, + { + name: "reading_from_hcl_with_include", + unitsReading: []string{ + "shared.hcl", + }, + unitsIncluding: []string{ + "reading-tfvars", + }, + expectedUnits: []string{ + "reading-hcl", + "reading-hcl-and-tfvars", + "reading-tfvars", + }, + }, + { + name: "reading_from_hcl_with_include_and_exclude", + unitsReading: []string{ + "shared.hcl", + "shared.tfvars", + }, + unitsIncluding: []string{ + "reading-tfvars", + }, + unitsExcluding: []string{ + "reading-hcl-and-tfvars", + }, + expectedUnits: []string{ + "reading-hcl", + "reading-tfvars", + }, + }, + } + + includedLogEntryRegex := regexp.MustCompile(`=> Module ./([^ ]+) \(excluded: false`) + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureUnitsReading) + rootPath := util.JoinPath(tmpEnvPath, testFixtureUnitsReading) + + cmd := "terragrunt run-all plan --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir " + rootPath + + for _, unit := range tt.unitsReading { + cmd = cmd + " --terragrunt-queue-include-units-reading " + unit + } + + for _, unit := range tt.unitsIncluding { + cmd = cmd + " --terragrunt-include-dir " + unit + } + + for _, unit := range tt.unitsExcluding { + cmd = cmd + " --terragrunt-exclude-dir " + unit + } + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, cmd) + require.NoError(t, err) + + includedUnits := []string{} + for _, line := range strings.Split(stderr, "\n") { + if includedLogEntryRegex.MatchString(line) { + includedUnits = append(includedUnits, includedLogEntryRegex.FindStringSubmatch(line)[1]) + } + } + + assert.ElementsMatch(t, tt.expectedUnits, includedUnits) + }) + } +} diff --git a/test/integration_test.go b/test/integration_test.go index 43eb5c8226..7159920948 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -3020,12 +3020,12 @@ func TestHclFmtStdin(t *testing.T) { t.Parallel() cleanupTerraformFolder(t, testFixtureHclfmtStdin) - tmpEnvPath := copyEnvironment(t, testFixtureHclfmtStdin) + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureHclfmtStdin) rootPath := util.JoinPath(tmpEnvPath, testFixtureHclfmtStdin) os.Stdin, _ = os.Open(util.JoinPath(rootPath, "terragrunt.hcl")) - stdout, _, err := runTerragruntCommandWithOutput(t, "terragrunt hclfmt --terragrunt-hclfmt-stdin") + stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt hclfmt --terragrunt-hclfmt-stdin") require.NoError(t, err) expectedDiff, err := os.ReadFile(util.JoinPath(rootPath, "expected.hcl"))