feat: Adding units-reading (#3550)

* WIP - feat: Adding units-reading Working on adding more hooks and tests. * fix: Fixing typo in spelling of `functions` * fix: Moving function calls to `locals` so that they are read before DAG build * feat: Adding more function calls * fix: Adding `mark_as_read` * fix: Fixing tests * fix: Cleaning up test * chore: Documenting reading functionality * fix: Markdown linting * fix: Avoid logging extra when multiple flags are set * fix: Fixing tests * Update docs/_docs/04_reference/cli-options.md
gruntwork-io · Nov 13, 2024 · 580998c · 580998c
1 parent 6839411
commit 580998c
Show file tree

Hide file tree

Showing 27 changed files with 727 additions and 114 deletions.
diff --git a/cli/app.go b/cli/app.go
@@ -343,6 +343,21 @@ func initialSetup(cliCtx *cli.Context, opts *options.TerragruntOptions) error {
 		opts.ExcludeByDefault = true
 	}
 
+	if !opts.ExcludeByDefault && len(opts.ModulesThatInclude) > 0 {
+		opts.Logger.Debugf("Modules that include set. Excluding by default.")
+		opts.ExcludeByDefault = true
+	}
+
+	if !opts.ExcludeByDefault && len(opts.UnitsReading) > 0 {
+		opts.Logger.Debugf("Units that read set. Excluding by default.")
+		opts.ExcludeByDefault = true
+	}
+
+	if !opts.ExcludeByDefault && opts.StrictInclude {
+		opts.Logger.Debugf("Strict include set. Excluding by default.")
+		opts.ExcludeByDefault = true
+	}
+
 	opts.IncludeDirs, err = util.GlobCanonicalPath(opts.WorkingDir, opts.IncludeDirs...)
 	if err != nil {
 		return err

diff --git a/cli/commands/flags.go b/cli/commands/flags.go
@@ -67,30 +67,6 @@ const (
 	TerragruntIAMWebIdentityTokenFlagName = "terragrunt-iam-web-identity-token"
 	TerragruntIAMWebIdentityTokenEnvName  = "TERRAGRUNT_IAM_ASSUME_ROLE_WEB_IDENTITY_TOKEN"
 
-	TerragruntIgnoreDependencyErrorsFlagName = "terragrunt-ignore-dependency-errors"
-	TerragruntIgnoreDependencyErrorsEnvName  = "TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS"
-
-	TerragruntIgnoreDependencyOrderFlagName = "terragrunt-ignore-dependency-order"
-	TerragruntIgnoreDependencyOrderEnvName  = "TERRAGRUNT_IGNORE_DEPENDENCY_ORDER"
-
-	TerragruntIgnoreExternalDependenciesFlagName = "terragrunt-ignore-external-dependencies"
-	TerragruntIgnoreExternalDependenciesEnvName  = "TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES"
-
-	TerragruntIncludeExternalDependenciesFlagName = "terragrunt-include-external-dependencies"
-	TerragruntIncludeExternalDependenciesEnvName  = "TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES"
-
-	TerragruntExcludesFileFlagName = "terragrunt-excludes-file"
-	TerragruntExcludesFileEnvName  = "TERRAGRUNT_EXCLUDES_FILE"
-
-	TerragruntExcludeDirFlagName = "terragrunt-exclude-dir"
-	TerragruntExcludeDirEnvName  = "TERRAGRUNT_EXCLUDE_DIR"
-
-	TerragruntIncludeDirFlagName = "terragrunt-include-dir"
-	TerragruntIncludeDirEnvName  = "TERRAGRUNT_INCLUDE_DIR"
-
-	TerragruntStrictIncludeFlagName = "terragrunt-strict-include"
-	TerragruntStrictIncludeEnvName  = "TERRAGRUNT_STRICT_INCLUDE"
-
 	TerragruntParallelismFlagName = "terragrunt-parallelism"
 	TerragruntParallelismEnvName  = "TERRAGRUNT_PARALLELISM"
 
@@ -130,6 +106,35 @@ const (
 	TerragruntNoDestroyDependenciesCheckFlagEnvName = "TERRAGRUNT_NO_DESTROY_DEPENDENCIES_CHECK"
 	TerragruntNoDestroyDependenciesCheckFlagName    = "terragrunt-no-destroy-dependencies-check"
 
+	// Queue related flags
+
+	TerragruntIgnoreDependencyErrorsFlagName = "terragrunt-ignore-dependency-errors"
+	TerragruntIgnoreDependencyErrorsEnvName  = "TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS"
+
+	TerragruntIgnoreDependencyOrderFlagName = "terragrunt-ignore-dependency-order"
+	TerragruntIgnoreDependencyOrderEnvName  = "TERRAGRUNT_IGNORE_DEPENDENCY_ORDER"
+
+	TerragruntIgnoreExternalDependenciesFlagName = "terragrunt-ignore-external-dependencies"
+	TerragruntIgnoreExternalDependenciesEnvName  = "TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES"
+
+	TerragruntIncludeExternalDependenciesFlagName = "terragrunt-include-external-dependencies"
+	TerragruntIncludeExternalDependenciesEnvName  = "TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES"
+
+	TerragruntExcludesFileFlagName = "terragrunt-excludes-file"
+	TerragruntExcludesFileEnvName  = "TERRAGRUNT_EXCLUDES_FILE"
+
+	TerragruntExcludeDirFlagName = "terragrunt-exclude-dir"
+	TerragruntExcludeDirEnvName  = "TERRAGRUNT_EXCLUDE_DIR"
+
+	TerragruntIncludeDirFlagName = "terragrunt-include-dir"
+	TerragruntIncludeDirEnvName  = "TERRAGRUNT_INCLUDE_DIR"
+
+	TerragruntStrictIncludeFlagName = "terragrunt-strict-include"
+	TerragruntStrictIncludeEnvName  = "TERRAGRUNT_STRICT_INCLUDE"
+
+	TerragruntUnitsReadingFlagName = "terragrunt-queue-include-units-reading"
+	TerragruntUnitsReadingEnvName  = "TERRAGRUNT_QUEUE_INCLUDE_UNITS_READING"
+
 	// Logs related flags/envs
 
 	TerragruntLogLevelFlagName = "terragrunt-log-level"
@@ -453,6 +458,12 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
 			Destination: &opts.ModulesThatInclude,
 			Usage:       "If flag is set, 'run-all' will only run the command against Terragrunt modules that include the specified file.",
 		},
+		&cli.SliceFlag[string]{
+			Name:        TerragruntUnitsReadingFlagName,
+			EnvVar:      TerragruntUnitsReadingEnvName,
+			Destination: &opts.UnitsReading,
+			Usage:       "If flag is set, 'run-all' will only run the command against Terragrunt units that read the specified file via an HCL function.",
+		},
 		&cli.BoolFlag{
 			Name:        TerragruntFailOnStateBucketCreationFlagName,
 			EnvVar:      TerragruntFailOnStateBucketCreationEnvName,

diff --git a/config/config_helpers.go b/config/config_helpers.go
@@ -68,6 +68,7 @@ const (
 	FuncNameEndsWith                                = "endswith"
 	FuncNameStrContains                             = "strcontains"
 	FuncNameTimeCmp                                 = "timecmp"
+	FuncNameMarkAsRead                              = "mark_as_read"
 
 	sopsCacheName = "sopsCache"
 )
@@ -167,6 +168,7 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E
 		FuncNameGetDefaultRetryableErrors:               wrapVoidToStringSliceAsFuncImpl(ctx, getDefaultRetryableErrors),
 		FuncNameReadTfvarsFile:                          wrapStringSliceToStringAsFuncImpl(ctx, readTFVarsFile),
 		FuncNameGetWorkingDir:                           wrapVoidToStringAsFuncImpl(ctx, getWorkingDir),
+		FuncNameMarkAsRead:                              wrapStringSliceToStringAsFuncImpl(ctx, markAsRead),
 
 		// Map with HCL functions introduced in Terraform after v0.15.3, since upgrade to a later version is not supported
 		// https://github.com/gruntwork-io/terragrunt/blob/master/go.mod#L22
@@ -627,12 +629,25 @@ func ParseTerragruntConfig(ctx *ParsingContext, configPath string, defaultVal *c
 	targetConfig := getCleanedTargetConfigPath(configPath, ctx.TerragruntOptions.TerragruntConfigPath)
 
 	targetConfigFileExists := util.FileExists(targetConfig)
+
 	if !targetConfigFileExists && defaultVal == nil {
 		return cty.NilVal, errors.New(TerragruntConfigNotFoundError{Path: targetConfig})
-	} else if !targetConfigFileExists {
+	}
+
+	if !targetConfigFileExists {
 		return *defaultVal, nil
 	}
 
+	path, err := util.CanonicalPath(targetConfig, ctx.TerragruntOptions.WorkingDir)
+	if err != nil {
+		return cty.NilVal, errors.New(err)
+	}
+
+	ctx.TerragruntOptions.AppendReadFile(
+		path,
+		ctx.TerragruntOptions.WorkingDir,
+	)
+
 	// We update the ctx of terragruntOptions to the config being read in.
 	opts, err := ctx.TerragruntOptions.Clone(targetConfig)
 	if err != nil {
@@ -812,6 +827,11 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {
 		return "", errors.New(err)
 	}
 
+	ctx.TerragruntOptions.AppendReadFile(
+		canonicalSourceFile,
+		ctx.TerragruntOptions.WorkingDir,
+	)
+
 	// Set environment variables from the TerragruntOptions.Env map.
 	// This is especially useful for integrations with things like the `terragrunt-auth-provider` flag,
 	// which can set environment variables that are used for decryption.
@@ -1008,6 +1028,11 @@ func readTFVarsFile(ctx *ParsingContext, args []string) (string, error) {
 		return "", errors.New(TFVarFileNotFoundError{File: varFile})
 	}
 
+	ctx.TerragruntOptions.AppendReadFile(
+		varFile,
+		ctx.TerragruntOptions.WorkingDir,
+	)
+
 	fileContents, err := os.ReadFile(varFile)
 	if err != nil {
 		return "", errors.New(fmt.Errorf("could not read file %q: %w", varFile, err))
@@ -1036,6 +1061,33 @@ func readTFVarsFile(ctx *ParsingContext, args []string) (string, error) {
 	return string(data), nil
 }
 
+// markAsRead marks a file as explicitly read. This is useful for detection via TerragruntUnitsReading flag.
+func markAsRead(ctx *ParsingContext, args []string) (string, error) {
+	if len(args) != 1 {
+		return "", errors.New(WrongNumberOfParamsError{Func: "mark_as_read", Expected: "1", Actual: len(args)})
+	}
+
+	file := args[0]
+
+	path, err := util.CanonicalPath(file, ctx.TerragruntOptions.WorkingDir)
+	if err != nil {
+		return "", errors.New(err)
+	}
+
+	ctx.TerragruntOptions.AppendReadFile(
+		path,
+		ctx.TerragruntOptions.WorkingDir,
+	)
+
+	return file, nil
+}
+
+// warnWhenFileNotMarkedAsRead warns when a file is not being marked as read, even though a user might expect it to be.
+// Situations where this is the case include:
+// - A user specifies a file in the UnitsReading flag and that file is being read while parsing the inputs attribute.
+//
+// When the file is not marked as read, the function will return true, otherwise false.
+
 // ParseAndDecodeVarFile uses the HCL2 file to parse the given varfile string into an HCL file body, and then decode it
 // into the provided output.
 func ParseAndDecodeVarFile(opts *options.TerragruntOptions, varFile string, fileContents []byte, out interface{}) error {