From ba1e4b5efcb63ee71860c1042b3a007e9b9107e4 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 3 Dec 2024 22:50:42 +0200 Subject: [PATCH] Errors block (#3584) * Error configs * Retry config update * add ignore config * Spacing cleanup * errors config parse * Errors clonning * Add cty values parsing * Errors config clone * retry config fix * Errors block extraction * Retry update * cty test update * metadata errors * cty errors parsing * errors config setup * add errors fields * Update cty test * Errors block proecessing * Default errors block handling * errors block parsing * Errors handling test * Add function to process errors * Retry errors execution * Errors handling * Ignore example execution * add check for not handling example2 errors * Run all ignore test * add test for retries * Add test with multiple retry errors * Add test for failing retries * Add example for signal generation * add searching of parent * Example run all test * Fixed linter errors * errors feature flag * Markdown docs update * Feature flags and errors block * Markdown update * Markdown update * Tests update * Log message update * Description update * add intreruption handling * options update * Docs update * Markdown update * Usage of compilled regexpes * Lint fixes --- cli/commands/terraform/action.go | 19 +- config/catalog.go | 5 + config/config.go | 89 +++++++ config/config_as_cty.go | 38 +++ config/config_as_cty_test.go | 20 ++ config/config_partial.go | 29 ++- config/dependency.go | 2 +- config/errors_block.go | 176 ++++++++++++++ config/include.go | 14 +- config/parsing_context.go | 3 - configstack/stack.go | 1 + .../config-blocks-and-attributes.md | 107 ++++++++ options/options.go | 229 ++++++++++++++++++ test/fixtures/errors/default/main.tf | 0 test/fixtures/errors/default/terragrunt.hcl | 31 +++ test/fixtures/errors/ignore-signal/main.tf | 12 + .../errors/ignore-signal/terragrunt.hcl | 27 +++ test/fixtures/errors/ignore/main.tf | 12 + test/fixtures/errors/ignore/terragrunt.hcl | 15 ++ test/fixtures/errors/retry-fail/main.tf | 12 + test/fixtures/errors/retry-fail/script.sh | 24 ++ .../fixtures/errors/retry-fail/terragrunt.hcl | 15 ++ test/fixtures/errors/retry/main.tf | 12 + test/fixtures/errors/retry/script.sh | 24 ++ test/fixtures/errors/retry/terragrunt.hcl | 15 ++ .../errors/run-all-ignore/app1/main.tf | 12 + .../errors/run-all-ignore/app1/terragrunt.hcl | 15 ++ .../errors/run-all-ignore/app2/main.tf | 3 + .../errors/run-all-ignore/app2/terragrunt.hcl | 0 test/fixtures/errors/run-all/app1/main.tf | 12 + .../errors/run-all/app1/terragrunt.hcl | 3 + test/fixtures/errors/run-all/app2/main.tf | 12 + test/fixtures/errors/run-all/app2/script.sh | 24 ++ .../errors/run-all/app2/terragrunt.hcl | 3 + test/fixtures/errors/run-all/common.hcl | 32 +++ test/integration_errors_test.go | 147 +++++++++++ util/collections.go | 16 ++ 37 files changed, 1194 insertions(+), 16 deletions(-) create mode 100644 config/errors_block.go create mode 100644 test/fixtures/errors/default/main.tf create mode 100644 test/fixtures/errors/default/terragrunt.hcl create mode 100644 test/fixtures/errors/ignore-signal/main.tf create mode 100644 test/fixtures/errors/ignore-signal/terragrunt.hcl create mode 100644 test/fixtures/errors/ignore/main.tf create mode 100644 test/fixtures/errors/ignore/terragrunt.hcl create mode 100644 test/fixtures/errors/retry-fail/main.tf create mode 100755 test/fixtures/errors/retry-fail/script.sh create mode 100644 test/fixtures/errors/retry-fail/terragrunt.hcl create mode 100644 test/fixtures/errors/retry/main.tf create mode 100755 test/fixtures/errors/retry/script.sh create mode 100644 test/fixtures/errors/retry/terragrunt.hcl create mode 100644 test/fixtures/errors/run-all-ignore/app1/main.tf create mode 100644 test/fixtures/errors/run-all-ignore/app1/terragrunt.hcl create mode 100644 test/fixtures/errors/run-all-ignore/app2/main.tf create mode 100644 test/fixtures/errors/run-all-ignore/app2/terragrunt.hcl create mode 100644 test/fixtures/errors/run-all/app1/main.tf create mode 100644 test/fixtures/errors/run-all/app1/terragrunt.hcl create mode 100644 test/fixtures/errors/run-all/app2/main.tf create mode 100755 test/fixtures/errors/run-all/app2/script.sh create mode 100644 test/fixtures/errors/run-all/app2/terragrunt.hcl create mode 100644 test/fixtures/errors/run-all/common.hcl create mode 100644 test/integration_errors_test.go diff --git a/cli/commands/terraform/action.go b/cli/commands/terraform/action.go index 050cf8c2fa..393137f5d3 100644 --- a/cli/commands/terraform/action.go +++ b/cli/commands/terraform/action.go @@ -115,6 +115,13 @@ func runTerraform(ctx context.Context, terragruntOptions *options.TerragruntOpti terragruntOptions.Engine = engine + errConfig, err := terragruntConfig.ErrorsConfig() + if err != nil { + return target.runErrorCallback(terragruntOptions, terragruntConfig, err) + } + + terragruntOptions.Errors = errConfig + terragruntOptionsClone, err := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath) if err != nil { return err @@ -122,7 +129,9 @@ func runTerraform(ctx context.Context, terragruntOptions *options.TerragruntOpti terragruntOptionsClone.TerraformCommand = CommandNameTerragruntReadConfig - if err := processHooks(ctx, terragruntConfig.Terraform.GetAfterHooks(), terragruntOptionsClone, terragruntConfig, nil); err != nil { + if err = terragruntOptionsClone.RunWithErrorHandling(ctx, func() error { + return processHooks(ctx, terragruntConfig.Terraform.GetAfterHooks(), terragruntOptionsClone, terragruntConfig, nil) + }); err != nil { return target.runErrorCallback(terragruntOptions, terragruntConfig, err) } @@ -142,7 +151,9 @@ func runTerraform(ctx context.Context, terragruntOptions *options.TerragruntOpti terragruntOptions.OriginalIAMRoleOptions, ) - if err := credsGetter.ObtainAndUpdateEnvIfNecessary(ctx, terragruntOptions, amazonsts.NewProvider(terragruntOptions)); err != nil { + if err := terragruntOptions.RunWithErrorHandling(ctx, func() error { + return credsGetter.ObtainAndUpdateEnvIfNecessary(ctx, terragruntOptions, amazonsts.NewProvider(terragruntOptions)) + }); err != nil { return err } @@ -234,7 +245,9 @@ func runTerraform(ctx context.Context, terragruntOptions *options.TerragruntOpti } } - if err := runTerragruntWithConfig(ctx, terragruntOptions, updatedTerragruntOptions, terragruntConfig, target); err != nil { + if err := terragruntOptions.RunWithErrorHandling(ctx, func() error { + return runTerragruntWithConfig(ctx, terragruntOptions, updatedTerragruntOptions, terragruntConfig, target) + }); err != nil { return target.runErrorCallback(terragruntOptions, terragruntConfig, err) } diff --git a/config/catalog.go b/config/catalog.go index f70043cf63..a6e731dda0 100644 --- a/config/catalog.go +++ b/config/catalog.go @@ -172,6 +172,11 @@ func convertToTerragruntCatalogConfig(ctx *ParsingContext, configPath string, te terragruntConfig.SetFieldMetadata(MetadataExclude, defaultMetadata) } + if terragruntConfigFromFile.Errors != nil { + terragruntConfig.Errors = terragruntConfigFromFile.Errors + terragruntConfig.SetFieldMetadata(MetadataErrors, defaultMetadata) + } + if ctx.Locals != nil && *ctx.Locals != cty.NilVal { // we should ignore any errors from `parseCtyValueToMap` as some `locals` values might have been incorrectly evaluated, that results to `json.Unmarshal` error. // for example if the locals block looks like `{"var1":, "var2":"value2"}`, `parseCtyValueToMap` returns the map with "var2" value and an syntax error, diff --git a/config/config.go b/config/config.go index 08590f3c30..59b0d831fa 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strconv" "strings" @@ -69,6 +70,9 @@ const ( MetadataInclude = "include" MetadataFeatureFlag = "feature" MetadataExclude = "exclude" + MetadataErrors = "errors" + MetadataRetry = "retry" + MetadataIgnore = "ignore" ) var ( @@ -125,6 +129,7 @@ type TerragruntConfig struct { Engine *EngineConfig FeatureFlags FeatureFlags Exclude *ExcludeConfig + Errors *ErrorsConfig // Fields used for internal tracking // Indicates whether this is the result of a partial evaluation @@ -197,6 +202,7 @@ type terragruntConfigFile struct { TerragruntDependencies []Dependency `hcl:"dependency,block"` FeatureFlags []*FeatureFlag `hcl:"feature,block"` Exclude *ExcludeConfig `hcl:"exclude,block"` + Errors *ErrorsConfig `hcl:"errors,block"` // We allow users to configure code generation via blocks: // @@ -1169,6 +1175,11 @@ func convertToTerragruntConfig(ctx *ParsingContext, configPath string, terragrun terragruntConfig.SetFieldMetadata(MetadataExclude, defaultMetadata) } + if terragruntConfigFromFile.Errors != nil { + terragruntConfig.Errors = terragruntConfigFromFile.Errors + terragruntConfig.SetFieldMetadata(MetadataErrors, defaultMetadata) + } + generateBlocks := []terragruntGenerateBlock{} generateBlocks = append(generateBlocks, terragruntConfigFromFile.GenerateBlocks...) @@ -1434,3 +1445,81 @@ func (cfg *TerragruntConfig) EngineOptions() (*options.EngineOptions, error) { Meta: meta, }, nil } + +// ErrorsConfig fetch errors configuration for options package +func (cfg *TerragruntConfig) ErrorsConfig() (*options.ErrorsConfig, error) { + if cfg.Errors == nil { + return nil, nil + } + + result := &options.ErrorsConfig{ + Retry: make(map[string]*options.RetryConfig), + Ignore: make(map[string]*options.IgnoreConfig), + } + + for _, retryBlock := range cfg.Errors.Retry { + if retryBlock == nil { + continue + } + + compiledPatterns := make([]*regexp.Regexp, 0, len(retryBlock.RetryableErrors)) + + for _, pattern := range retryBlock.RetryableErrors { + compiled, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid retry pattern %q in block %q: %w", + pattern, retryBlock.Label, err) + } + + compiledPatterns = append(compiledPatterns, compiled) + } + + result.Retry[retryBlock.Label] = &options.RetryConfig{ + Name: retryBlock.Label, + RetryableErrors: compiledPatterns, + MaxAttempts: retryBlock.MaxAttempts, + SleepIntervalSec: retryBlock.SleepIntervalSec, + } + } + + for _, ignoreBlock := range cfg.Errors.Ignore { + if ignoreBlock == nil { + continue + } + + var signals map[string]interface{} + + if ignoreBlock.Signals != nil { + value, err := convertValuesMapToCtyVal(ignoreBlock.Signals) + if err != nil { + return nil, err + } + + signals, err = ParseCtyValueToMap(value) + if err != nil { + return nil, err + } + } + + compiledPatterns := make([]*regexp.Regexp, 0, len(ignoreBlock.IgnorableErrors)) + + for _, pattern := range ignoreBlock.IgnorableErrors { + compiled, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid retry pattern %q in block %q: %w", + pattern, ignoreBlock.Label, err) + } + + compiledPatterns = append(compiledPatterns, compiled) + } + + result.Ignore[ignoreBlock.Label] = &options.IgnoreConfig{ + Name: ignoreBlock.Label, + IgnorableErrors: compiledPatterns, + Message: ignoreBlock.Message, + Signals: signals, + } + } + + return result, nil +} diff --git a/config/config_as_cty.go b/config/config_as_cty.go index 554efd87d8..012c996802 100644 --- a/config/config_as_cty.go +++ b/config/config_as_cty.go @@ -59,6 +59,15 @@ func TerragruntConfigAsCty(config *TerragruntConfig) (cty.Value, error) { output[MetadataExclude] = excludeConfigCty } + errorsConfigCty, err := errorsConfigAsCty(config.Errors) + if err != nil { + return cty.NilVal, err + } + + if errorsConfigCty != cty.NilVal { + output[MetadataErrors] = errorsConfigCty + } + terraformConfigCty, err := terraformConfigAsCty(config.Terraform) if err != nil { return cty.NilVal, err @@ -649,6 +658,35 @@ func featureFlagsBlocksAsCty(featureFlagBlocks FeatureFlags) (cty.Value, error) return convertValuesMapToCtyVal(out) } +// Serialize errors configuration as cty.Value. +func errorsConfigAsCty(config *ErrorsConfig) (cty.Value, error) { + if config == nil { + return cty.NilVal, nil + } + + output := map[string]cty.Value{} + + retryCty, err := goTypeToCty(config.Retry) + if err != nil { + return cty.NilVal, err + } + + if retryCty != cty.NilVal { + output[MetadataRetry] = retryCty + } + + ignoreCty, err := goTypeToCty(config.Ignore) + if err != nil { + return cty.NilVal, err + } + + if ignoreCty != cty.NilVal { + output[MetadataIgnore] = ignoreCty + } + + return convertValuesMapToCtyVal(output) +} + // Converts arbitrary go types that are json serializable to a cty Value by using json as an intermediary // representation. This avoids the strict type nature of cty, where you need to know the output type beforehand to // serialize to cty. diff --git a/config/config_as_cty_test.go b/config/config_as_cty_test.go index aa0b3d01e4..78b64b6118 100644 --- a/config/config_as_cty_test.go +++ b/config/config_as_cty_test.go @@ -108,6 +108,24 @@ func TestTerragruntConfigAsCtyDrift(t *testing.T) { Default: &cty.Zero, }, }, + Errors: &config.ErrorsConfig{ + Retry: []*config.RetryBlock{ + { + Label: "test", + RetryableErrors: []string{"test"}, + MaxAttempts: 0, + SleepIntervalSec: 0, + }, + }, + Ignore: []*config.IgnoreBlock{ + { + Label: "test", + IgnorableErrors: nil, + Message: "", + Signals: nil, + }, + }, + }, GenerateConfigs: map[string]codegen.GenerateConfig{ "provider": { Path: "foo", @@ -260,6 +278,8 @@ func terragruntConfigStructFieldToMapKey(t *testing.T, fieldName string) (string return "feature", true case "Exclude": return "exclude", true + case "Errors": + return "errors", true default: t.Fatalf("Unknown struct property: %s", fieldName) // This should not execute diff --git a/config/config_partial.go b/config/config_partial.go index f46cfc4b5c..98bfe20f18 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -34,6 +34,7 @@ const ( RemoteStateBlock FeatureFlagsBlock ExcludeBlock + ErrorsBlock ) // terragruntIncludeMultiple is a struct that can be used to only decode the include block with labels. @@ -54,10 +55,10 @@ type terragruntFeatureFlags struct { Remain hcl.Body `hcl:",remain"` } -// terragruntExclude is a struct that can be used to only decode the exclude block. -type terragruntExclude struct { - Exclude *ExcludeConfig `hcl:"exclude,block"` - Remain hcl.Body `hcl:",remain"` +// terragruntErrors struct to decode errors block +type terragruntErrors struct { + Errors *ErrorsConfig `hcl:"errors,block"` + Remain hcl.Body `hcl:",remain"` } // terragruntTerraform is a struct that can be used to only decode the terraform block. @@ -535,9 +536,7 @@ func PartialParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChi } case ExcludeBlock: - decoded := terragruntExclude{} - err := file.Decode(&decoded, evalParsingContext) - + decoded, err := processExcludes(ctx, output, file) if err != nil { return nil, err } @@ -548,12 +547,26 @@ func PartialParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChi output.Exclude = decoded.Exclude } + case ErrorsBlock: + decoded := terragruntErrors{} + err := file.Decode(&decoded, evalParsingContext) + + if err != nil { + return nil, err + } + + if output.Errors != nil { + output.Errors.Merge(decoded.Errors) + } else { + output.Errors = decoded.Errors + } + default: return nil, InvalidPartialBlockName{decode} } } - // If this file includes another, parse and merge the partial blocks. Otherwise just return this config. + // If this file includes another, parse and merge the partial blocks. Otherwise, just return this config. if len(ctx.TrackInclude.CurrentList) > 0 { config, err := handleInclude(ctx, output, true) if err != nil { diff --git a/config/dependency.go b/config/dependency.go index 9e1245e3d9..83a804074c 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -635,7 +635,7 @@ func getOutputJSONWithCaching(ctx *ParsingContext, targetConfig string) ([]byte, // - The config path to the dependency module's config // - The original config path to the dependency module's config // -// That way, everything in that dependnecy happens within its own ctx. +// That way, everything in that dependency happens within its own ctx. func cloneTerragruntOptionsForDependency(ctx *ParsingContext, targetConfigPath string) (*options.TerragruntOptions, error) { targetOptions, err := ctx.TerragruntOptions.Clone(targetConfigPath) if err != nil { diff --git a/config/errors_block.go b/config/errors_block.go new file mode 100644 index 0000000000..9a9069da8d --- /dev/null +++ b/config/errors_block.go @@ -0,0 +1,176 @@ +package config + +import ( + "github.com/gruntwork-io/terragrunt/util" + "github.com/zclconf/go-cty/cty" +) + +// ErrorsConfig represents the top-level errors configuration +type ErrorsConfig struct { + Retry []*RetryBlock `cty:"retry" hcl:"retry,block"` + Ignore []*IgnoreBlock `cty:"ignore" hcl:"ignore,block"` +} + +// RetryBlock represents a labeled retry block +type RetryBlock struct { + Label string `cty:"name" hcl:"name,label"` + RetryableErrors []string `cty:"retryable_errors" hcl:"retryable_errors"` + MaxAttempts int `cty:"max_attempts" hcl:"max_attempts"` + SleepIntervalSec int `cty:"sleep_interval_sec" hcl:"sleep_interval_sec"` +} + +// IgnoreBlock represents a labeled ignore block +type IgnoreBlock struct { + Label string `cty:"name" hcl:"name,label"` + IgnorableErrors []string `cty:"ignorable_errors" hcl:"ignorable_errors"` + Message string `cty:"message" hcl:"message,optional"` + Signals map[string]cty.Value `cty:"signals" hcl:"signals,optional"` +} + +// Clone creates a deep copy of ErrorsConfig +func (c *ErrorsConfig) Clone() *ErrorsConfig { + if c == nil { + return nil + } + + clone := &ErrorsConfig{ + Retry: make([]*RetryBlock, len(c.Retry)), + Ignore: make([]*IgnoreBlock, len(c.Ignore)), + } + + // Clone Retry blocks + for i, retry := range c.Retry { + clone.Retry[i] = retry.Clone() + } + + // Clone Ignore blocks + for i, ignore := range c.Ignore { + clone.Ignore[i] = ignore.Clone() + } + + return clone +} + +// Merge combines the current ErrorsConfig with another one, with the other config taking precedence +func (c *ErrorsConfig) Merge(other *ErrorsConfig) { + if other == nil { + return + } + + if c == nil { + *c = *other + return + } + + retryMap := make(map[string]*RetryBlock) + for _, block := range c.Retry { + retryMap[block.Label] = block + } + + ignoreMap := make(map[string]*IgnoreBlock) + for _, block := range c.Ignore { + ignoreMap[block.Label] = block + } + + // Merge retry blocks + for _, otherBlock := range other.Retry { + if existing, exists := retryMap[otherBlock.Label]; exists { + existing.RetryableErrors = util.MergeStringSlices(existing.RetryableErrors, otherBlock.RetryableErrors) + + if otherBlock.MaxAttempts > 0 { + existing.MaxAttempts = otherBlock.MaxAttempts + } + + if otherBlock.SleepIntervalSec > 0 { + existing.SleepIntervalSec = otherBlock.SleepIntervalSec + } + } else { + // Add new block + retryMap[otherBlock.Label] = otherBlock + } + } + + // Merge ignore blocks + for _, otherBlock := range other.Ignore { + if existing, exists := ignoreMap[otherBlock.Label]; exists { + existing.IgnorableErrors = util.MergeStringSlices(existing.IgnorableErrors, otherBlock.IgnorableErrors) + + if otherBlock.Message != "" { + existing.Message = otherBlock.Message + } + + if otherBlock.Signals != nil { + if existing.Signals == nil { + existing.Signals = make(map[string]cty.Value) + } + + for k, v := range otherBlock.Signals { + existing.Signals[k] = v + } + } + } else { + // Add new block + ignoreMap[otherBlock.Label] = otherBlock + } + } + + // Convert maps back to slices + c.Retry = make([]*RetryBlock, 0, len(retryMap)) + for _, block := range retryMap { + c.Retry = append(c.Retry, block) + } + + c.Ignore = make([]*IgnoreBlock, 0, len(ignoreMap)) + for _, block := range ignoreMap { + c.Ignore = append(c.Ignore, block) + } +} + +// Clone creates a deep copy of RetryBlock +func (r *RetryBlock) Clone() *RetryBlock { + if r == nil { + return nil + } + + clone := &RetryBlock{ + Label: r.Label, + MaxAttempts: r.MaxAttempts, + SleepIntervalSec: r.SleepIntervalSec, + } + + // Deep copy RetryableErrors slice + if r.RetryableErrors != nil { + clone.RetryableErrors = make([]string, len(r.RetryableErrors)) + copy(clone.RetryableErrors, r.RetryableErrors) + } + + return clone +} + +// Clone creates a deep copy of IgnoreBlock +func (i *IgnoreBlock) Clone() *IgnoreBlock { + if i == nil { + return nil + } + + clone := &IgnoreBlock{ + Label: i.Label, + Message: i.Message, + } + + // Deep copy IgnorableErrors slice + if i.IgnorableErrors != nil { + clone.IgnorableErrors = make([]string, len(i.IgnorableErrors)) + copy(clone.IgnorableErrors, i.IgnorableErrors) + } + + // Deep copy Signals map + if i.Signals != nil { + clone.Signals = make(map[string]cty.Value, len(i.Signals)) + for k, v := range i.Signals { + clone.Signals[k] = v + } + } + + return clone +} diff --git a/config/include.go b/config/include.go index ad531f1d02..6dc44ec25e 100644 --- a/config/include.go +++ b/config/include.go @@ -181,7 +181,7 @@ func handleIncludeForDependency(ctx *ParsingContext, childDecodedDependency Terr } includedPartialParse, err := partialParseIncludedConfig( - ctx.WithDecodeList(DependencyBlock, FeatureFlagsBlock, ExcludeBlock), &includeConfig) + ctx.WithDecodeList(DependencyBlock, FeatureFlagsBlock, ExcludeBlock, ErrorsBlock), &includeConfig) if err != nil { return nil, err } @@ -271,6 +271,10 @@ func (cfg *TerragruntConfig) Merge(sourceConfig *TerragruntConfig, terragruntOpt cfg.Exclude = sourceConfig.Exclude.Clone() } + if sourceConfig.Errors != nil { + cfg.Errors = sourceConfig.Errors.Clone() + } + if sourceConfig.RemoteState != nil { cfg.RemoteState = sourceConfig.RemoteState } @@ -403,6 +407,14 @@ func (cfg *TerragruntConfig) DeepMerge(sourceConfig *TerragruntConfig, terragrun cfg.Exclude.Merge(sourceConfig.Exclude) } + if sourceConfig.Errors != nil { + if cfg.Errors == nil { + cfg.Errors = &ErrorsConfig{} + } + + cfg.Errors.Merge(sourceConfig.Errors) + } + if sourceConfig.Skip != nil { cfg.Skip = sourceConfig.Skip } diff --git a/config/parsing_context.go b/config/parsing_context.go index f9ee78f2bd..e55f60b0ec 100644 --- a/config/parsing_context.go +++ b/config/parsing_context.go @@ -27,9 +27,6 @@ type ParsingContext struct { // Features are the feature flags that are enabled for the current terragrunt config. Features *cty.Value - // Exclude is the configuration for excluding certain terragrunt configurations. - Exclude *cty.Value - // DecodedDependencies are references of other terragrunt config. This contains the following attributes that map to // various fields related to that config: // - outputs: The map of outputs from the terraform state obtained by running `terragrunt output` on that target config. diff --git a/configstack/stack.go b/configstack/stack.go index 33f6bb27be..7eec5cdd2e 100644 --- a/configstack/stack.go +++ b/configstack/stack.go @@ -593,6 +593,7 @@ func (stack *Stack) resolveTerraformModule(ctx context.Context, terragruntConfig config.DependenciesBlock, config.DependencyBlock, config.FeatureFlagsBlock, + config.ErrorsBlock, ) // Credentials have to be acquired before the config is parsed, as the config may contain interpolation functions diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index febe376fb7..07d6bf2ed5 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -57,6 +57,7 @@ The following is a reference of all the supported blocks and attributes in the c - [generate](#generate) - [engine](#engine) - [feature](#feature) +- [errors](#errors) ### terraform @@ -1291,6 +1292,112 @@ This setup is useful for scenarios where output evaluation is still needed, even Consider using this for units that are expensive to continuously update, and can be opted in when necessary. +### errors + +The `errors` block contains all the configurations for handling errors. Each configuration block, such as `retry` and `ignore`, +is nested within the `errors` block to define specific error-handling strategies. + +**Retry Configuration** +The `retry` block within the `errors` block defines rules for retrying operations when specific errors occur. +This is useful for handling intermittent errors that may resolve after a short delay or multiple attempts. + +Example: Retry Configuration + +```hcl +errors { + retry "retry_example" { + retryable_errors = [".*Error: transient.*"] # Matches errors containing 'Error: transient' + max_attempts = 5 # Retry up to 5 times + sleep_interval_sec = 10 # Wait 10 seconds between retries + } +} +``` + +Parameters: + +- `retryable_errors`: A list of regex patterns to match errors eligible for retry. + - Example: `".*Error: transient.*"` matches errors containing `Error: transient`. +- `max_attempts`: The maximum number of retry attempts. + - Example: `5` retries. +- `sleep_interval_sec`: Time (in seconds) to wait between retries. + - Example: `10` seconds. + +**Ignore Configuration** +The `ignore` block within the `errors` block defines rules for ignoring specific errors. This is useful when certain +errors are known to be safe and should not halt operations. + +Example: Ignore Configuration + +```hcl +errors { + ignore "ignore_example" { + ignorable_errors = [ + ".*Error: safe-to-ignore.*", # Ignore errors containing 'Error: safe-to-ignore' + "!.*Error: critical.*" # Do not ignore errors containing 'Error: critical' + ] + message = "Ignoring safe-to-ignore errors" # Optional message displayed when ignoring errors + signals = { + safe_to_revert = true # Indicates the operation is safe to revert on failure + } + } +} +``` + +Parameters: + +- `ignorable_errors`: A list of regex patterns to define errors to ignore. + - `"Error: safe-to-ignore.*"`: Ignores errors containing `Error: safe-to-ignore`. + - `"!Error: critical.*"`: Ensures errors containing `Error: critical` are not ignored. +- `message` (Optional): A warning message displayed when an error is ignored. + - Example: `"Ignoring safe-to-ignore errors"`. +- `signals` (Optional): Key-value pairs used to emit signals to external systems. + - Example: `safe_to_revert = true` indicates it is safe to revert the operation if it fails. + +Populating values into the `signals` attribute results in a JSON file named `error-signals.json` being emitted on failure. +This file can be inspected in CI/CD systems to determine the recommended course of action to address the failure. + +Example: + +If an error occurs and the author of the unit has signaled `safe_to_revert = true`, the CI/CD system could follow a standard process: + +- Identify all units with files named `error-signals.json`. +- Checkout the previous commit for those units. +- Apply the units in their previous state, effectively reverting their updates. + +This approach ensures consistent and automated error handling in complex pipelines. + +**Combined Example** +Below is a combined example showcasing both retry and ignore configurations within the `errors` block. + +```hcl +errors { + # Retry block for transient errors + retry "transient_errors" { + retryable_errors = [".*Error: transient network issue.*"] + max_attempts = 3 + sleep_interval_sec = 5 + } + + # Ignore block for known safe-to-ignore errors + ignore "known_safe_errors" { + ignorable_errors = [ + ".*Error: safe warning.*", + "!.*Error: do not ignore.*" + ] + message = "Ignoring safe warning errors" + signals = { + alert_team = false + } + } +} +``` + +Notes: + +- All retry and ignore configurations must be defined within the `errors` block. +- The `retry` block is prioritized over legacy retry fields (`retryable_errors`, `retry_max_attempts`, `retry_sleep_interval_sec`). +- Conditional logic can be used within `ignorable_errors` to enable or disable rules dynamically. + ## Attributes - [Blocks](#blocks) diff --git a/options/options.go b/options/options.go index 8c7a2b326b..abd036b795 100644 --- a/options/options.go +++ b/options/options.go @@ -3,11 +3,13 @@ package options import ( "context" + "encoding/json" "fmt" "io" "math" "os" "path/filepath" + "regexp" "time" "github.com/gruntwork-io/terragrunt/internal/errors" @@ -36,6 +38,8 @@ const ( // Default to naming it `terragrunt_rendered.json` in the terragrunt config directory. DefaultJSONOutName = "terragrunt_rendered.json" + DefaultSignalsFile = "error-signals.json" + DefaultTFDataDir = ".terraform" DefaultIAMAssumeRoleDuration = 3600 @@ -366,6 +370,9 @@ type TerragruntOptions struct { // ReadFiles is a map of files to the Units // that read them using HCL functions in the unit. ReadFiles *xsync.MapOf[string, []string] + + // Errors is a configuration for error handling. + Errors *ErrorsConfig } // TerragruntOptionsFunc is a functional option type used to pass options in certain integration tests @@ -642,6 +649,7 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp // copy array StrictControls: util.CloneStringList(opts.StrictControls), FeatureFlags: opts.FeatureFlags, + Errors: cloneErrorsConfig(opts.Errors), }, nil } @@ -824,5 +832,226 @@ type EngineOptions struct { Meta map[string]interface{} } +// ErrorsConfig extracted errors handling configuration. +type ErrorsConfig struct { + Retry map[string]*RetryConfig + Ignore map[string]*IgnoreConfig +} + +// RetryConfig represents the configuration for retrying specific errors. +type RetryConfig struct { + Name string + RetryableErrors []*regexp.Regexp + MaxAttempts int + SleepIntervalSec int +} + +// IgnoreConfig represents the configuration for ignoring specific errors. +type IgnoreConfig struct { + Name string + IgnorableErrors []*regexp.Regexp + Message string + Signals map[string]interface{} +} + +func cloneErrorsConfig(config *ErrorsConfig) *ErrorsConfig { + if config == nil { + return nil + } + + // Create a new Errors + cloned := &ErrorsConfig{ + Retry: make(map[string]*RetryConfig), + Ignore: make(map[string]*IgnoreConfig), + } + + // Clone Retry configurations + for key, retryConfig := range config.Retry { + if retryConfig != nil { + cloned.Retry[key] = &RetryConfig{ + Name: retryConfig.Name, + MaxAttempts: retryConfig.MaxAttempts, + SleepIntervalSec: retryConfig.SleepIntervalSec, + RetryableErrors: make([]*regexp.Regexp, len(retryConfig.RetryableErrors)), + } + // Deep copy the RetryableErrors slice + copy(cloned.Retry[key].RetryableErrors, retryConfig.RetryableErrors) + } + } + + // Clone Ignore configurations + for key, ignoreConfig := range config.Ignore { + if ignoreConfig != nil { + cloned.Ignore[key] = &IgnoreConfig{ + Name: ignoreConfig.Name, + Message: ignoreConfig.Message, + IgnorableErrors: make([]*regexp.Regexp, len(ignoreConfig.IgnorableErrors)), + Signals: make(map[string]interface{}), + } + // Deep copy the IgnorableErrors slice + copy(cloned.Ignore[key].IgnorableErrors, ignoreConfig.IgnorableErrors) + + // Deep copy the Signals map + for sigKey, sigVal := range ignoreConfig.Signals { + cloned.Ignore[key].Signals[sigKey] = sigVal + } + } + } + + return cloned +} + +// RunWithErrorHandling runs the given operation and handles any errors according to the configuration. +func (opts *TerragruntOptions) RunWithErrorHandling(ctx context.Context, operation func() error) error { + if opts.Errors == nil { + return operation() + } + + currentAttempt := 1 + + for { + err := operation() + if err == nil { + return nil + } + + // Process the error through our error handling configuration + action, processErr := opts.Errors.ProcessError(err, currentAttempt) + if processErr != nil { + return fmt.Errorf("error processing error handling rules: %w", processErr) + } + + if action == nil { + return err + } + + if action.ShouldIgnore { + opts.Logger.Warnf("Ignoring error, reason: %s", action.IgnoreMessage) + + // Handle ignore signals if any are configured + if len(action.IgnoreSignals) > 0 { + if err := opts.handleIgnoreSignals(action.IgnoreSignals); err != nil { + return err + } + } + + return nil + } + + if action.ShouldRetry { + opts.Logger.Warnf( + "Encountered retryable error: %s\nAttempt %d of %d. Waiting %d second(s) before retrying...", + action.RetryMessage, + currentAttempt, + action.RetryAttempts, + action.RetrySleepSecs, + ) + + // Sleep before retry + select { + case <-time.After(time.Duration(action.RetrySleepSecs) * time.Second): + // try again + case <-ctx.Done(): + return errors.New(ctx.Err()) + } + + currentAttempt++ + + continue + } + + return err + } +} + +func (opts *TerragruntOptions) handleIgnoreSignals(signals map[string]interface{}) error { + workingDir := opts.WorkingDir + signalsFile := filepath.Join(workingDir, DefaultSignalsFile) + signalsJSON, err := json.MarshalIndent(signals, "", " ") + + if err != nil { + return err + } + + const ownerPerms = 0644 + if err := os.WriteFile(signalsFile, signalsJSON, ownerPerms); err != nil { + return fmt.Errorf("failed to write signals file %s: %w", signalsFile, err) + } + + opts.Logger.Warnf("Written error signals to %s", signalsFile) + + return nil +} + +// ErrorAction represents the action to take when an error occurs +type ErrorAction struct { + ShouldIgnore bool + ShouldRetry bool + IgnoreMessage string + IgnoreSignals map[string]interface{} + RetryMessage string + RetryAttempts int + RetrySleepSecs int +} + +// ProcessError evaluates an error against the configuration and returns the appropriate action +func (c *ErrorsConfig) ProcessError(err error, currentAttempt int) (*ErrorAction, error) { + if err == nil { + return nil, nil + } + + errStr := err.Error() + action := &ErrorAction{} + + // First check ignore rules + for _, ignoreBlock := range c.Ignore { + isIgnorable := matchesAnyRegexpPattern(errStr, ignoreBlock.IgnorableErrors) + if isIgnorable { + action.ShouldIgnore = true + action.IgnoreMessage = ignoreBlock.Message + action.IgnoreSignals = make(map[string]interface{}) + + // Convert cty.Value map to regular map + for k, v := range ignoreBlock.Signals { + action.IgnoreSignals[k] = v + } + + return action, nil + } + } + + // Then check retry rules + for _, retryBlock := range c.Retry { + isRetryable := matchesAnyRegexpPattern(errStr, retryBlock.RetryableErrors) + if isRetryable { + if currentAttempt >= retryBlock.MaxAttempts { + return nil, errors.New(fmt.Sprintf("max retry attempts (%d) reached for error: %v", + retryBlock.MaxAttempts, err)) + } + + action.RetryMessage = retryBlock.Name + action.ShouldRetry = true + action.RetryAttempts = retryBlock.MaxAttempts + action.RetrySleepSecs = retryBlock.SleepIntervalSec + + return action, nil + } + } + + return nil, err +} + +// matchesAnyRegexpPattern checks if the input string matches any of the provided compiled patterns +func matchesAnyRegexpPattern(input string, patterns []*regexp.Regexp) bool { + for _, pattern := range patterns { + matched := pattern.MatchString(input) + if matched { + return matched + } + } + + return false +} + // ErrRunTerragruntCommandNotSet is a custom error type indicating that the command is not set. var ErrRunTerragruntCommandNotSet = errors.New("the RunTerragrunt option has not been set on this TerragruntOptions object") diff --git a/test/fixtures/errors/default/main.tf b/test/fixtures/errors/default/main.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/fixtures/errors/default/terragrunt.hcl b/test/fixtures/errors/default/terragrunt.hcl new file mode 100644 index 0000000000..669d56016a --- /dev/null +++ b/test/fixtures/errors/default/terragrunt.hcl @@ -0,0 +1,31 @@ +feature "feature_name" { + default = false +} + +errors { + # Retry configuration block that allows for retrying errors that are known to be intermittent + # Note that this replaces `retryable_errors`, `retry_max_attempts` and `retry_sleep_interval_sec` fields. + # Those fields will still be supported for backwards compatibility, but this block will take precedence. + retry "foo" { + retryable_errors = !feature.feature_name.value ? [] : [ + ".*Error: foo.*" + ] + max_attempts = 3 + sleep_interval_sec = 5 + } + + # Ignore configuration block that allows for ignoring errors that are known to be safe to ignore + ignore "bar" { + # Specify a pattern that will be detected in the error for ignores, or just ignore any error + ignorable_errors = [ + ".*Error: bar.*", # If STDERR includes "Error: bar", ignore it + "!.*Error: baz.*" # If STDERR includes "Error: baz", do not ignore it + ] + message = "Ignoring error bar" # Add an optional warning message if it fails + # Key-value map that can be used to emit signals to external systems on failure + signals = { + safe_to_revert = true # Signal that the apply is safe to revert on failure + } + + } +} \ No newline at end of file diff --git a/test/fixtures/errors/ignore-signal/main.tf b/test/fixtures/errors/ignore-signal/main.tf new file mode 100644 index 0000000000..5f4d557202 --- /dev/null +++ b/test/fixtures/errors/ignore-signal/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "error_generator" { + provisioner "local-exec" { + command = "echo 'Generating example1 error' && exit 1" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/ignore-signal/terragrunt.hcl b/test/fixtures/errors/ignore-signal/terragrunt.hcl new file mode 100644 index 0000000000..88260d3560 --- /dev/null +++ b/test/fixtures/errors/ignore-signal/terragrunt.hcl @@ -0,0 +1,27 @@ +errors { + ignore "example1" { + ignorable_errors = [ + ".*example1.*", + ] + message = "Ignoring error example1" + + signals = { + failed = false + failed_example1 = true + message = "Failed example1" + } + } + + ignore "example2" { + ignorable_errors = [ + ".*example2.*", + ] + message = "Ignoring error example2" + + signals = { + failed = false + failed_example2 = true + message = "Failed example2" + } + } +} \ No newline at end of file diff --git a/test/fixtures/errors/ignore/main.tf b/test/fixtures/errors/ignore/main.tf new file mode 100644 index 0000000000..5f4d557202 --- /dev/null +++ b/test/fixtures/errors/ignore/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "error_generator" { + provisioner "local-exec" { + command = "echo 'Generating example1 error' && exit 1" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/ignore/terragrunt.hcl b/test/fixtures/errors/ignore/terragrunt.hcl new file mode 100644 index 0000000000..40775e8a01 --- /dev/null +++ b/test/fixtures/errors/ignore/terragrunt.hcl @@ -0,0 +1,15 @@ +errors { + ignore "example1" { + ignorable_errors = [ + ".*example1.*", + ] + message = "Ignoring error example1" + } + + ignore "example2" { + ignorable_errors = [ + ".*example2.*", + ] + message = "Ignoring error example2" + } +} \ No newline at end of file diff --git a/test/fixtures/errors/retry-fail/main.tf b/test/fixtures/errors/retry-fail/main.tf new file mode 100644 index 0000000000..9a61592576 --- /dev/null +++ b/test/fixtures/errors/retry-fail/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "script_runner" { + provisioner "local-exec" { + command = "./script.sh 10" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/retry-fail/script.sh b/test/fixtures/errors/retry-fail/script.sh new file mode 100755 index 0000000000..aa117b7b68 --- /dev/null +++ b/test/fixtures/errors/retry-fail/script.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# script that will fail before $1 attempts + +RETRY_ATTEMPTS="$1" +COUNTER_FILE="attempt_counter.txt" + +if [[ ! -f "$COUNTER_FILE" ]]; then + echo "0" > "$COUNTER_FILE" +fi + +CURRENT_COUNT=$(($(cat "$COUNTER_FILE") + 1)) + +echo "$CURRENT_COUNT" > "$COUNTER_FILE" + +echo "Current attempt: $CURRENT_COUNT" + +if [ "$CURRENT_COUNT" -eq "$RETRY_ATTEMPTS" ]; then + echo "Success !" + echo "0" > "$COUNTER_FILE" + exit 0 +else + echo "Script error: Attempt $CURRENT_COUNT failed. Will succeed on attempt $RETRY_ATTEMPTS." >&2 + exit 1 +fi \ No newline at end of file diff --git a/test/fixtures/errors/retry-fail/terragrunt.hcl b/test/fixtures/errors/retry-fail/terragrunt.hcl new file mode 100644 index 0000000000..388c5227c6 --- /dev/null +++ b/test/fixtures/errors/retry-fail/terragrunt.hcl @@ -0,0 +1,15 @@ +errors { + + retry "script_errors" { + retryable_errors = [".*Script error.*"] + max_attempts = 3 + sleep_interval_sec = 2 + } + + retry "aws_errors" { + retryable_errors = [".*AWS error.*"] + max_attempts = 3 + sleep_interval_sec = 2 + } + +} \ No newline at end of file diff --git a/test/fixtures/errors/retry/main.tf b/test/fixtures/errors/retry/main.tf new file mode 100644 index 0000000000..85991e287c --- /dev/null +++ b/test/fixtures/errors/retry/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "script_runner" { + provisioner "local-exec" { + command = "./script.sh 3" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/retry/script.sh b/test/fixtures/errors/retry/script.sh new file mode 100755 index 0000000000..aeac91a219 --- /dev/null +++ b/test/fixtures/errors/retry/script.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# script that will fail before $1 attempts + +RETRY_ATTEMPTS="$1" +COUNTER_FILE="attempt_counter.txt" + +if [[ ! -f "$COUNTER_FILE" ]]; then + echo "0" > "$COUNTER_FILE" +fi + +CURRENT_COUNT=$(($(cat "$COUNTER_FILE") + 1)) + +echo "$CURRENT_COUNT" > "$COUNTER_FILE" + +echo "Current attempt: $CURRENT_COUNT" + +if [ "$CURRENT_COUNT" -eq "$RETRY_ATTEMPTS" ]; then + echo "Success !" + echo "0" > "$COUNTER_FILE" + exit 0 +else + echo "Script error: Attempt $CURRENT_COUNT failed. Will succeed on attempt $RETRY_ATTEMPTS." >&2 + exit 1 +fi \ No newline at end of file diff --git a/test/fixtures/errors/retry/terragrunt.hcl b/test/fixtures/errors/retry/terragrunt.hcl new file mode 100644 index 0000000000..388c5227c6 --- /dev/null +++ b/test/fixtures/errors/retry/terragrunt.hcl @@ -0,0 +1,15 @@ +errors { + + retry "script_errors" { + retryable_errors = [".*Script error.*"] + max_attempts = 3 + sleep_interval_sec = 2 + } + + retry "aws_errors" { + retryable_errors = [".*AWS error.*"] + max_attempts = 3 + sleep_interval_sec = 2 + } + +} \ No newline at end of file diff --git a/test/fixtures/errors/run-all-ignore/app1/main.tf b/test/fixtures/errors/run-all-ignore/app1/main.tf new file mode 100644 index 0000000000..5f4d557202 --- /dev/null +++ b/test/fixtures/errors/run-all-ignore/app1/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "error_generator" { + provisioner "local-exec" { + command = "echo 'Generating example1 error' && exit 1" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/run-all-ignore/app1/terragrunt.hcl b/test/fixtures/errors/run-all-ignore/app1/terragrunt.hcl new file mode 100644 index 0000000000..40775e8a01 --- /dev/null +++ b/test/fixtures/errors/run-all-ignore/app1/terragrunt.hcl @@ -0,0 +1,15 @@ +errors { + ignore "example1" { + ignorable_errors = [ + ".*example1.*", + ] + message = "Ignoring error example1" + } + + ignore "example2" { + ignorable_errors = [ + ".*example2.*", + ] + message = "Ignoring error example2" + } +} \ No newline at end of file diff --git a/test/fixtures/errors/run-all-ignore/app2/main.tf b/test/fixtures/errors/run-all-ignore/app2/main.tf new file mode 100644 index 0000000000..d9ee932d05 --- /dev/null +++ b/test/fixtures/errors/run-all-ignore/app2/main.tf @@ -0,0 +1,3 @@ +output "app2" { + value = "value-from-app-2" +} \ No newline at end of file diff --git a/test/fixtures/errors/run-all-ignore/app2/terragrunt.hcl b/test/fixtures/errors/run-all-ignore/app2/terragrunt.hcl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/fixtures/errors/run-all/app1/main.tf b/test/fixtures/errors/run-all/app1/main.tf new file mode 100644 index 0000000000..5f4d557202 --- /dev/null +++ b/test/fixtures/errors/run-all/app1/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "error_generator" { + provisioner "local-exec" { + command = "echo 'Generating example1 error' && exit 1" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/run-all/app1/terragrunt.hcl b/test/fixtures/errors/run-all/app1/terragrunt.hcl new file mode 100644 index 0000000000..3ba7f1118b --- /dev/null +++ b/test/fixtures/errors/run-all/app1/terragrunt.hcl @@ -0,0 +1,3 @@ +include "common" { + path = find_in_parent_folders("common.hcl") +} diff --git a/test/fixtures/errors/run-all/app2/main.tf b/test/fixtures/errors/run-all/app2/main.tf new file mode 100644 index 0000000000..85991e287c --- /dev/null +++ b/test/fixtures/errors/run-all/app2/main.tf @@ -0,0 +1,12 @@ +resource "null_resource" "script_runner" { + provisioner "local-exec" { + command = "./script.sh 3" + + interpreter = ["/bin/sh", "-c"] + on_failure = fail + } + + triggers = { + always_run = timestamp() + } +} diff --git a/test/fixtures/errors/run-all/app2/script.sh b/test/fixtures/errors/run-all/app2/script.sh new file mode 100755 index 0000000000..aeac91a219 --- /dev/null +++ b/test/fixtures/errors/run-all/app2/script.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# script that will fail before $1 attempts + +RETRY_ATTEMPTS="$1" +COUNTER_FILE="attempt_counter.txt" + +if [[ ! -f "$COUNTER_FILE" ]]; then + echo "0" > "$COUNTER_FILE" +fi + +CURRENT_COUNT=$(($(cat "$COUNTER_FILE") + 1)) + +echo "$CURRENT_COUNT" > "$COUNTER_FILE" + +echo "Current attempt: $CURRENT_COUNT" + +if [ "$CURRENT_COUNT" -eq "$RETRY_ATTEMPTS" ]; then + echo "Success !" + echo "0" > "$COUNTER_FILE" + exit 0 +else + echo "Script error: Attempt $CURRENT_COUNT failed. Will succeed on attempt $RETRY_ATTEMPTS." >&2 + exit 1 +fi \ No newline at end of file diff --git a/test/fixtures/errors/run-all/app2/terragrunt.hcl b/test/fixtures/errors/run-all/app2/terragrunt.hcl new file mode 100644 index 0000000000..3ba7f1118b --- /dev/null +++ b/test/fixtures/errors/run-all/app2/terragrunt.hcl @@ -0,0 +1,3 @@ +include "common" { + path = find_in_parent_folders("common.hcl") +} diff --git a/test/fixtures/errors/run-all/common.hcl b/test/fixtures/errors/run-all/common.hcl new file mode 100644 index 0000000000..70c6fb3ca6 --- /dev/null +++ b/test/fixtures/errors/run-all/common.hcl @@ -0,0 +1,32 @@ +feature "unstable" { + default = true +} + +errors { + ignore "example1" { + ignorable_errors = feature.unstable.value ? [ + ".*example1.*", + ] : [] + message = "Ignoring error example1" + } + + ignore "example2" { + ignorable_errors = feature.unstable.value ? [ + ".*example2.*", + ] : [] + message = "Ignoring error example2" + } + + retry "script_errors" { + retryable_errors = feature.unstable.value ? [".*Script error.*"] : [] + max_attempts = 3 + sleep_interval_sec = 2 + } + + retry "aws_errors" { + retryable_errors = feature.unstable.value ? [".*AWS error.*"] : [] + max_attempts = 3 + sleep_interval_sec = 2 + } + +} \ No newline at end of file diff --git a/test/integration_errors_test.go b/test/integration_errors_test.go new file mode 100644 index 0000000000..73ee0bcb13 --- /dev/null +++ b/test/integration_errors_test.go @@ -0,0 +1,147 @@ +package test_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/gruntwork-io/terragrunt/test/helpers" + "github.com/gruntwork-io/terragrunt/util" + "github.com/stretchr/testify/require" +) + +const ( + testSimpleErrors = "fixtures/errors/default" + testIgnoreErrors = "fixtures/errors/ignore" + testIgnoreSignalErrors = "fixtures/errors/ignore-signal" + testRunAllIgnoreErrors = "fixtures/errors/run-all-ignore" + testRetryErrors = "fixtures/errors/retry" + testRetryFailErrors = "fixtures/errors/retry-fail" + testRunAllErrors = "fixtures/errors/run-all" +) + +func TestErrorsHandling(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testSimpleErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testSimpleErrors) + rootPath := util.JoinPath(tmpEnvPath, testSimpleErrors) + + _, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.NoError(t, err) +} + +func TestIgnoreError(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testIgnoreErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testIgnoreErrors) + rootPath := util.JoinPath(tmpEnvPath, testIgnoreErrors) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.NoError(t, err) + assert.Contains(t, stderr, "Ignoring error example1") + assert.NotContains(t, stderr, "Ignoring error example2") +} + +func TestRunAllIgnoreError(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testRunAllIgnoreErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testRunAllIgnoreErrors) + rootPath := util.JoinPath(tmpEnvPath, testRunAllIgnoreErrors) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.NoError(t, err) + assert.Contains(t, stderr, "Ignoring error example1") + assert.NotContains(t, stderr, "Ignoring error example2") + assert.Contains(t, stderr, "value-from-app-2") +} + +func TestRetryError(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testRetryErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testRetryErrors) + rootPath := util.JoinPath(tmpEnvPath, testRetryErrors) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.NoError(t, err) + assert.Contains(t, stderr, "Encountered retryable error: script_errors") + assert.NotContains(t, stderr, "aws_errors") +} + +func TestRetryFailError(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testRetryFailErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testRetryFailErrors) + rootPath := util.JoinPath(tmpEnvPath, testRetryFailErrors) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.Error(t, err) + assert.Contains(t, stderr, "Encountered retryable error: script_errors") +} + +func TestIgnoreSignal(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testIgnoreSignalErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testIgnoreSignalErrors) + rootPath := util.JoinPath(tmpEnvPath, testIgnoreSignalErrors) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.NoError(t, err) + assert.Contains(t, stderr, "Ignoring error example1") + assert.NotContains(t, stderr, "Ignoring error example2") + + signalsFile := filepath.Join(rootPath, "error-signals.json") + assert.FileExists(t, signalsFile) + + content, err := os.ReadFile(signalsFile) + require.NoError(t, err, "Failed to read error-signals.json") + + var signals struct { + Message string `json:"message"` + } + + err = json.Unmarshal(content, &signals) + require.NoError(t, err, "Failed to parse error-signals.json") + assert.Equal(t, "Failed example1", signals.Message, "Unexpected error message") +} + +func TestRunAllError(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testRunAllErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testRunAllErrors) + rootPath := util.JoinPath(tmpEnvPath, testRunAllErrors) + + _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.NoError(t, err) + assert.Contains(t, stderr, "Ignoring error example1") + assert.NotContains(t, stderr, "Ignoring error example2") + assert.Contains(t, stderr, "Encountered retryable error: script_errors") +} + +func TestRunAllFail(t *testing.T) { + t.Parallel() + + cleanupTerraformFolder(t, testRunAllErrors) + tmpEnvPath := helpers.CopyEnvironment(t, testRunAllErrors) + rootPath := util.JoinPath(tmpEnvPath, testRunAllErrors) + + _, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all apply -auto-approve --feature unstable=false --terragrunt-non-interactive --terragrunt-working-dir "+rootPath) + + require.Error(t, err) +} diff --git a/util/collections.go b/util/collections.go index 9fea5425a1..4d6868c3b4 100644 --- a/util/collections.go +++ b/util/collections.go @@ -228,3 +228,19 @@ func SplitUrls(s, sep string) []string { func SplitComma(s, sep string) []string { return strings.Split(s, sep) } + +// MergeStringSlices combines two string slices removing duplicates +func MergeStringSlices(a, b []string) []string { + seen := make(map[string]struct{}) + result := make([]string, 0, len(a)+len(b)) + + for _, s := range append(a, b...) { + if _, exists := seen[s]; !exists { + seen[s] = struct{}{} + + result = append(result, s) + } + } + + return result +}