From 9bb215357bc8ec818f163e2dac420bf093a42cb1 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 21:22:28 +0000 Subject: [PATCH 01/12] Terragrunt log level handling --- .circleci/config.yml | 2 +- modules/terraform/cmd.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bf50dac5..210beefb4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ env: &env TERRAFORM_VERSION: 1.5.7 TOFU_VERSION: 1.8.0 PACKER_VERSION: 1.10.0 - TERRAGRUNT_VERSION: v0.52.0 + TERRAGRUNT_VERSION: v0.68.7 OPA_VERSION: v0.33.1 GO_VERSION: 1.21.1 GO111MODULE: auto diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 48e1324f7..ec1e23359 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,6 +51,7 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") + args = append(args, "--terragrunt-log-disable") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { From baad51e52598d905ef12e8b9c5d43c4be2406145 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 22:03:09 +0000 Subject: [PATCH 02/12] Added tests for fetching tg output --- modules/terraform/cmd.go | 2 +- modules/terraform/output.go | 32 +++++++++++++++++++++++++++++++- modules/terraform/output_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index ec1e23359..4362c35c3 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,7 +51,7 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") - args = append(args, "--terragrunt-log-disable") + args = append(args, "--terragrunt-forward-tf-stdout") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { diff --git a/modules/terraform/output.go b/modules/terraform/output.go index 6e294e7a2..e60c9fdd1 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "reflect" + "regexp" "strconv" + "strings" "github.com/gruntwork-io/terratest/modules/testing" "github.com/stretchr/testify/require" @@ -279,7 +281,11 @@ func OutputJsonE(t testing.TestingT, options *Options, key string) (string, erro args = append(args, key) } - return RunTerraformCommandAndGetStdoutE(t, options, args...) + rawJson, err := RunTerraformCommandAndGetStdoutE(t, options, args...) + if err != nil { + return rawJson, err + } + return cleanJson(rawJson) } // OutputStruct calls terraform output for the given variable and stores the @@ -348,3 +354,27 @@ func OutputAll(t testing.TestingT, options *Options) map[string]interface{} { func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, error) { return OutputForKeysE(t, options, nil) } + +// clean the ANSI characters from the JSON and update formating +func cleanJson(input string) (string, error) { + ansiLineRegex := regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + cleaned := ansiLineRegex.ReplaceAllString(input, "") + lines := strings.Split(cleaned, "\n") + var result []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + result = append(result, trimmed) + } + } + ansiClean := strings.Join(result, "\n") + var jsonObj interface{} + if err := json.Unmarshal([]byte(ansiClean), &jsonObj); err != nil { + return "", err + } + normalized, err := json.MarshalIndent(jsonObj, "", " ") + if err != nil { + return "", err + } + return string(normalized), nil +} diff --git a/modules/terraform/output_test.go b/modules/terraform/output_test.go index f3285d69f..dc26d4814 100644 --- a/modules/terraform/output_test.go +++ b/modules/terraform/output_test.go @@ -2,8 +2,11 @@ package terraform import ( "fmt" + "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/gruntwork-io/terratest/modules/files" "github.com/stretchr/testify/require" ) @@ -433,3 +436,26 @@ func TestOutputsForKeysError(t *testing.T) { require.Error(t, err) } + +func TestTgOutputJsonParsing(t *testing.T) { + t.Parallel() + + testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-output-map", t.Name()) + require.NoError(t, err) + + WriteFile(t, filepath.Join(testFolder, "terragrunt.hcl"), []byte{}) + + options := &Options{ + TerraformDir: testFolder, + TerraformBinary: "terragrunt", + } + + InitAndApply(t, options) + + output, err := OutputAllE(t, options) + + require.NoError(t, err) + assert.NotNil(t, output) + assert.NotEmpty(t, output) + assert.Contains(t, output, "mogwai") +} From ddc10c31d99cc30a7df2b7554688dc1d41bef618 Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 22:07:15 +0000 Subject: [PATCH 03/12] Added test for unicode characters --- modules/terraform/output_test.go | 1 + test/fixtures/terraform-output-map/output.tf | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/terraform/output_test.go b/modules/terraform/output_test.go index dc26d4814..b23f30f0d 100644 --- a/modules/terraform/output_test.go +++ b/modules/terraform/output_test.go @@ -458,4 +458,5 @@ func TestTgOutputJsonParsing(t *testing.T) { assert.NotNil(t, output) assert.NotEmpty(t, output) assert.Contains(t, output, "mogwai") + assert.Equal(t, "söme chäräcter", output["not_a_map_unicode"]) } diff --git a/test/fixtures/terraform-output-map/output.tf b/test/fixtures/terraform-output-map/output.tf index a722e8f6b..6a7aee48c 100644 --- a/test/fixtures/terraform-output-map/output.tf +++ b/test/fixtures/terraform-output-map/output.tf @@ -11,3 +11,6 @@ output "not_a_map" { value = "This is not a map." } +output "not_a_map_unicode" { + value = "söme chäräcter" +} From 57fc39ba126128d247de203f701048388c13ccef Mon Sep 17 00:00:00 2001 From: Denis O Date: Tue, 5 Nov 2024 22:16:04 +0000 Subject: [PATCH 04/12] Terragrunt output tests --- modules/terraform/output_test.go | 39 ++++++++++++++++++++++++ test/fixtures/terraform-output/output.tf | 4 +++ 2 files changed, 43 insertions(+) diff --git a/modules/terraform/output_test.go b/modules/terraform/output_test.go index b23f30f0d..f2d0d927d 100644 --- a/modules/terraform/output_test.go +++ b/modules/terraform/output_test.go @@ -34,6 +34,40 @@ func TestOutputString(t *testing.T) { num1 := Output(t, options, "number1") require.Equal(t, num1, "3", "Number %q should match %q", "3", num1) + + unicodeString := Output(t, options, "unicode_string") + require.Equal(t, "söme chäräcter", unicodeString) +} + +func TestTgOutputString(t *testing.T) { + t.Parallel() + + testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-output", t.Name()) + require.NoError(t, err) + + WriteFile(t, filepath.Join(testFolder, "terragrunt.hcl"), []byte{}) + + options := &Options{ + TerraformDir: testFolder, + TerraformBinary: "terragrunt", + } + + InitAndApply(t, options) + + b := Output(t, options, "bool") + require.Equal(t, b, "true", "Bool %q should match %q", "true", b) + + str := Output(t, options, "string") + require.Equal(t, str, "This is a string.", "String %q should match %q", "This is a string.", str) + + num := Output(t, options, "number") + require.Equal(t, num, "3.14", "Number %q should match %q", "3.14", num) + + num1 := Output(t, options, "number1") + require.Equal(t, num1, "3", "Number %q should match %q", "3", num1) + + unicodeString := Output(t, options, "unicode_string") + require.Equal(t, "söme chäräcter", unicodeString) } func TestOutputList(t *testing.T) { @@ -313,6 +347,11 @@ func TestOutputJson(t *testing.T) { "sensitive": false, "type": "string", "value": "This is a string." + }, + "unicode_string": { + "sensitive": false, + "type": "string", + "value": "söme chäräcter" } }` diff --git a/test/fixtures/terraform-output/output.tf b/test/fixtures/terraform-output/output.tf index e9d5ae693..5b92808e3 100644 --- a/test/fixtures/terraform-output/output.tf +++ b/test/fixtures/terraform-output/output.tf @@ -13,3 +13,7 @@ output "number" { output "number1" { value = 3 } + +output "unicode_string" { + value = "söme chäräcter" +} From 071224f720bd5d3c3c418a4c173eb03ae98e31b8 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 6 Nov 2024 14:13:42 +0000 Subject: [PATCH 05/12] Arguments cleanup --- modules/terraform/cmd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 4362c35c3..48e1324f7 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,7 +51,6 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") - args = append(args, "--terragrunt-forward-tf-stdout") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { From 515b60c777ff79f1049e57a141434696479686f5 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 6 Nov 2024 19:14:25 +0000 Subject: [PATCH 06/12] Default log level update --- modules/terraform/cmd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 48e1324f7..17eb92aa7 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,6 +51,7 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") + args = append(args, "--terragrunt-log-level", "warn") } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { From 4ea7ce871f53cfe512494888ea1c5a4be4dc21b7 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 7 Nov 2024 15:37:59 +0000 Subject: [PATCH 07/12] Formatting update --- modules/terraform/cmd.go | 5 ++++- modules/terraform/output.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 17eb92aa7..d9052da9e 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -51,7 +51,10 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.TerraformBinary == "terragrunt" { args = append(args, "--terragrunt-non-interactive") - args = append(args, "--terragrunt-log-level", "warn") + _, formattingIset := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] + if !formattingIset { + options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" + } } if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) { diff --git a/modules/terraform/output.go b/modules/terraform/output.go index e60c9fdd1..1fc93141a 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -13,6 +13,10 @@ import ( "github.com/stretchr/testify/require" ) +const skipJsonLogLine = "log=info msg=" + +var ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + // Output calls terraform output for the given variable and return its string value representation. // It only designed to work with primitive terraform types: string, number and bool. // Please use OutputStruct for anything else. @@ -357,24 +361,29 @@ func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, e // clean the ANSI characters from the JSON and update formating func cleanJson(input string) (string, error) { - ansiLineRegex := regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + // Remove ANSI escape codes cleaned := ansiLineRegex.ReplaceAllString(input, "") + lines := strings.Split(cleaned, "\n") var result []string for _, line := range lines { trimmed := strings.TrimSpace(line) - if trimmed != "" { + if trimmed != "" && !strings.Contains(trimmed, skipJsonLogLine) { result = append(result, trimmed) } } ansiClean := strings.Join(result, "\n") + var jsonObj interface{} if err := json.Unmarshal([]byte(ansiClean), &jsonObj); err != nil { return "", err } + + // Format JSON output with indentation normalized, err := json.MarshalIndent(jsonObj, "", " ") if err != nil { return "", err } + return string(normalized), nil } From d6ea8dbf818823ffc7d5e08d2b2784cf9d506e84 Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 7 Nov 2024 17:26:42 +0000 Subject: [PATCH 08/12] updated default path --- modules/terraform/cmd.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index d9052da9e..03a3849a7 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -39,6 +39,9 @@ const ( // TerraformDefaultPath to run terraform TerraformDefaultPath = "terraform" + + // TerragruntDefaultPath to run terragrunt + TerragruntDefaultPath = "terragrunt" ) var DefaultExecutable = defaultTerraformExecutable() @@ -49,8 +52,12 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { options.TerraformBinary = DefaultExecutable } - if options.TerraformBinary == "terragrunt" { + if options.TerraformBinary == TerragruntDefaultPath { args = append(args, "--terragrunt-non-interactive") + // for newer Terragrunt version, setting simplified log formatting + if options.EnvVars == nil { + options.EnvVars = map[string]string{} + } _, formattingIset := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] if !formattingIset { options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" From fd220404e0bad05f1fe30319f9209b7ceea7c7cd Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 7 Nov 2024 20:12:18 +0000 Subject: [PATCH 09/12] Fixed logs cleaning --- modules/terraform/cmd.go | 4 ++-- modules/terraform/output.go | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index 03a3849a7..f5f082bbf 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -58,8 +58,8 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.EnvVars == nil { options.EnvVars = map[string]string{} } - _, formattingIset := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] - if !formattingIset { + _, tgLogSet := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] + if !tgLogSet { options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" } } diff --git a/modules/terraform/output.go b/modules/terraform/output.go index 1fc93141a..93a2ed299 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -13,9 +13,12 @@ import ( "github.com/stretchr/testify/require" ) -const skipJsonLogLine = "log=info msg=" +const skipJsonLogLine = " msg=" -var ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) +var ( + ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) + tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`) +) // Output calls terraform output for the given variable and return its string value representation. // It only designed to work with primitive terraform types: string, number and bool. @@ -363,6 +366,7 @@ func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, e func cleanJson(input string) (string, error) { // Remove ANSI escape codes cleaned := ansiLineRegex.ReplaceAllString(input, "") + cleaned = tgLogLevel.ReplaceAllString(cleaned, "") lines := strings.Split(cleaned, "\n") var result []string From a294bb8ddcf51a9d41044d3c252b0f397939cc7a Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Dec 2024 17:17:35 +0000 Subject: [PATCH 10/12] Custom log format --- modules/terraform/cmd.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index f5f082bbf..c6c5d2380 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -58,9 +58,13 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { if options.EnvVars == nil { options.EnvVars = map[string]string{} } - _, tgLogSet := options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] + _, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"] if !tgLogSet { - options.EnvVars["TERRAGRUNT_DISABLE_LOG_FORMATTING"] = "true" + options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value" + } + _, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] + if !tgLogFormat { + options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] = "%msg(color=disable)" } } From 44a56162e9d5b2918513b046a299b23735e5acde Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 5 Dec 2024 17:26:23 +0000 Subject: [PATCH 11/12] version update --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a499b0a8..0a0ac0cdc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ env: &env TERRAFORM_VERSION: 1.5.7 TOFU_VERSION: 1.8.0 PACKER_VERSION: 1.10.0 - TERRAGRUNT_VERSION: v0.68.7 + TERRAGRUNT_VERSION: v0.69.8 OPA_VERSION: v0.33.1 GO_VERSION: 1.21.1 GO111MODULE: auto From 6b1772c0723005fbf5e89b68d5aa3e3576614a29 Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 9 Dec 2024 17:57:04 +0000 Subject: [PATCH 12/12] PR comments --- modules/terraform/cmd.go | 2 ++ modules/terraform/output.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/terraform/cmd.go b/modules/terraform/cmd.go index c6c5d2380..ea4baf511 100644 --- a/modules/terraform/cmd.go +++ b/modules/terraform/cmd.go @@ -60,6 +60,8 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) { } _, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"] if !tgLogSet { + // key-value format for terragrunt logs to avoid colors and have plain form + // https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-log-format options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value" } _, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] diff --git a/modules/terraform/output.go b/modules/terraform/output.go index 93a2ed299..b4b67ec7b 100644 --- a/modules/terraform/output.go +++ b/modules/terraform/output.go @@ -16,8 +16,10 @@ import ( const skipJsonLogLine = " msg=" var ( + // ansiLineRegex matches lines starting with ANSI escape codes for text formatting (e.g., colors, styles). ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`) - tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`) + // tgLogLevel matches log lines containing fields for time, level, prefix, binary, and message, each with non-whitespace values. + tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`) ) // Output calls terraform output for the given variable and return its string value representation.