Merge pull request #1462 from gruntwork-io/bug/tg-output-handling-1453

JSON output fetching fixes

.circleci/config.yml

@@ -7,7 +7,7 @@ env: &env
     TERRAFORM_VERSION: 1.5.7
     TOFU_VERSION: 1.8.0
     PACKER_VERSION: 1.10.0
-    TERRAGRUNT_VERSION: v0.52.0
+    TERRAGRUNT_VERSION: v0.69.8
     OPA_VERSION: v0.33.1
     GO_VERSION: 1.21.1
     GO111MODULE: auto

modules/terraform/cmd.go

@@ -39,6 +39,9 @@ const (
 
 	// TerraformDefaultPath to run terraform
 	TerraformDefaultPath = "terraform"
+
+	// TerragruntDefaultPath to run terragrunt
+	TerragruntDefaultPath = "terragrunt"
 )
 
 var DefaultExecutable = defaultTerraformExecutable()
@@ -49,8 +52,22 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) {
 		options.TerraformBinary = DefaultExecutable
 	}
 
-	if options.TerraformBinary == "terragrunt" {
+	if options.TerraformBinary == TerragruntDefaultPath {
 		args = append(args, "--terragrunt-non-interactive")
+		// for newer Terragrunt version, setting simplified log formatting
+		if options.EnvVars == nil {
+			options.EnvVars = map[string]string{}
+		}
+		_, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"]
+		if !tgLogSet {
+			// key-value format for terragrunt logs to avoid colors and have plain form
+			// https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-log-format
+			options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value"
+		}
+		_, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"]
+		if !tgLogFormat {
+			options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] = "%msg(color=disable)"
+		}
 	}
 
 	if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) {

modules/terraform/output.go

@@ -5,12 +5,23 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/gruntwork-io/terratest/modules/testing"
 	"github.com/stretchr/testify/require"
 )
 
+const skipJsonLogLine = " msg="
+
+var (
+	// ansiLineRegex matches lines starting with ANSI escape codes for text formatting (e.g., colors, styles).
+	ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`)
+	// tgLogLevel matches log lines containing fields for time, level, prefix, binary, and message, each with non-whitespace values.
+	tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`)
+)
+
 // Output calls terraform output for the given variable and return its string value representation.
 // It only designed to work with primitive terraform types: string, number and bool.
 // Please use OutputStruct for anything else.
@@ -279,7 +290,11 @@ func OutputJsonE(t testing.TestingT, options *Options, key string) (string, erro
 		args = append(args, key)
 	}
 
-	return RunTerraformCommandAndGetStdoutE(t, options, args...)
+	rawJson, err := RunTerraformCommandAndGetStdoutE(t, options, args...)
+	if err != nil {
+		return rawJson, err
+	}
+	return cleanJson(rawJson)
 }
 
 // OutputStruct calls terraform output for the given variable and stores the
@@ -348,3 +363,33 @@ func OutputAll(t testing.TestingT, options *Options) map[string]interface{} {
 func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, error) {
 	return OutputForKeysE(t, options, nil)
 }
+
+// clean the ANSI characters from the JSON and update formating
+func cleanJson(input string) (string, error) {
+	// Remove ANSI escape codes
+	cleaned := ansiLineRegex.ReplaceAllString(input, "")
+	cleaned = tgLogLevel.ReplaceAllString(cleaned, "")
+
+	lines := strings.Split(cleaned, "\n")
+	var result []string
+	for _, line := range lines {
+		trimmed := strings.TrimSpace(line)
+		if trimmed != "" && !strings.Contains(trimmed, skipJsonLogLine) {
+			result = append(result, trimmed)
+		}
+	}
+	ansiClean := strings.Join(result, "\n")
+
+	var jsonObj interface{}
+	if err := json.Unmarshal([]byte(ansiClean), &jsonObj); err != nil {
+		return "", err
+	}
+
+	// Format JSON output with indentation
+	normalized, err := json.MarshalIndent(jsonObj, "", "  ")
+	if err != nil {
+		return "", err
+	}
+
+	return string(normalized), nil
+}

modules/terraform/output_test.go

@@ -2,8 +2,11 @@ package terraform
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/gruntwork-io/terratest/modules/files"
 	"github.com/stretchr/testify/require"
 )
@@ -31,6 +34,40 @@ func TestOutputString(t *testing.T) {
 
 	num1 := Output(t, options, "number1")
 	require.Equal(t, num1, "3", "Number %q should match %q", "3", num1)
+
+	unicodeString := Output(t, options, "unicode_string")
+	require.Equal(t, "söme chäräcter", unicodeString)
+}
+
+func TestTgOutputString(t *testing.T) {
+	t.Parallel()
+
+	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-output", t.Name())
+	require.NoError(t, err)
+
+	WriteFile(t, filepath.Join(testFolder, "terragrunt.hcl"), []byte{})
+
+	options := &Options{
+		TerraformDir:    testFolder,
+		TerraformBinary: "terragrunt",
+	}
+
+	InitAndApply(t, options)
+
+	b := Output(t, options, "bool")
+	require.Equal(t, b, "true", "Bool %q should match %q", "true", b)
+
+	str := Output(t, options, "string")
+	require.Equal(t, str, "This is a string.", "String %q should match %q", "This is a string.", str)
+
+	num := Output(t, options, "number")
+	require.Equal(t, num, "3.14", "Number %q should match %q", "3.14", num)
+
+	num1 := Output(t, options, "number1")
+	require.Equal(t, num1, "3", "Number %q should match %q", "3", num1)
+
+	unicodeString := Output(t, options, "unicode_string")
+	require.Equal(t, "söme chäräcter", unicodeString)
 }
 
 func TestOutputList(t *testing.T) {
@@ -310,6 +347,11 @@ func TestOutputJson(t *testing.T) {
     "sensitive": false,
     "type": "string",
     "value": "This is a string."
+  },
+  "unicode_string": {
+    "sensitive": false,
+    "type": "string",
+    "value": "söme chäräcter"
   }
 }`
 
@@ -433,3 +475,27 @@ func TestOutputsForKeysError(t *testing.T) {
 
 	require.Error(t, err)
 }
+
+func TestTgOutputJsonParsing(t *testing.T) {
+	t.Parallel()
+
+	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-output-map", t.Name())
+	require.NoError(t, err)
+
+	WriteFile(t, filepath.Join(testFolder, "terragrunt.hcl"), []byte{})
+
+	options := &Options{
+		TerraformDir:    testFolder,
+		TerraformBinary: "terragrunt",
+	}
+
+	InitAndApply(t, options)
+
+	output, err := OutputAllE(t, options)
+
+	require.NoError(t, err)
+	assert.NotNil(t, output)
+	assert.NotEmpty(t, output)
+	assert.Contains(t, output, "mogwai")
+	assert.Equal(t, "söme chäräcter", output["not_a_map_unicode"])
+}

test/fixtures/terraform-output-map/output.tf

@@ -11,3 +11,6 @@ output "not_a_map" {
   value = "This is not a map."
 }
 
+output "not_a_map_unicode" {
+  value = "söme chäräcter"
+}

test/fixtures/terraform-output/output.tf

@@ -13,3 +13,7 @@ output "number" {
 output "number1" {
   value = 3
 }
+
+output "unicode_string" {
+  value = "söme chäräcter"
+}