Merge branch 'main' into stdout-and-stderr

.circleci/config.yml

@@ -7,7 +7,7 @@ env: &env
     TERRAFORM_VERSION: 1.5.7
     TOFU_VERSION: 1.8.0
     PACKER_VERSION: 1.10.0
-    TERRAGRUNT_VERSION: v0.52.0
+    TERRAGRUNT_VERSION: v0.69.8
     OPA_VERSION: v0.33.1
     GO_VERSION: 1.21.1
     GO111MODULE: auto

.github/pull_request_template.md

@@ -14,6 +14,7 @@ Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Grunt
 - [ ] Run the relevant tests successfully, including pre-commit checks.
 - [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable.
 - [ ] Include release notes. If this PR is backward incompatible, include a migration guide.
+- [ ] Make a plan for release of the functionality in this PR. If it delivers value to an end user, you are responsible for ensuring it is released promptly, and correctly. If you are not a maintainer, you are responsible for finding a maintainer to do this for you.
 
 ## Release Notes (draft)
 

examples/azure/terraform-azure-aks-example/main.tf

@@ -58,7 +58,7 @@ resource "azurerm_kubernetes_cluster" "k8s" {
     client_id     = var.client_id
     client_secret = var.client_secret
   }
-  automatic_channel_upgrade = "stable"
+  automatic_upgrade_channel = "stable"
   tags = {
     Environment = "Development"
   }

modules/aws/iam.go

@@ -2,6 +2,8 @@ package aws
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -59,6 +61,57 @@ func GetIamCurrentUserArnE(t testing.TestingT) (string, error) {
 	return *resp.User.Arn, nil
 }
 
+// GetIamPolicyDocument gets the most recent policy (JSON) document for an IAM policy.
+func GetIamPolicyDocument(t testing.TestingT, region string, policyARN string) string {
+	out, err := GetIamPolicyDocumentE(t, region, policyARN)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+// GetIamPolicyDocumentE gets the most recent policy (JSON) document for an IAM policy.
+func GetIamPolicyDocumentE(t testing.TestingT, region string, policyARN string) (string, error) {
+	iamClient, err := NewIamClientE(t, region)
+	if err != nil {
+		return "", err
+	}
+
+	versions, err := iamClient.ListPolicyVersions(context.Background(), &iam.ListPolicyVersionsInput{
+		PolicyArn: &policyARN,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	var defaultVersion string
+	for _, version := range versions.Versions {
+		if version.IsDefaultVersion == true {
+			defaultVersion = *version.VersionId
+		}
+	}
+
+	document, err := iamClient.GetPolicyVersion(context.Background(), &iam.GetPolicyVersionInput{
+		PolicyArn: aws.String(policyARN),
+		VersionId: aws.String(defaultVersion),
+	})
+	if err != nil {
+		return "", err
+	}
+
+	unescapedDocument := document.PolicyVersion.Document
+	if unescapedDocument == nil {
+		return "", fmt.Errorf("no policy document found for policy %s", policyARN)
+	}
+
+	escapedDocument, err := url.QueryUnescape(*unescapedDocument)
+	if err != nil {
+		return "", err
+	}
+
+	return escapedDocument, nil
+}
+
 // CreateMfaDevice creates an MFA device using the given IAM client.
 func CreateMfaDevice(t testing.TestingT, iamClient *iam.Client, deviceName string) *types.VirtualMFADevice {
 	mfaDevice, err := CreateMfaDeviceE(t, iamClient, deviceName)

modules/aws/iam_test.go

@@ -1,9 +1,15 @@
 package aws
 
 import (
+	"context"
+	"strings"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/iam"
+	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetIamCurrentUserName(t *testing.T) {
@@ -19,3 +25,49 @@ func TestGetIamCurrentUserArn(t *testing.T) {
 	username := GetIamCurrentUserArn(t)
 	assert.Regexp(t, "^arn:aws:iam::[0-9]{12}:user/.+$", username)
 }
+
+func TestGetIAMPolicyDocument(t *testing.T) {
+	t.Parallel()
+
+	region := GetRandomRegion(t, nil, nil)
+
+	t.Run("Exists", func(t *testing.T) {
+		iamClient, err := NewIamClientE(t, region)
+		require.NoError(t, err)
+
+		policyDocument := `{
+			"Version": "2012-10-17",
+			"Statement": [
+				{
+					"Sid": "Stmt1530709892083",
+					"Action": "*",
+					"Effect": "Allow",
+					"Resource": "*"
+				}
+			]
+		}`
+		input := &iam.CreatePolicyInput{
+			PolicyName:     aws.String(strings.ToLower(random.UniqueId())),
+			PolicyDocument: aws.String(policyDocument),
+		}
+		policy, err := iamClient.CreatePolicy(context.Background(), input)
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			t.Log("Deleting IAM Policy Document")
+			_, err := iamClient.DeletePolicy(context.Background(), &iam.DeletePolicyInput{
+				PolicyArn: policy.Policy.Arn,
+			})
+			require.NoError(t, err)
+		})
+
+		p := GetIamPolicyDocument(t, region, *policy.Policy.Arn)
+		t.Log("Retrieved Policy Document:", p)
+		assert.JSONEq(t, policyDocument, p)
+	})
+
+	t.Run("DoesNotExist", func(t *testing.T) {
+		_, err := GetIamPolicyDocumentE(t, region, "arn:aws:iam::1234567890:policy/does-not-exist")
+		require.Error(t, err)
+	})
+}

modules/helm/template.go

@@ -110,6 +110,9 @@ func RenderRemoteTemplateE(t testing.TestingT, options *Options, chartURL string
 
 	// ... and add the helm chart name, the remote repo and chart URL at the end
 	args = append(args, releaseName, "--repo", chartURL)
+	if options.Version != "" {
+		args = append(args, "--version", options.Version)
+	}
 
 	// Finally, call out to helm template command
 	return RunHelmCommandAndGetStdOutE(t, options, "template", args...)

modules/helm/template_test.go

@@ -27,6 +27,7 @@ func TestRemoteChartRender(t *testing.T) {
 		remoteChartSource  = "https://charts.bitnami.com/bitnami"
 		remoteChartName    = "nginx"
 		remoteChartVersion = "13.2.24"
+		registry           = "registry-1.docker.io"
 	)
 
 	t.Parallel()
@@ -42,11 +43,12 @@ func TestRemoteChartRender(t *testing.T) {
 	options := &Options{
 		SetValues: map[string]string{
 			"image.repository": remoteChartName,
-			"image.registry":   "",
+			"image.registry":   registry,
 			"image.tag":        remoteChartVersion,
 		},
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 		Logger:         logger.Discard,
+		Version:        remoteChartVersion,
 	}
 
 	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
@@ -62,10 +64,10 @@ func TestRemoteChartRender(t *testing.T) {
 	require.Equal(t, namespaceName, deployment.Namespace)
 
 	// Finally, we verify the deployment pod template spec is set to the expected container image value
-	expectedContainerImage := remoteChartName + ":" + remoteChartVersion
+	expectedContainerImage := registry + "/" + remoteChartName + ":" + remoteChartVersion
 	deploymentContainers := deployment.Spec.Template.Spec.Containers
 	require.Equal(t, len(deploymentContainers), 1)
-	require.Equal(t, deploymentContainers[0].Image, expectedContainerImage)
+	require.Equal(t, expectedContainerImage, deploymentContainers[0].Image)
 }
 
 // Test that we can dump all the manifest locally a remote chart (e.g bitnami/nginx)
@@ -81,15 +83,15 @@ func TestRemoteChartRenderDiff(t *testing.T) {
 
 	initialSnapshot := t.TempDir()
 	updatedSnapshot := t.TempDir()
-	renderChartDump(t, "5.0.0", initialSnapshot)
-	output := renderChartDump(t, "5.1.0", updatedSnapshot)
+	renderChartDump(t, "13.2.20", initialSnapshot)
+	output := renderChartDump(t, "13.2.24", updatedSnapshot)
 
 	options := &Options{
 		Logger:       logger.Default,
 		SnapshotPath: initialSnapshot,
 	}
 	// diff in: spec.initContainers.preserve-logs-symlinks.imag, spec.containers.nginx.image, tls certificates
-	require.Equal(t, 5, DiffAgainstSnapshot(t, options, output, "nginx"))
+	require.Equal(t, 4, DiffAgainstSnapshot(t, options, output, "nginx"))
 }
 
 // render chart dump and return the rendered output
@@ -111,6 +113,7 @@ func renderChartDump(t *testing.T, remoteChartVersion, snapshotDir string) strin
 		},
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 		Logger:         logger.Discard,
+		Version:        remoteChartVersion,
 	}
 
 	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since

modules/k8s/tunnel.go

@@ -189,15 +189,18 @@ func (tunnel *Tunnel) ForwardPortE(t testing.TestingT) error {
 		tunnel.logger.Logf(t, "Error creating a new Kubernetes client: %s", err)
 		return err
 	}
-	kubeConfigPath, err := tunnel.kubectlOptions.GetConfigPath(t)
-	if err != nil {
-		tunnel.logger.Logf(t, "Error getting kube config path: %s", err)
-		return err
-	}
-	config, err := LoadApiClientConfigE(kubeConfigPath, tunnel.kubectlOptions.ContextName)
-	if err != nil {
-		tunnel.logger.Logf(t, "Error loading Kubernetes config: %s", err)
-		return err
+	config := tunnel.kubectlOptions.RestConfig
+	if config == nil {
+		kubeConfigPath, err := tunnel.kubectlOptions.GetConfigPath(t)
+		if err != nil {
+			tunnel.logger.Logf(t, "Error getting kube config path: %s", err)
+			return err
+		}
+		config, err = LoadApiClientConfigE(kubeConfigPath, tunnel.kubectlOptions.ContextName)
+		if err != nil {
+			tunnel.logger.Logf(t, "Error loading Kubernetes config: %s", err)
+			return err
+		}
 	}
 
 	// Find the pod to port forward to

modules/terraform/cmd.go

@@ -39,6 +39,9 @@ const (
 
 	// TerraformDefaultPath to run terraform
 	TerraformDefaultPath = "terraform"
+
+	// TerragruntDefaultPath to run terragrunt
+	TerragruntDefaultPath = "terragrunt"
 )
 
 var DefaultExecutable = defaultTerraformExecutable()
@@ -49,8 +52,22 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) {
 		options.TerraformBinary = DefaultExecutable
 	}
 
-	if options.TerraformBinary == "terragrunt" {
+	if options.TerraformBinary == TerragruntDefaultPath {
 		args = append(args, "--terragrunt-non-interactive")
+		// for newer Terragrunt version, setting simplified log formatting
+		if options.EnvVars == nil {
+			options.EnvVars = map[string]string{}
+		}
+		_, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"]
+		if !tgLogSet {
+			// key-value format for terragrunt logs to avoid colors and have plain form
+			// https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-log-format
+			options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value"
+		}
+		_, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"]
+		if !tgLogFormat {
+			options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] = "%msg(color=disable)"
+		}
 	}
 
 	if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) {

modules/terraform/output.go

@@ -5,12 +5,23 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/gruntwork-io/terratest/modules/testing"
 	"github.com/stretchr/testify/require"
 )
 
+const skipJsonLogLine = " msg="
+
+var (
+	// ansiLineRegex matches lines starting with ANSI escape codes for text formatting (e.g., colors, styles).
+	ansiLineRegex = regexp.MustCompile(`(?m)^\x1b\[[0-9;]*m.*`)
+	// tgLogLevel matches log lines containing fields for time, level, prefix, binary, and message, each with non-whitespace values.
+	tgLogLevel = regexp.MustCompile(`.*time=\S+ level=\S+ prefix=\S+ binary=\S+ msg=.*`)
+)
+
 // Output calls terraform output for the given variable and return its string value representation.
 // It only designed to work with primitive terraform types: string, number and bool.
 // Please use OutputStruct for anything else.
@@ -279,7 +290,11 @@ func OutputJsonE(t testing.TestingT, options *Options, key string) (string, erro
 		args = append(args, key)
 	}
 
-	return RunTerraformCommandAndGetStdoutE(t, options, args...)
+	rawJson, err := RunTerraformCommandAndGetStdoutE(t, options, args...)
+	if err != nil {
+		return rawJson, err
+	}
+	return cleanJson(rawJson)
 }
 
 // OutputStruct calls terraform output for the given variable and stores the
@@ -348,3 +363,33 @@ func OutputAll(t testing.TestingT, options *Options) map[string]interface{} {
 func OutputAllE(t testing.TestingT, options *Options) (map[string]interface{}, error) {
 	return OutputForKeysE(t, options, nil)
 }
+
+// clean the ANSI characters from the JSON and update formating
+func cleanJson(input string) (string, error) {
+	// Remove ANSI escape codes
+	cleaned := ansiLineRegex.ReplaceAllString(input, "")
+	cleaned = tgLogLevel.ReplaceAllString(cleaned, "")
+
+	lines := strings.Split(cleaned, "\n")
+	var result []string
+	for _, line := range lines {
+		trimmed := strings.TrimSpace(line)
+		if trimmed != "" && !strings.Contains(trimmed, skipJsonLogLine) {
+			result = append(result, trimmed)
+		}
+	}
+	ansiClean := strings.Join(result, "\n")
+
+	var jsonObj interface{}
+	if err := json.Unmarshal([]byte(ansiClean), &jsonObj); err != nil {
+		return "", err
+	}
+
+	// Format JSON output with indentation
+	normalized, err := json.MarshalIndent(jsonObj, "", "  ")
+	if err != nil {
+		return "", err
+	}
+
+	return string(normalized), nil
+}