Merge branch 'main' into import-project-ssh-key

.circleci/config.yml

@@ -7,7 +7,7 @@ env: &env
     TERRAFORM_VERSION: 1.5.7
     TOFU_VERSION: 1.8.0
     PACKER_VERSION: 1.10.0
-    TERRAGRUNT_VERSION: v0.52.0
+    TERRAGRUNT_VERSION: v0.69.8
     OPA_VERSION: v0.33.1
     GO_VERSION: 1.21.1
     GO111MODULE: auto

.github/pull_request_template.md

@@ -14,6 +14,7 @@ Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Grunt
 - [ ] Run the relevant tests successfully, including pre-commit checks.
 - [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable.
 - [ ] Include release notes. If this PR is backward incompatible, include a migration guide.
+- [ ] Make a plan for release of the functionality in this PR. If it delivers value to an end user, you are responsible for ensuring it is released promptly, and correctly. If you are not a maintainer, you are responsible for finding a maintainer to do this for you.
 
 ## Release Notes (draft)
 

examples/azure/terraform-azure-aks-example/main.tf

@@ -58,7 +58,7 @@ resource "azurerm_kubernetes_cluster" "k8s" {
     client_id     = var.client_id
     client_secret = var.client_secret
   }
-  automatic_channel_upgrade = "stable"
+  automatic_upgrade_channel = "stable"
   tags = {
     Environment = "Development"
   }

modules/aws/iam.go

@@ -2,6 +2,8 @@ package aws
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -59,6 +61,57 @@ func GetIamCurrentUserArnE(t testing.TestingT) (string, error) {
 	return *resp.User.Arn, nil
 }
 
+// GetIamPolicyDocument gets the most recent policy (JSON) document for an IAM policy.
+func GetIamPolicyDocument(t testing.TestingT, region string, policyARN string) string {
+	out, err := GetIamPolicyDocumentE(t, region, policyARN)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+// GetIamPolicyDocumentE gets the most recent policy (JSON) document for an IAM policy.
+func GetIamPolicyDocumentE(t testing.TestingT, region string, policyARN string) (string, error) {
+	iamClient, err := NewIamClientE(t, region)
+	if err != nil {
+		return "", err
+	}
+
+	versions, err := iamClient.ListPolicyVersions(context.Background(), &iam.ListPolicyVersionsInput{
+		PolicyArn: &policyARN,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	var defaultVersion string
+	for _, version := range versions.Versions {
+		if version.IsDefaultVersion == true {
+			defaultVersion = *version.VersionId
+		}
+	}
+
+	document, err := iamClient.GetPolicyVersion(context.Background(), &iam.GetPolicyVersionInput{
+		PolicyArn: aws.String(policyARN),
+		VersionId: aws.String(defaultVersion),
+	})
+	if err != nil {
+		return "", err
+	}
+
+	unescapedDocument := document.PolicyVersion.Document
+	if unescapedDocument == nil {
+		return "", fmt.Errorf("no policy document found for policy %s", policyARN)
+	}
+
+	escapedDocument, err := url.QueryUnescape(*unescapedDocument)
+	if err != nil {
+		return "", err
+	}
+
+	return escapedDocument, nil
+}
+
 // CreateMfaDevice creates an MFA device using the given IAM client.
 func CreateMfaDevice(t testing.TestingT, iamClient *iam.Client, deviceName string) *types.VirtualMFADevice {
 	mfaDevice, err := CreateMfaDeviceE(t, iamClient, deviceName)

modules/aws/iam_test.go

@@ -1,9 +1,15 @@
 package aws
 
 import (
+	"context"
+	"strings"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/iam"
+	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetIamCurrentUserName(t *testing.T) {
@@ -19,3 +25,49 @@ func TestGetIamCurrentUserArn(t *testing.T) {
 	username := GetIamCurrentUserArn(t)
 	assert.Regexp(t, "^arn:aws:iam::[0-9]{12}:user/.+$", username)
 }
+
+func TestGetIAMPolicyDocument(t *testing.T) {
+	t.Parallel()
+
+	region := GetRandomRegion(t, nil, nil)
+
+	t.Run("Exists", func(t *testing.T) {
+		iamClient, err := NewIamClientE(t, region)
+		require.NoError(t, err)
+
+		policyDocument := `{
+			"Version": "2012-10-17",
+			"Statement": [
+				{
+					"Sid": "Stmt1530709892083",
+					"Action": "*",
+					"Effect": "Allow",
+					"Resource": "*"
+				}
+			]
+		}`
+		input := &iam.CreatePolicyInput{
+			PolicyName:     aws.String(strings.ToLower(random.UniqueId())),
+			PolicyDocument: aws.String(policyDocument),
+		}
+		policy, err := iamClient.CreatePolicy(context.Background(), input)
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			t.Log("Deleting IAM Policy Document")
+			_, err := iamClient.DeletePolicy(context.Background(), &iam.DeletePolicyInput{
+				PolicyArn: policy.Policy.Arn,
+			})
+			require.NoError(t, err)
+		})
+
+		p := GetIamPolicyDocument(t, region, *policy.Policy.Arn)
+		t.Log("Retrieved Policy Document:", p)
+		assert.JSONEq(t, policyDocument, p)
+	})
+
+	t.Run("DoesNotExist", func(t *testing.T) {
+		_, err := GetIamPolicyDocumentE(t, region, "arn:aws:iam::1234567890:policy/does-not-exist")
+		require.Error(t, err)
+	})
+}

modules/helm/template.go

@@ -110,6 +110,9 @@ func RenderRemoteTemplateE(t testing.TestingT, options *Options, chartURL string
 
 	// ... and add the helm chart name, the remote repo and chart URL at the end
 	args = append(args, releaseName, "--repo", chartURL)
+	if options.Version != "" {
+		args = append(args, "--version", options.Version)
+	}
 
 	// Finally, call out to helm template command
 	return RunHelmCommandAndGetStdOutE(t, options, "template", args...)

modules/helm/template_test.go

@@ -27,6 +27,7 @@ func TestRemoteChartRender(t *testing.T) {
 		remoteChartSource  = "https://charts.bitnami.com/bitnami"
 		remoteChartName    = "nginx"
 		remoteChartVersion = "13.2.24"
+		registry           = "registry-1.docker.io"
 	)
 
 	t.Parallel()
@@ -42,11 +43,12 @@ func TestRemoteChartRender(t *testing.T) {
 	options := &Options{
 		SetValues: map[string]string{
 			"image.repository": remoteChartName,
-			"image.registry":   "",
+			"image.registry":   registry,
 			"image.tag":        remoteChartVersion,
 		},
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 		Logger:         logger.Discard,
+		Version:        remoteChartVersion,
 	}
 
 	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
@@ -62,10 +64,10 @@ func TestRemoteChartRender(t *testing.T) {
 	require.Equal(t, namespaceName, deployment.Namespace)
 
 	// Finally, we verify the deployment pod template spec is set to the expected container image value
-	expectedContainerImage := remoteChartName + ":" + remoteChartVersion
+	expectedContainerImage := registry + "/" + remoteChartName + ":" + remoteChartVersion
 	deploymentContainers := deployment.Spec.Template.Spec.Containers
 	require.Equal(t, len(deploymentContainers), 1)
-	require.Equal(t, deploymentContainers[0].Image, expectedContainerImage)
+	require.Equal(t, expectedContainerImage, deploymentContainers[0].Image)
 }
 
 // Test that we can dump all the manifest locally a remote chart (e.g bitnami/nginx)
@@ -81,15 +83,15 @@ func TestRemoteChartRenderDiff(t *testing.T) {
 
 	initialSnapshot := t.TempDir()
 	updatedSnapshot := t.TempDir()
-	renderChartDump(t, "5.0.0", initialSnapshot)
-	output := renderChartDump(t, "5.1.0", updatedSnapshot)
+	renderChartDump(t, "13.2.20", initialSnapshot)
+	output := renderChartDump(t, "13.2.24", updatedSnapshot)
 
 	options := &Options{
 		Logger:       logger.Default,
 		SnapshotPath: initialSnapshot,
 	}
 	// diff in: spec.initContainers.preserve-logs-symlinks.imag, spec.containers.nginx.image, tls certificates
-	require.Equal(t, 5, DiffAgainstSnapshot(t, options, output, "nginx"))
+	require.Equal(t, 4, DiffAgainstSnapshot(t, options, output, "nginx"))
 }
 
 // render chart dump and return the rendered output
@@ -111,6 +113,7 @@ func renderChartDump(t *testing.T, remoteChartVersion, snapshotDir string) strin
 		},
 		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
 		Logger:         logger.Discard,
+		Version:        remoteChartVersion,
 	}
 
 	// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since

modules/k8s/tunnel.go

@@ -189,15 +189,18 @@ func (tunnel *Tunnel) ForwardPortE(t testing.TestingT) error {
 		tunnel.logger.Logf(t, "Error creating a new Kubernetes client: %s", err)
 		return err
 	}
-	kubeConfigPath, err := tunnel.kubectlOptions.GetConfigPath(t)
-	if err != nil {
-		tunnel.logger.Logf(t, "Error getting kube config path: %s", err)
-		return err
-	}
-	config, err := LoadApiClientConfigE(kubeConfigPath, tunnel.kubectlOptions.ContextName)
-	if err != nil {
-		tunnel.logger.Logf(t, "Error loading Kubernetes config: %s", err)
-		return err
+	config := tunnel.kubectlOptions.RestConfig
+	if config == nil {
+		kubeConfigPath, err := tunnel.kubectlOptions.GetConfigPath(t)
+		if err != nil {
+			tunnel.logger.Logf(t, "Error getting kube config path: %s", err)
+			return err
+		}
+		config, err = LoadApiClientConfigE(kubeConfigPath, tunnel.kubectlOptions.ContextName)
+		if err != nil {
+			tunnel.logger.Logf(t, "Error loading Kubernetes config: %s", err)
+			return err
+		}
 	}
 
 	// Find the pod to port forward to

modules/shell/command.go

@@ -84,6 +84,27 @@ func RunCommandAndGetStdOutE(t testing.TestingT, command Command) (string, error
 	return output.Stdout(), nil
 }
 
+// RunCommandAndGetStdOutErr runs a shell command and returns solely its stdout and stderr as a string. The stdout and
+// stderr of that command will also be logged with Command.Log to make debugging easier. If there are any errors, fail
+// the test.
+func RunCommandAndGetStdOutErr(t testing.TestingT, command Command) (stdout string, stderr string) {
+	stdout, stderr, err := RunCommandAndGetStdOutErrE(t, command)
+	require.NoError(t, err)
+	return stdout, stderr
+}
+
+// RunCommandAndGetStdOutErrE runs a shell command and returns solely its stdout and stderr as a string. The stdout
+// and stderr of that command will also be printed to the stdout and stderr of this Go program to make debugging easier.
+// Any returned error will be of type ErrWithCmdOutput, containing the output streams and the underlying error.
+func RunCommandAndGetStdOutErrE(t testing.TestingT, command Command) (stdout string, stderr string, err error) {
+	output, err := runCommand(t, command)
+	if err != nil {
+		return output.Stdout(), output.Stderr(), &ErrWithCmdOutput{err, output}
+	}
+
+	return output.Stdout(), output.Stderr(), nil
+}
+
 type ErrWithCmdOutput struct {
 	Underlying error
 	Output     *output

modules/shell/command_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/random"
@@ -183,3 +184,29 @@ func TestCommandOutputType(t *testing.T) {
 		assert.Len(t, o.Output.Combined(), len(stdout)+len(stderr)+1) // +1 for newline
 	}
 }
+
+func TestCommandWithStdoutAndStdErr(t *testing.T) {
+	t.Parallel()
+
+	stdout := "hello world"
+	stderr := "this command has failed"
+	command := Command{
+		Command: "sh",
+		Args:    []string{"-c", `echo "` + stdout + `" && echo "` + stderr + `" >&2`},
+		Logger:  logger.Discard,
+	}
+
+	t.Run("MustNotError", func(t *testing.T) {
+		ostdout, ostderr := RunCommandAndGetStdOutErr(t, command)
+		assert.Equal(t, stdout, ostdout)
+		assert.Equal(t, stderr, ostderr)
+	})
+
+	t.Run("ReturnError", func(t *testing.T) {
+		ostdout, ostderr, err := RunCommandAndGetStdOutErrE(t, command)
+		require.NoError(t, err)
+		assert.Equal(t, stdout, ostdout)
+		assert.Equal(t, stderr, ostderr)
+	})
+
+}

modules/terraform/cmd.go

@@ -39,6 +39,9 @@ const (
 
 	// TerraformDefaultPath to run terraform
 	TerraformDefaultPath = "terraform"
+
+	// TerragruntDefaultPath to run terragrunt
+	TerragruntDefaultPath = "terragrunt"
 )
 
 var DefaultExecutable = defaultTerraformExecutable()
@@ -49,8 +52,22 @@ func GetCommonOptions(options *Options, args ...string) (*Options, []string) {
 		options.TerraformBinary = DefaultExecutable
 	}
 
-	if options.TerraformBinary == "terragrunt" {
+	if options.TerraformBinary == TerragruntDefaultPath {
 		args = append(args, "--terragrunt-non-interactive")
+		// for newer Terragrunt version, setting simplified log formatting
+		if options.EnvVars == nil {
+			options.EnvVars = map[string]string{}
+		}
+		_, tgLogSet := options.EnvVars["TERRAGRUNT_LOG_FORMAT"]
+		if !tgLogSet {
+			// key-value format for terragrunt logs to avoid colors and have plain form
+			// https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-log-format
+			options.EnvVars["TERRAGRUNT_LOG_FORMAT"] = "key-value"
+		}
+		_, tgLogFormat := options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"]
+		if !tgLogFormat {
+			options.EnvVars["TERRAGRUNT_LOG_CUSTOM_FORMAT"] = "%msg(color=disable)"
+		}
 	}
 
 	if options.Parallelism > 0 && len(args) > 0 && collections.ListContains(commandsWithParallelism, args[0]) {