fix: Split stdout and stderr streams even when using custom logging (#…

…3686)

* fix: Split stdout and stderr streams even when using custom logging

* fix: Fixing tests

cli/commands/terraform/action.go

@@ -283,7 +283,7 @@ func generateConfig(terragruntConfig *config.TerragruntConfig, updatedTerragrunt
 
 // Runs terraform with the given options and CLI args.
 // This will forward all the args and extra_arguments directly to Terraform.
-
+//
 // This function takes in the "original" terragrunt options which has the unmodified 'WorkingDir' from before downloading the code from the source URL,
 // and the "updated" terragrunt options that will contain the updated 'WorkingDir' into which the code has been downloaded
 func runTerragruntWithConfig(ctx context.Context, originalTerragruntOptions *options.TerragruntOptions, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig, target *Target) error {
@@ -674,6 +674,7 @@ func prepareInitOptions(terragruntOptions *options.TerragruntOptions) (*options.
 	initOptions.TerraformCliArgs = []string{terraform.CommandNameInit}
 	initOptions.WorkingDir = terragruntOptions.WorkingDir
 	initOptions.TerraformCommand = terraform.CommandNameInit
+	initOptions.Headless = true
 
 	initOutputForCommands := []string{terraform.CommandNamePlan, terraform.CommandNameApply}
 	terraformCommand := util.FirstArg(terragruntOptions.TerraformCliArgs)

options/options.go

@@ -386,6 +386,15 @@ type TerragruntOptions struct {
 
 	// Errors is a configuration for error handling.
 	Errors *ErrorsConfig
+
+	// Headless is set when Terragrunt is running in
+	// headless mode. In this mode, Terragrunt will not
+	// return stdout/stderr directly to the caller.
+	//
+	// It will instead write the output to INFO,
+	// as it's not something intended for a user
+	// to use in a programmatic way.
+	Headless bool
 }
 
 // TerragruntOptionsFunc is a functional option type used to pass options in certain integration tests
@@ -672,6 +681,7 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp
 		Errors:                cloneErrorsConfig(opts.Errors),
 		ScaffoldNoIncludeRoot: opts.ScaffoldNoIncludeRoot,
 		ScaffoldRootFileName:  opts.ScaffoldRootFileName,
+		Headless:              opts.Headless,
 	}, nil
 }
 

shell/run_shell_cmd.go

@@ -139,25 +139,31 @@ func RunShellCommandWithOutput(
 				WithField(placeholders.TFCmdArgsKeyName, args)
 
 			if opts.JSONLogFormat && !cli.Args(args).Normalize(cli.SingleDashFlag).Contains(terraform.FlagNameJSON) {
-				outWriter = writer.New(
-					writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
-					writer.WithDefaultLevel(log.StdoutLevel),
+				outWriter = buildOutWriter(
+					opts,
+					logger,
+					outWriter,
+					errWriter,
 				)
 
-				errWriter = writer.New(
-					writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
-					writer.WithDefaultLevel(log.StderrLevel),
+				errWriter = buildErrWriter(
+					opts,
+					logger,
+					errWriter,
 				)
 			} else if !shouldForceForwardTFStdout(args) {
-				outWriter = writer.New(
-					writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
-					writer.WithDefaultLevel(log.StdoutLevel),
+				outWriter = buildOutWriter(
+					opts,
+					logger,
+					outWriter,
+					errWriter,
 					writer.WithMsgSeparator(logMsgSeparator),
 				)
 
-				errWriter = writer.New(
-					writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
-					writer.WithDefaultLevel(log.StderrLevel),
+				errWriter = buildErrWriter(
+					opts,
+					logger,
+					errWriter,
 					writer.WithMsgSeparator(logMsgSeparator),
 					writer.WithParseFunc(terraform.ParseLogFunc(tfLogMsgPrefix, false)),
 				)
@@ -245,6 +251,53 @@ func RunShellCommandWithOutput(
 	return &output, err
 }
 
+// buildOutWriter returns the writer for the command's stdout.
+//
+// When Terragrunt is running in Headless mode, we want to forward
+// any stdout to the INFO log level, otherwise, we want to forward
+// stdout to the STDOUT log level.
+//
+// Also accepts any additional writer options desired.
+func buildOutWriter(opts *options.TerragruntOptions, logger log.Logger, outWriter, errWriter io.Writer, writerOptions ...writer.Option) io.Writer {
+	logLevel := log.StdoutLevel
+
+	if opts.Headless {
+		logLevel = log.InfoLevel
+		outWriter = errWriter
+	}
+
+	options := []writer.Option{
+		writer.WithLogger(logger.WithOptions(log.WithOutput(outWriter))),
+		writer.WithDefaultLevel(logLevel),
+	}
+	options = append(options, writerOptions...)
+
+	return writer.New(options...)
+}
+
+// buildErrWriter returns the writer for the command's stderr.
+//
+// When Terragrunt is running in Headless mode, we want to forward
+// any stderr to the ERROR log level, otherwise, we want to forward
+// stderr to the STDERR log level.
+//
+// Also accepts any additional writer options desired.
+func buildErrWriter(opts *options.TerragruntOptions, logger log.Logger, errWriter io.Writer, writerOptions ...writer.Option) io.Writer {
+	logLevel := log.StderrLevel
+
+	if opts.Headless {
+		logLevel = log.ErrorLevel
+	}
+
+	options := []writer.Option{
+		writer.WithLogger(logger.WithOptions(log.WithOutput(errWriter))),
+		writer.WithDefaultLevel(logLevel),
+	}
+	options = append(options, writerOptions...)
+
+	return writer.New(options...)
+}
+
 // isTerraformCommandThatNeedsPty returns true if the sub command of terraform we are running requires a pty.
 func isTerraformCommandThatNeedsPty(args []string) (bool, error) {
 	if len(args) == 0 || !util.ListContainsElement(terraformCommandsThatNeedPty, args[0]) {

test/integration_docs_test.go

@@ -25,13 +25,13 @@ func TestDocsQuickStart(t *testing.T) {
 		tmpEnvPath := helpers.CopyEnvironment(t, stepPath)
 		rootPath := util.JoinPath(tmpEnvPath, stepPath)
 
-		_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt plan --terragrunt-non-interactive --terragrunt-log-level trace --terragrunt-working-dir "+rootPath)
+		stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt plan --terragrunt-non-interactive --terragrunt-log-level trace --terragrunt-working-dir "+rootPath)
 		require.NoError(t, err)
-		assert.Contains(t, stderr, "Plan: 1 to add, 0 to change, 0 to destroy.")
+		assert.Contains(t, stdout, "Plan: 1 to add, 0 to change, 0 to destroy.")
 
-		_, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level trace --terragrunt-working-dir "+rootPath)
+		stdout, _, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-log-level trace --terragrunt-working-dir "+rootPath)
 		require.NoError(t, err)
-		assert.Contains(t, stderr, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.")
+		assert.Contains(t, stdout, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed.")
 
 	})
 

test/integration_errors_test.go

@@ -57,12 +57,12 @@ func TestRunAllIgnoreError(t *testing.T) {
 	tmpEnvPath := helpers.CopyEnvironment(t, testRunAllIgnoreErrors)
 	rootPath := util.JoinPath(tmpEnvPath, testRunAllIgnoreErrors)
 
-	_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)
+	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)
 
 	require.NoError(t, err)
 	assert.Contains(t, stderr, "Ignoring error example1")
 	assert.NotContains(t, stderr, "Ignoring error example2")
-	assert.Contains(t, stderr, "value-from-app-2")
+	assert.Contains(t, stdout, "value-from-app-2")
 }
 
 func TestRetryError(t *testing.T) {

test/integration_test.go

@@ -200,31 +200,36 @@ func TestLogCustomFormatOutput(t *testing.T) {
 
 	testCases := []struct {
 		logCustomFormat    string
-		expectedOutputRegs []*regexp.Regexp
+		expectedStdOutRegs []*regexp.Regexp
+		expectedStdErrRegs []*regexp.Regexp
 	}{
 		{
-			logCustomFormat: "%time %level %prefix %msg",
-			expectedOutputRegs: []*regexp.Regexp{
+			logCustomFormat:    "%time %level %prefix %msg",
+			expectedStdOutRegs: []*regexp.Regexp{},
+			expectedStdErrRegs: []*regexp.Regexp{
 				regexp.MustCompile(`\d{2}:\d{2}:\d{2}\.\d{3} debug ` + absPathReg + regexp.QuoteMeta(" Terragrunt Version:")),
 				regexp.MustCompile(`\d{2}:\d{2}:\d{2}\.\d{3} debug ` + absPathReg + `/dep Module ` + absPathReg + `/dep must wait for 0 dependencies to finish`),
 				regexp.MustCompile(`\d{2}:\d{2}:\d{2}\.\d{3} debug ` + absPathReg + `/app Module ` + absPathReg + `/app must wait for 1 dependencies to finish`),
 			},
 		},
 		{
-			logCustomFormat: "%interval %level(case=upper) %prefix(path=short-relative,prefix='[',suffix='] ')%msg(path=relative)",
-			expectedOutputRegs: []*regexp.Regexp{
+			logCustomFormat:    "%interval %level(case=upper) %prefix(path=short-relative,prefix='[',suffix='] ')%msg(path=relative)",
+			expectedStdOutRegs: []*regexp.Regexp{},
+			expectedStdErrRegs: []*regexp.Regexp{
 				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" DEBUG Terragrunt Version:")),
 				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" DEBUG [dep] Module ./dep must wait for 0 dependencies to finish")),
 				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" DEBUG [app] Module ./app must wait for 1 dependencies to finish")),
 			},
 		},
 		{
 			logCustomFormat: "%interval%(content=' plain-text ')%level(case=upper,width=6) %prefix(path=short-relative,suffix=' ')%tf-path(suffix=' ')%tf-command-args(suffix=': ')%msg(path=relative)",
-			expectedOutputRegs: []*regexp.Regexp{
-				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" plain-text DEBUG  Terragrunt Version:")),
+			expectedStdOutRegs: []*regexp.Regexp{
 				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" plain-text STDOUT dep "+wrappedBinary()+" init -input=false -no-color: Initializing the backend...")),
 				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" plain-text STDOUT app "+wrappedBinary()+" init -input=false -no-color: Initializing the backend...")),
 			},
+			expectedStdErrRegs: []*regexp.Regexp{
+				regexp.MustCompile(`\d{4}` + regexp.QuoteMeta(" plain-text DEBUG  Terragrunt Version:")),
+			},
 		},
 	}
 
@@ -241,10 +246,14 @@ func TestLogCustomFormatOutput(t *testing.T) {
 			rootPath, err := filepath.EvalSymlinks(rootPath)
 			require.NoError(t, err)
 
-			_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all init --terragrunt-log-level trace --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-custom-format=%q --terragrunt-working-dir %s", testCase.logCustomFormat, rootPath))
+			stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all init --terragrunt-log-level trace --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-custom-format=%q --terragrunt-working-dir %s", testCase.logCustomFormat, rootPath))
 			require.NoError(t, err)
 
-			for _, reg := range testCase.expectedOutputRegs {
+			for _, reg := range testCase.expectedStdOutRegs {
+				assert.Regexp(t, reg, stdout)
+			}
+
+			for _, reg := range testCase.expectedStdErrRegs {
 				assert.Regexp(t, reg, stderr)
 			}
 		})
@@ -296,12 +305,12 @@ func TestLogWithAbsPath(t *testing.T) {
 	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureLogFormatter)
 	rootPath := util.JoinPath(tmpEnvPath, testFixtureLogFormatter)
 
-	_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level trace --terragrunt-log-show-abs-paths --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-format=pretty --terragrunt-working-dir "+rootPath)
+	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init --terragrunt-log-level trace --terragrunt-log-show-abs-paths --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-format=pretty --terragrunt-working-dir "+rootPath)
 	require.NoError(t, err)
 
 	for _, prefixName := range []string{"app", "dep"} {
 		prefixName = filepath.Join(rootPath, prefixName)
-		assert.Contains(t, stderr, "STDOUT ["+prefixName+"] "+wrappedBinary()+": Initializing provider plugins...")
+		assert.Contains(t, stdout, "STDOUT ["+prefixName+"] "+wrappedBinary()+": Initializing provider plugins...")
 		assert.Contains(t, stderr, "DEBUG  ["+prefixName+"] Reading Terragrunt config file at "+prefixName+"/terragrunt.hcl")
 	}
 }
@@ -356,11 +365,11 @@ func TestLogFormatPrettyOutput(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, prefixName := range []string{"app", "dep"} {
-		assert.Contains(t, stderr, "STDOUT ["+prefixName+"] "+wrappedBinary()+": Initializing provider plugins...")
+		assert.Contains(t, stdout, "STDOUT ["+prefixName+"] "+wrappedBinary()+": Initializing provider plugins...")
 		assert.Contains(t, stderr, "DEBUG  ["+prefixName+"] Reading Terragrunt config file at ./"+prefixName+"/terragrunt.hcl")
 	}
 
-	assert.Empty(t, stdout)
+	assert.NotEmpty(t, stdout)
 	assert.Contains(t, stderr, "DEBUG  Terragrunt Version:")
 }
 
@@ -371,17 +380,15 @@ func TestLogStdoutLevel(t *testing.T) {
 	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureLogStdoutLevel)
 	rootPath := util.JoinPath(tmpEnvPath, testFixtureLogStdoutLevel)
 
-	stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-log-level trace --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-format=pretty  --terragrunt-working-dir "+rootPath)
+	stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-log-level trace --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-format=pretty  --terragrunt-working-dir "+rootPath)
 	require.NoError(t, err)
 
-	assert.Empty(t, stdout)
-	assert.Contains(t, stderr, "STDOUT "+wrappedBinary()+": Changes to Outputs")
+	assert.Contains(t, stdout, "STDOUT "+wrappedBinary()+": Changes to Outputs")
 
-	stdout, stderr, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt destroy -auto-approve --terragrunt-log-level trace --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-format=pretty  --terragrunt-working-dir "+rootPath)
+	stdout, _, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt destroy -auto-approve --terragrunt-log-level trace --terragrunt-non-interactive -no-color --terragrunt-no-color --terragrunt-log-format=pretty  --terragrunt-working-dir "+rootPath)
 	require.NoError(t, err)
 
-	assert.Empty(t, stdout)
-	assert.Contains(t, stderr, "STDOUT "+wrappedBinary()+": Changes to Outputs")
+	assert.Contains(t, stdout, "STDOUT "+wrappedBinary()+": Changes to Outputs")
 }
 
 func TestLogFormatKeyValueOutput(t *testing.T) {
@@ -395,11 +402,11 @@ func TestLogFormatKeyValueOutput(t *testing.T) {
 			tmpEnvPath := helpers.CopyEnvironment(t, testFixtureLogFormatter)
 			rootPath := util.JoinPath(tmpEnvPath, testFixtureLogFormatter)
 
-			_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init -no-color --terragrunt-log-level trace --terragrunt-non-interactive "+flag+" --terragrunt-working-dir "+rootPath)
+			stdout, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run-all init -no-color --terragrunt-log-level trace --terragrunt-non-interactive "+flag+" --terragrunt-working-dir "+rootPath)
 			require.NoError(t, err)
 
 			for _, prefixName := range []string{"app", "dep"} {
-				assert.Contains(t, stderr, "level=stdout prefix="+prefixName+" tf-path="+wrappedBinary()+" msg=Initializing provider plugins...\n")
+				assert.Contains(t, stdout, "level=stdout prefix="+prefixName+" tf-path="+wrappedBinary()+" msg=Initializing provider plugins...\n")
 				assert.Contains(t, stderr, "level=debug prefix="+prefixName+" msg=Reading Terragrunt config file at ./"+prefixName+"/terragrunt.hcl\n")
 			}
 		})
@@ -418,7 +425,7 @@ func TestLogRawModuleOutput(t *testing.T) {
 
 	stdoutInline := strings.ReplaceAll(stdout, "\n", "")
 	assert.Contains(t, stdoutInline, "Initializing the backend...Initializing provider plugins...")
-	assert.NotRegexp(t, regexp.MustCompile(`(?i)(`+strings.Join(log.AllLevels.Names(), "|")+`)+`), stdoutInline)
+	assert.NotRegexp(t, `(?i)(`+strings.Join(log.AllLevels.Names(), "|")+`)+`, stdoutInline)
 }
 
 func TestTerragruntExcludesFile(t *testing.T) {