feat: introducing "copy_terraform_lock_file" to fine tune Lock File H…

…andling (#2889) * feat: introducing "copy_terraform_lock_file" to fine tune Lock File Handling Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * fix: renaming test name Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * fix: linting doc files Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * docs: applying suggestions Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * fix: adding comment about early return Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * fix: aligning code with the latest changes from main Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * test: fixing the TestRenderJsonMetadataTerraform test Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> * docs: applying suggestions Signed-off-by: Rodrigo Fior Kuntzer <[email protected]> --------- Signed-off-by: Rodrigo Fior Kuntzer <[email protected]>
gruntwork-io · Sep 24, 2024 · 6f5e448 · 6f5e448
1 parent 70797fd
commit 6f5e448
Show file tree

Hide file tree

Showing 16 changed files with 220 additions and 24 deletions.
diff --git a/cli/commands/terraform/action.go b/cli/commands/terraform/action.go
@@ -327,7 +327,7 @@ func runTerragruntWithConfig(ctx context.Context, originalTerragruntOptions *opt
 		runTerraformError := RunTerraformWithRetry(ctx, terragruntOptions)
 
 		var lockFileError error
-		if shouldCopyLockFile(terragruntOptions.TerraformCliArgs) {
+		if ShouldCopyLockFile(terragruntOptions.TerraformCliArgs, terragruntConfig.Terraform) {
 			// Copy the lock file from the Terragrunt working dir (e.g., .terragrunt-cache/xxx/<some-module>) to the
 			// user's working dir (e.g., /live/stage/vpc). That way, the lock file will end up right next to the user's
 			// terragrunt.hcl and can be checked into version control. Note that in the past, Terragrunt allowed the
@@ -372,6 +372,7 @@ func confirmActionWithDependentModules(ctx context.Context, terragruntOptions *o
 	return true
 }
 
+// ShouldCopyLockFile verifies if the lock file should be copied to the user's working directory
 // Terraform 0.14 now manages a lock file for providers. This can be updated
 // in three ways:
 // * `terraform init` in a module where no `.terraform.lock.hcl` exists
@@ -386,7 +387,15 @@ func confirmActionWithDependentModules(ctx context.Context, terragruntOptions *o
 // There are lots of details at [hashicorp/terraform#27264](https://github.com/hashicorp/terraform/issues/27264#issuecomment-743389837)
 // The `providers lock` sub command enables you to ensure that the lock file is
 // fully populated.
-func shouldCopyLockFile(args []string) bool {
+func ShouldCopyLockFile(args []string, terraformConfig *config.TerraformConfig) bool {
+	// If the user has explicitly set CopyTerraformLockFile to false, then we should not copy the lock file on any command
+	// This is useful for users who want to manage the lock file themselves outside the working directory
+	// if the user has not set CopyTerraformLockFile or if they have explicitly defined it to true,
+	// then we should copy the lock file on init and providers lock as defined above and not do and early return here
+	if terraformConfig != nil && terraformConfig.CopyTerraformLockFile != nil && !*terraformConfig.CopyTerraformLockFile {
+		return false
+	}
+
 	if util.FirstArg(args) == terraform.CommandNameInit {
 		return true
 	}

diff --git a/cli/commands/terraform/action_test.go b/cli/commands/terraform/action_test.go
@@ -498,3 +498,80 @@ func createTempFile(t *testing.T) string {
 
 	return filepath.ToSlash(tmpFile.Name())
 }
+
+func TestShouldCopyLockFile(t *testing.T) {
+	t.Parallel()
+
+	type args struct {
+		args            []string
+		terraformConfig *config.TerraformConfig
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "init without terraform config",
+			args: args{
+				args: []string{"init"},
+			},
+			want: true,
+		},
+		{
+			name: "providers lock without terraform config",
+			args: args{
+				args: []string{"providers", "lock"},
+			},
+			want: true,
+		},
+		{
+			name: "providers schema without terraform config",
+			args: args{
+				args: []string{"providers", "schema"},
+			},
+			want: false,
+		},
+		{
+			name: "plan without terraform config",
+			args: args{
+				args: []string{"plan"},
+			},
+			want: false,
+		},
+		{
+			name: "init with empty terraform config",
+			args: args{
+				args:            []string{"init"},
+				terraformConfig: &config.TerraformConfig{},
+			},
+			want: true,
+		},
+		{
+			name: "init with CopyTerraformLockFile enabled",
+			args: args{
+				args: []string{"init"},
+				terraformConfig: &config.TerraformConfig{
+					CopyTerraformLockFile: &[]bool{true}[0],
+				},
+			},
+			want: true,
+		},
+		{
+			name: "init with CopyTerraformLockFile disabled",
+			args: args{
+				args: []string{"init"},
+				terraformConfig: &config.TerraformConfig{
+					CopyTerraformLockFile: &[]bool{false}[0],
+				},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			assert.Equalf(t, tt.want, terraform.ShouldCopyLockFile(tt.args.args, tt.args.terraformConfig), "shouldCopyLockFile(%v, %v)", tt.args.args, tt.args.terraformConfig)
+		})
+	}
+}
diff --git a/config/config.go b/config/config.go
@@ -438,6 +438,8 @@ type TerraformConfig struct {
 	// Ideally we can avoid the pointer to list slice, but if it is not a pointer, Terraform requires the attribute to
 	// be defined and we want to make this optional.
 	IncludeInCopy *[]string `hcl:"include_in_copy,attr"`
+
+	CopyTerraformLockFile *bool `hcl:"copy_terraform_lock_file,attr"`
 }
 
 func (cfg *TerraformConfig) String() string {

diff --git a/config/config_as_cty.go b/config/config_as_cty.go
@@ -495,12 +495,13 @@ func engineConfigAsCty(config *EngineConfig) (cty.Value, error) {
 // CtyTerraformConfig is an alternate representation of TerraformConfig that converts internal blocks into a map that
 // maps the name to the underlying struct, as opposed to a list representation.
 type CtyTerraformConfig struct {
-	ExtraArgs     map[string]TerraformExtraArguments `cty:"extra_arguments"`
-	Source        *string                            `cty:"source"`
-	IncludeInCopy *[]string                          `cty:"include_in_copy"`
-	BeforeHooks   map[string]Hook                    `cty:"before_hook"`
-	AfterHooks    map[string]Hook                    `cty:"after_hook"`
-	ErrorHooks    map[string]ErrorHook               `cty:"error_hook"`
+	ExtraArgs             map[string]TerraformExtraArguments `cty:"extra_arguments"`
+	Source                *string                            `cty:"source"`
+	IncludeInCopy         *[]string                          `cty:"include_in_copy"`
+	CopyTerraformLockFile *bool                              `cty:"copy_terraform_lock_file"`
+	BeforeHooks           map[string]Hook                    `cty:"before_hook"`
+	AfterHooks            map[string]Hook                    `cty:"after_hook"`
+	ErrorHooks            map[string]ErrorHook               `cty:"error_hook"`
 }
 
 // Serialize TerraformConfig to a cty Value, but with maps instead of lists for the blocks.
@@ -510,12 +511,13 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) {
 	}
 
 	configCty := CtyTerraformConfig{
-		Source:        config.Source,
-		IncludeInCopy: config.IncludeInCopy,
-		ExtraArgs:     map[string]TerraformExtraArguments{},
-		BeforeHooks:   map[string]Hook{},
-		AfterHooks:    map[string]Hook{},
-		ErrorHooks:    map[string]ErrorHook{},
+		Source:                config.Source,
+		IncludeInCopy:         config.IncludeInCopy,
+		CopyTerraformLockFile: config.CopyTerraformLockFile,
+		ExtraArgs:             map[string]TerraformExtraArguments{},
+		BeforeHooks:           map[string]Hook{},
+		AfterHooks:            map[string]Hook{},
+		ErrorHooks:            map[string]ErrorHook{},
 	}
 
 	for _, arg := range config.ExtraArgs {

diff --git a/config/include.go b/config/include.go
@@ -278,6 +278,10 @@ func (cfg *TerragruntConfig) Merge(sourceConfig *TerragruntConfig, terragruntOpt
 				cfg.Terraform.Source = sourceConfig.Terraform.Source
 			}
 
+			if sourceConfig.Terraform.CopyTerraformLockFile != nil {
+				cfg.Terraform.CopyTerraformLockFile = sourceConfig.Terraform.CopyTerraformLockFile
+			}
+
 			mergeExtraArgs(terragruntOptions, sourceConfig.Terraform.ExtraArgs, &cfg.Terraform.ExtraArgs)
 
 			mergeHooks(terragruntOptions, sourceConfig.Terraform.BeforeHooks, &cfg.Terraform.BeforeHooks)
@@ -446,6 +450,10 @@ func (cfg *TerragruntConfig) DeepMerge(sourceConfig *TerragruntConfig, terragrun
 				cfg.Terraform.Source = sourceConfig.Terraform.Source
 			}
 
+			if sourceConfig.Terraform.CopyTerraformLockFile != nil {
+				cfg.Terraform.CopyTerraformLockFile = sourceConfig.Terraform.CopyTerraformLockFile
+			}
+
 			if sourceConfig.Terraform.IncludeInCopy != nil {
 				srcList := *sourceConfig.Terraform.IncludeInCopy
 

diff --git a/config/include_test.go b/config/include_test.go
@@ -144,6 +144,11 @@ func TestMergeConfigIntoIncludedConfig(t *testing.T) {
 			&config.TerragruntConfig{IamWebIdentityToken: "token"},
 			&config.TerragruntConfig{IamWebIdentityToken: "token"},
 		},
+		{
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}},
+		},
 	}
 
 	for _, testCase := range testCases {
@@ -288,6 +293,12 @@ func TestDeepMergeConfigIntoIncludedConfig(t *testing.T) {
 			&config.TerragruntConfig{Inputs: originalMap},
 			&config.TerragruntConfig{Inputs: mergedMap},
 		},
+		{
+			"terraform copy_terraform_lock_file",
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}},
+		},
 	}
 
 	for _, tt := range tc {

diff --git a/docs/_docs/02_features/lock-file-handling.md b/docs/_docs/02_features/lock-file-handling.md
@@ -96,3 +96,16 @@ should end up looking something like this:
 
 Also, any time you change the providers you're using, and re-run `init`, the lock file will be updated, so make sure
 to check the updates into version control too.
+
+### Disabling the copy of the generated lock file
+
+In certain use cases, like when using a remote module containing a lock file within it, you probably
+don't want Terragrunt to also copy the lock file into your working directory. In these scenarios you, can opt-out of copying
+the `.terraform.lock.hcl` file by using `copy_terraform_lock_file = false` in the `terraform` configuration block as follows:
+
+```hcl
+terraform {
+  ...
+  copy_terraform_lock_file = false
+}
+```
diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md
@@ -105,6 +105,11 @@ The `terraform` block supports the following arguments:
     can specify that in this list to ensure it gets copied over to the scratch copy
     (e.g., `include_in_copy = [".python-version"]`).
 
+- `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock
+  file into your source repository from your working directory as described in
+  [Lock File Handling]({{site.baseurl}}/docs/features/lock-file-handling/). This attribute allows you to disable the copy
+  of the generated or existing `.terraform.lock.hcl` from the temp folder into the working directory. Default is `true`.
+
 - `extra_arguments` (block): Nested blocks used to specify extra CLI arguments to pass to the `tofu`/`terraform` binary. Learn more
   about its usage in the [Keep your CLI flags DRY]({{site.baseurl}}/docs/features/keep-your-cli-flags-dry/) use case overview. Supports
   the following arguments:

diff --git a/test/fixtures/download/local-disable-copy-terraform-lock-file/terragrunt.hcl b/test/fixtures/download/local-disable-copy-terraform-lock-file/terragrunt.hcl
@@ -0,0 +1,8 @@
+inputs = {
+  name = "World"
+}
+
+terraform {
+  source                   = "../hello-world"
+  copy_terraform_lock_file = false
+}
diff --git a/test/fixtures/download/local-include-disable-copy-lock-file/module-a/terragrunt.hcl b/test/fixtures/download/local-include-disable-copy-lock-file/module-a/terragrunt.hcl
@@ -0,0 +1,7 @@
+inputs = {
+  name = "Module A"
+}
+
+terraform {
+  source = "../../hello-world"
+}
diff --git a/test/fixtures/download/local-include-disable-copy-lock-file/module-b/terragrunt.hcl b/test/fixtures/download/local-include-disable-copy-lock-file/module-b/terragrunt.hcl
@@ -0,0 +1,14 @@
+inputs = {
+  name = "Module B"
+}
+
+terraform {
+  source = "../../hello-world"
+  copy_terraform_lock_file = false
+}
+
+prevent_destroy = true
+
+include {
+  path = find_in_parent_folders("terragrunt.hcl")
+}
diff --git a/test/fixtures/download/local-include-disable-copy-lock-file/terragrunt.hcl b/test/fixtures/download/local-include-disable-copy-lock-file/terragrunt.hcl
@@ -0,0 +1,7 @@
+terraform {
+  include_in_copy = ["**/.terraform-version"]
+}
+
+dependencies {
+  paths = ["../module-a"]
+}
diff --git a/test/fixtures/read-config/full/source.hcl b/test/fixtures/read-config/full/source.hcl
@@ -24,8 +24,9 @@ remote_state {
 }
 
 terraform {
-  source          = "./delorean"
-  include_in_copy = ["time_machine.*"]
+  source                   = "./delorean"
+  include_in_copy          = ["time_machine.*"]
+  copy_terraform_lock_file = true
 
   extra_arguments "var-files" {
     commands = ["apply", "plan"]

diff --git a/test/integration_download_test.go b/test/integration_download_test.go
@@ -39,6 +39,8 @@ const (
 	testFixtureLocalPreventDestroyDependencies        = "fixtures/download/local-with-prevent-destroy-dependencies"
 	testFixtureLocalIncludePreventDestroyDependencies = "fixtures/download/local-include-with-prevent-destroy-dependencies"
 	testFixtureNotExistingSource                      = "fixtures/download/invalid-path"
+	testFixtureDisableCopyLockFilePath                = "fixtures/download/local-disable-copy-terraform-lock-file"
+	testFixtureIncludeDisableCopyLockFilePath         = "fixtures/download/local-include-disable-copy-lock-file/module-b"
 )
 
 func TestLocalDownload(t *testing.T) {
@@ -55,6 +57,34 @@ func TestLocalDownload(t *testing.T) {
 	runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureLocalDownloadPath)
 }
 
+func TestLocalDownloadDisableCopyTerraformLockFile(t *testing.T) {
+	t.Parallel()
+
+	cleanupTerraformFolder(t, testFixtureDisableCopyLockFilePath)
+
+	runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureDisableCopyLockFilePath)
+
+	// The terraform lock file should not be copied if `copy_terraform_lock_file = false`
+	assert.NoFileExists(t, util.JoinPath(testFixtureDisableCopyLockFilePath, util.TerraformLockFile))
+
+	// Run a second time to make sure the temporary folder can be reused without errors
+	runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureDisableCopyLockFilePath)
+}
+
+func TestLocalIncludeDisableCopyTerraformLockFile(t *testing.T) {
+	t.Parallel()
+
+	cleanupTerraformFolder(t, testFixtureIncludeDisableCopyLockFilePath)
+
+	runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureIncludeDisableCopyLockFilePath)
+
+	// The terraform lock file should not be copied if `copy_terraform_lock_file = false`
+	assert.NoFileExists(t, util.JoinPath(testFixtureIncludeDisableCopyLockFilePath, util.TerraformLockFile))
+
+	// Run a second time to make sure the temporary folder can be reused without errors
+	runTerragrunt(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+testFixtureIncludeDisableCopyLockFilePath)
+}
+
 func TestLocalDownloadWithHiddenFolder(t *testing.T) {
 	t.Parallel()
 

diff --git a/test/integration_json_test.go b/test/integration_json_test.go
@@ -429,12 +429,13 @@ func TestRenderJsonMetadataTerraform(t *testing.T) {
 	var expectedTerraform = map[string]interface{}{
 		"metadata": terragruntMetadata,
 		"value": map[string]interface{}{
-			"after_hook":      map[string]interface{}{},
-			"before_hook":     map[string]interface{}{},
-			"error_hook":      map[string]interface{}{},
-			"extra_arguments": map[string]interface{}{},
-			"include_in_copy": nil,
-			"source":          "../terraform",
+			"after_hook":               map[string]interface{}{},
+			"before_hook":              map[string]interface{}{},
+			"error_hook":               map[string]interface{}{},
+			"extra_arguments":          map[string]interface{}{},
+			"include_in_copy":          nil,
+			"source":                   "../terraform",
+			"copy_terraform_lock_file": nil,
 		},
 	}
 

diff --git a/test/integration_test.go b/test/integration_test.go
@@ -2043,8 +2043,9 @@ func TestReadTerragruntConfigFull(t *testing.T) {
 	assert.Equal(
 		t,
 		map[string]interface{}{
-			"source":          "./delorean",
-			"include_in_copy": []interface{}{"time_machine.*"},
+			"source":                   "./delorean",
+			"include_in_copy":          []interface{}{"time_machine.*"},
+			"copy_terraform_lock_file": true,
 			"extra_arguments": map[string]interface{}{
 				"var-files": map[string]interface{}{
 					"name":               "var-files",