Skip to content

Commit

Permalink
feat(remote_state): implement outputs fetching from gcs
Browse files Browse the repository at this point in the history 
This implements outputs fetching from the GCS backend, in a similar fashion to
what has been implemented for S3.

I've also added some misc fixes:
* correct typo: steateBody -> stateBody
* drop unused Path parameter on GCS config (it's not supported)

Tested on my architecture, everything seems to work!

Ref: https://developer.hashicorp.com/terraform/language/settings/backends/gcs
  • Loading branch information
@TPXP
TPXP committed Aug 14, 2024
1 parent dd602a3 commit e08bbb4
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 25 deletions.
69 changes: 67 additions & 2 deletions config/dependency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
Expand All @@ -13,6 +14,7 @@ import (

"github.com/gruntwork-io/terragrunt/internal/cache"

"cloud.google.com/go/storage"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/go-getter"
Expand Down Expand Up @@ -815,6 +817,16 @@ func getTerragruntOutputJsonFromRemoteState(
}
ctx.TerragruntOptions.Logger.Debugf("Retrieved output from %s as json: %s using s3 bucket", targetConfigPath, jsonBytes)
return jsonBytes, nil
case "gcs":
jsonBytes, err := getTerragruntOutputJsonFromRemoteStateGCS(
targetTGOptions,
remoteState,
)
if err != nil {
return nil, err
}
ctx.TerragruntOptions.Logger.Debugf("Retrieved output from %s as json: %s using GCS bucket", targetConfigPath, jsonBytes)
return jsonBytes, nil
default:
ctx.TerragruntOptions.Logger.Errorf("FetchDependencyOutputFromState is not supported for backend %s, falling back to normal method", backend)
}
Expand Down Expand Up @@ -888,11 +900,64 @@ func getTerragruntOutputJsonFromRemoteStateS3(terragruntOptions *options.Terragr
terragruntOptions.Logger.Warnf("Failed to close remote state response %v", err)
}
}(result.Body)
steateBody, err := io.ReadAll(result.Body)
stateBody, err := io.ReadAll(result.Body)
if err != nil {
return nil, err
}
jsonState := string(stateBody)
jsonMap := make(map[string]interface{})
err = json.Unmarshal([]byte(jsonState), &jsonMap)
if err != nil {
return nil, err
}
jsonOutputs, err := json.Marshal(jsonMap["outputs"])
if err != nil {
return nil, err
}
return jsonOutputs, nil
}

// getTerragruntOutputJsonFromRemoteStateGCS pulls the output directly from an GCS bucket without calling Terraform
func getTerragruntOutputJsonFromRemoteStateGCS(terragruntOptions *options.TerragruntOptions, remoteState *remote.RemoteState) ([]byte, error) {
terragruntOptions.Logger.Debugf("Fetching outputs directly from gcs://%s/%s/default.tfstate", remoteState.Config["bucket"], remoteState.Config["prefix"])

gcsConfigExtended, err := remote.ParseExtendedGCSConfig(remoteState.Config)
if err != nil {
return nil, err
}

if err := remote.ValidateGCSConfig(gcsConfigExtended); err != nil {
return nil, err
}

var gcsConfig = gcsConfigExtended.RemoteStateConfigGCS

gcsClient, err := remote.CreateGCSClient(gcsConfig)
if err != nil {
return nil, err
}

bucket := gcsClient.Bucket(gcsConfig.Bucket)
object := bucket.Object(gcsConfig.Prefix + "/default.tfstate")

reader, err := object.NewReader(context.Background())

if err != nil {
return nil, err
}

defer func(reader *storage.Reader) {
err := reader.Close()
if err != nil {
terragruntOptions.Logger.Warnf("Failed to close remote state response %v", err)
}
}(reader)

stateBody, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
jsonState := string(steateBody)
jsonState := string(stateBody)
jsonMap := make(map[string]interface{})
err = json.Unmarshal([]byte(jsonState), &jsonMap)
if err != nil {
Expand Down
45 changes: 22 additions & 23 deletions remote/remote_state_gcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
* has to create them.
*/
type ExtendedRemoteStateConfigGCS struct {
remoteStateConfigGCS RemoteStateConfigGCS
RemoteStateConfigGCS RemoteStateConfigGCS `mapstructure:"remote_state_config_gcs"`

Project string `mapstructure:"project"`
Location string `mapstructure:"location"`
Expand All @@ -57,7 +57,6 @@ type RemoteStateConfigGCS struct {
Credentials string `mapstructure:"credentials"`
AccessToken string `mapstructure:"access_token"`
Prefix string `mapstructure:"prefix"`
Path string `mapstructure:"path"`
EncryptionKey string `mapstructure:"encryption_key"`

ImpersonateServiceAccount string `mapstructure:"impersonate_service_account"`
Expand Down Expand Up @@ -169,16 +168,16 @@ func (gcsInitializer GCSInitializer) buildInitializerCacheKey(gcsConfig *RemoteS
// Initialize the remote state GCS bucket specified in the given config. This function will validate the config
// parameters, create the GCS bucket if it doesn't already exist, and check that versioning is enabled.
func (gcsInitializer GCSInitializer) Initialize(ctx context.Context, remoteState *RemoteState, terragruntOptions *options.TerragruntOptions) error {
gcsConfigExtended, err := parseExtendedGCSConfig(remoteState.Config)
gcsConfigExtended, err := ParseExtendedGCSConfig(remoteState.Config)
if err != nil {
return err
}

if err := validateGCSConfig(gcsConfigExtended); err != nil {
if err := ValidateGCSConfig(gcsConfigExtended); err != nil {
return err
}

var gcsConfig = gcsConfigExtended.remoteStateConfigGCS
var gcsConfig = gcsConfigExtended.RemoteStateConfigGCS

cacheKey := gcsInitializer.buildInitializerCacheKey(&gcsConfig)
if initialized, hit := initializedRemoteStateCache.Get(ctx, cacheKey); initialized && hit {
Expand Down Expand Up @@ -245,7 +244,7 @@ func parseGCSConfig(config map[string]interface{}) (*RemoteStateConfigGCS, error
}

// Parse the given map into a GCS config
func parseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteStateConfigGCS, error) {
func ParseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteStateConfigGCS, error) {
var gcsConfig RemoteStateConfigGCS
var extendedConfig ExtendedRemoteStateConfigGCS

Expand All @@ -257,14 +256,14 @@ func parseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteState
return nil, errors.WithStackTrace(err)
}

extendedConfig.remoteStateConfigGCS = gcsConfig
extendedConfig.RemoteStateConfigGCS = gcsConfig

return &extendedConfig, nil
}

// Validate all the parameters of the given GCS remote state configuration
func validateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS) error {
var config = extendedConfig.remoteStateConfigGCS
func ValidateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS) error {
var config = extendedConfig.RemoteStateConfigGCS

if config.Bucket == "" {
return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("bucket"))
Expand All @@ -277,8 +276,8 @@ func validateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS) error {
// confirms, create the bucket and enable versioning for it.
func createGCSBucketIfNecessary(ctx context.Context, gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
// TODO: Remove lint suppression
if !DoesGCSBucketExist(gcsClient, &config.remoteStateConfigGCS) { //nolint:contextcheck
terragruntOptions.Logger.Debugf("Remote state GCS bucket %s does not exist. Attempting to create it", config.remoteStateConfigGCS.Bucket)
if !DoesGCSBucketExist(gcsClient, &config.RemoteStateConfigGCS) { //nolint:contextcheck
terragruntOptions.Logger.Debugf("Remote state GCS bucket %s does not exist. Attempting to create it", config.RemoteStateConfigGCS.Bucket)

// A project must be specified in order for terragrunt to automatically create a storage bucket.
if config.Project == "" {
Expand All @@ -291,18 +290,18 @@ func createGCSBucketIfNecessary(ctx context.Context, gcsClient *storage.Client,
}

if terragruntOptions.FailIfBucketCreationRequired {
return BucketCreationNotAllowed(config.remoteStateConfigGCS.Bucket)
return BucketCreationNotAllowed(config.RemoteStateConfigGCS.Bucket)
}

prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.remoteStateConfigGCS.Bucket)
prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.RemoteStateConfigGCS.Bucket)
shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
if err != nil {
return err
}

if shouldCreateBucket {
// To avoid any eventual consistency issues with creating a GCS bucket we use a retry loop.
description := "Create GCS bucket " + config.remoteStateConfigGCS.Bucket
description := "Create GCS bucket " + config.RemoteStateConfigGCS.Bucket

return util.DoWithRetry(ctx, description, gcpMaxRetries, gcpSleepBetweenRetries, logrus.DebugLevel, func(ctx context.Context) error {
// TODO: Remove lint suppression
Expand Down Expand Up @@ -340,7 +339,7 @@ func CreateGCSBucketWithVersioning(gcsClient *storage.Client, config *ExtendedRe
return err
}

if err := WaitUntilGCSBucketExists(gcsClient, &config.remoteStateConfigGCS, terragruntOptions); err != nil {
if err := WaitUntilGCSBucketExists(gcsClient, &config.RemoteStateConfigGCS, terragruntOptions); err != nil {
return err
}

Expand All @@ -353,14 +352,14 @@ func CreateGCSBucketWithVersioning(gcsClient *storage.Client, config *ExtendedRe

func AddLabelsToGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
if config.GCSBucketLabels == nil || len(config.GCSBucketLabels) == 0 {
terragruntOptions.Logger.Debugf("No labels specified for bucket %s.", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("No labels specified for bucket %s.", config.RemoteStateConfigGCS.Bucket)
return nil
}

terragruntOptions.Logger.Debugf("Adding labels to GCS bucket with %s", config.GCSBucketLabels)

ctx := context.Background()
bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)
bucket := gcsClient.Bucket(config.RemoteStateConfigGCS.Bucket)

bucketAttrs := storage.BucketAttrsToUpdate{}

Expand All @@ -380,15 +379,15 @@ func AddLabelsToGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteState

// Create the GCS bucket specified in the given config
func CreateGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
terragruntOptions.Logger.Debugf("Creating GCS bucket %s in project %s", config.remoteStateConfigGCS.Bucket, config.Project)
terragruntOptions.Logger.Debugf("Creating GCS bucket %s in project %s", config.RemoteStateConfigGCS.Bucket, config.Project)

// The project ID to which the bucket belongs. This is only used when creating a new bucket during initialization.
// Since buckets have globally unique names, the project ID is not required to access the bucket during normal
// operation.
projectID := config.Project

ctx := context.Background()
bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)
bucket := gcsClient.Bucket(config.RemoteStateConfigGCS.Bucket)

bucketAttrs := &storage.BucketAttrs{}

Expand All @@ -398,19 +397,19 @@ func CreateGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfi
}

if config.SkipBucketVersioning {
terragruntOptions.Logger.Debugf("Versioning is disabled for the remote state GCS bucket %s using 'skip_bucket_versioning' config.", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("Versioning is disabled for the remote state GCS bucket %s using 'skip_bucket_versioning' config.", config.RemoteStateConfigGCS.Bucket)
} else {
terragruntOptions.Logger.Debugf("Enabling versioning on GCS bucket %s", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("Enabling versioning on GCS bucket %s", config.RemoteStateConfigGCS.Bucket)
bucketAttrs.VersioningEnabled = true
}

if config.EnableBucketPolicyOnly {
terragruntOptions.Logger.Debugf("Enabling uniform bucket-level access on GCS bucket %s", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("Enabling uniform bucket-level access on GCS bucket %s", config.RemoteStateConfigGCS.Bucket)
bucketAttrs.BucketPolicyOnly = storage.BucketPolicyOnly{Enabled: true}
}

err := bucket.Create(ctx, projectID, bucketAttrs)
return errors.WithStackTraceAndPrefix(err, "Error creating GCS bucket %s", config.remoteStateConfigGCS.Bucket)
return errors.WithStackTraceAndPrefix(err, "Error creating GCS bucket %s", config.RemoteStateConfigGCS.Bucket)
}

// GCP is eventually consistent, so after creating a GCS bucket, this method can be used to wait until the information
Expand Down

0 comments on commit e08bbb4

Please sign in to comment.