sets up interopability

docker-compose.yml

@@ -9,9 +9,15 @@ x-catalog-config: &catalog-config
 
 x-flow-worker-env: &flow-worker-env
   TEMPORAL_HOST_PORT: temporal:7233
-  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
-  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
-  AWS_REGION: ${AWS_REGION:-""}
+  # For GCS, these will be your HMAC keys instead
+  # For more information: 
+  # https://cloud.google.com/storage/docs/authentication/managing-hmackeys
+  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
+  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
+  # For GCS, set this to "auto" without the quotes
+  AWS_REGION: ${AWS_REGION:-}
+  # For GCS, set this as: https://storage.googleapis.com
+  AWS_ENDPOINT: ${AWS_ENDPOINT:-}
   # enables worker profiling using Go's pprof
   ENABLE_PROFILING: "true"
   # enables exporting of mirror metrics to Prometheus for visualization using Grafana

flow/connectors/utils/aws.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 )
@@ -15,14 +16,15 @@ type AWSSecrets struct {
 	SecretAccessKey string
 	AwsRoleArn      string
 	Region          string
+	Endpoint        string
 }
 
 func GetAWSSecrets() (*AWSSecrets, error) {
 	awsRegion := os.Getenv("AWS_REGION")
 	if awsRegion == "" {
 		return nil, fmt.Errorf("AWS_REGION must be set")
 	}
-
+	awsEndpoint := os.Getenv("AWS_ENDPOINT")
 	awsKey := os.Getenv("AWS_ACCESS_KEY_ID")
 	awsSecret := os.Getenv("AWS_SECRET_ACCESS_KEY")
 	awsRoleArn := os.Getenv("AWS_ROLE_ARN")
@@ -37,6 +39,7 @@ func GetAWSSecrets() (*AWSSecrets, error) {
 		SecretAccessKey: awsSecret,
 		AwsRoleArn:      awsRoleArn,
 		Region:          awsRegion,
+		Endpoint:        awsEndpoint,
 	}, nil
 }
 
@@ -73,7 +76,10 @@ func CreateS3Client() (*s3.S3, error) {
 	}
 
 	sess := session.Must(session.NewSession(&aws.Config{
-		Region: aws.String(awsSecrets.Region),
+		Region:   aws.String(awsSecrets.Region),
+		Endpoint: aws.String(awsSecrets.Endpoint),
+		Credentials: credentials.NewStaticCredentials(
+			awsSecrets.AccessKeyID, awsSecrets.SecretAccessKey, ""),
 	}))
 
 	s3svc := s3.New(sess)