From 9851dd85342d49048d7ca1d5d8e960578f3ea348 Mon Sep 17 00:00:00 2001
From: Amogh-Bharadwaj <amogh@peerdb.io>
Date: Wed, 11 Oct 2023 15:15:29 +0530
Subject: [PATCH] sets up interopability

---
 docker-compose.yml           | 12 +++++++++---
 flow/connectors/utils/aws.go | 10 ++++++++--
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index a950a51c2b..6da2cadfb2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,9 +9,15 @@ x-catalog-config: &catalog-config
 
 x-flow-worker-env: &flow-worker-env
   TEMPORAL_HOST_PORT: temporal:7233
-  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
-  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
-  AWS_REGION: ${AWS_REGION:-""}
+  # For GCS, these will be your HMAC keys instead
+  # For more information: 
+  # https://cloud.google.com/storage/docs/authentication/managing-hmackeys
+  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
+  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
+  # For GCS, set this to "auto" without the quotes
+  AWS_REGION: ${AWS_REGION:-}
+  # For GCS, set this as: https://storage.googleapis.com
+  AWS_ENDPOINT: ${AWS_ENDPOINT:-}
   # enables worker profiling using Go's pprof
   ENABLE_PROFILING: "true"
   # enables exporting of mirror metrics to Prometheus for visualization using Grafana
diff --git a/flow/connectors/utils/aws.go b/flow/connectors/utils/aws.go
index e936ea6192..af2d75f36f 100644
--- a/flow/connectors/utils/aws.go
+++ b/flow/connectors/utils/aws.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 )
@@ -15,6 +16,7 @@ type AWSSecrets struct {
 	SecretAccessKey string
 	AwsRoleArn      string
 	Region          string
+	Endpoint        string
 }
 
 func GetAWSSecrets() (*AWSSecrets, error) {
@@ -22,7 +24,7 @@ func GetAWSSecrets() (*AWSSecrets, error) {
 	if awsRegion == "" {
 		return nil, fmt.Errorf("AWS_REGION must be set")
 	}
-
+	awsEndpoint := os.Getenv("AWS_ENDPOINT")
 	awsKey := os.Getenv("AWS_ACCESS_KEY_ID")
 	awsSecret := os.Getenv("AWS_SECRET_ACCESS_KEY")
 	awsRoleArn := os.Getenv("AWS_ROLE_ARN")
@@ -37,6 +39,7 @@ func GetAWSSecrets() (*AWSSecrets, error) {
 		SecretAccessKey: awsSecret,
 		AwsRoleArn:      awsRoleArn,
 		Region:          awsRegion,
+		Endpoint:        awsEndpoint,
 	}, nil
 }
 
@@ -73,7 +76,10 @@ func CreateS3Client() (*s3.S3, error) {
 	}
 
 	sess := session.Must(session.NewSession(&aws.Config{
-		Region: aws.String(awsSecrets.Region),
+		Region:   aws.String(awsSecrets.Region),
+		Endpoint: aws.String(awsSecrets.Endpoint),
+		Credentials: credentials.NewStaticCredentials(
+			awsSecrets.AccessKeyID, awsSecrets.SecretAccessKey, ""),
 	}))
 
 	s3svc := s3.New(sess)