NEOS-1208 add gcp cloud storage to connections backend (#2206)

nucleuscloud · Jun 25, 2024 · 9b3a671 · 9b3a671
1 parent 6ed9cef
commit 9b3a671
Show file tree

Hide file tree

Showing 6 changed files with 810 additions and 341 deletions.
diff --git a/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.go b/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.go
diff --git a/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.validate.go b/backend/gen/go/protos/mgmt/v1alpha1/connection.pb.validate.go
diff --git a/backend/protos/mgmt/v1alpha1/connection.proto b/backend/protos/mgmt/v1alpha1/connection.proto
@@ -82,19 +82,25 @@ message Connection {
   string account_id = 8;
 }
 
+// Configuration for all of the supported Neosync connection types
 message ConnectionConfig {
   oneof config {
     option (buf.validate.oneof).required = true;
 
+    // Configures a PostgreSQL connection
     PostgresConnectionConfig pg_config = 1;
+    // Configures an AWS S3 Connection
     AwsS3ConnectionConfig aws_s3_config = 2;
+    // Configures a MySQL connection
     MysqlConnectionConfig mysql_config = 3;
     // Configures a connection to a directory available on the local file system
     LocalDirectoryConnectionConfig local_dir_config = 4;
     // Connection config for an OpenAI (or compatible) Connection
     OpenAiConnectionConfig openai_config = 5;
     // Configures a MongoDB Connection
     MongoConnectionConfig mongo_config = 6;
+    // Configures a GCP Cloud Storage Connection
+    GcpCloudStorageConnectionConfig gcp_cloudstorage_config = 7;
   }
 }
 
@@ -252,6 +258,17 @@ message AwsS3Credentials {
   optional string role_external_id = 7;
 }
 
+// Configuration for GCP Cloud Storage Buckets
+message GcpCloudStorageConnectionConfig {
+  // The GCP Cloud Storage bucket that will be accessed.
+  string bucket = 1 [(buf.validate.field).string.min_len = 1];
+  // The path prefix that will be appended to each file
+  optional string path_prefix = 2;
+
+  // stringified json of the service account credentials file
+  optional string service_account_credentials = 3;
+}
+
 message IsConnectionNameAvailableRequest {
   string account_id = 1 [(buf.validate.field).string.uuid = true];
   string connection_name = 2 [(buf.validate.field).string.pattern = "^[a-z0-9-]{3,30}$"];

diff --git a/backend/sql/postgresql/models/models.go b/backend/sql/postgresql/models/models.go
@@ -9,12 +9,13 @@ import (
 )
 
 type ConnectionConfig struct {
-	PgConfig             *PostgresConnectionConfig       `json:"pgConfig,omitempty"`
-	AwsS3Config          *AwsS3ConnectionConfig          `json:"awsS3Config,omitempty"`
-	MysqlConfig          *MysqlConnectionConfig          `json:"mysqlConfig,omitempty"`
-	LocalDirectoryConfig *LocalDirectoryConnectionConfig `json:"localDirConfig,omitempty"`
-	OpenAiConfig         *OpenAiConnectionConfig         `json:"openaiConfig,omitempty"`
-	MongoConfig          *MongoConnectionConfig          `json:"mongoConfig,omitempty"`
+	PgConfig              *PostgresConnectionConfig       `json:"pgConfig,omitempty"`
+	AwsS3Config           *AwsS3ConnectionConfig          `json:"awsS3Config,omitempty"`
+	MysqlConfig           *MysqlConnectionConfig          `json:"mysqlConfig,omitempty"`
+	LocalDirectoryConfig  *LocalDirectoryConnectionConfig `json:"localDirConfig,omitempty"`
+	OpenAiConfig          *OpenAiConnectionConfig         `json:"openaiConfig,omitempty"`
+	MongoConfig           *MongoConnectionConfig          `json:"mongoConfig,omitempty"`
+	GcpCloudStorageConfig *GcpCloudStorageConfig          `json:"gcpCloudStorageConfig,omitempty"`
 }
 
 func (c *ConnectionConfig) ToDto() (*mgmtv1alpha1.ConnectionConfig, error) {
@@ -133,6 +134,16 @@ func (c *ConnectionConfig) ToDto() (*mgmtv1alpha1.ConnectionConfig, error) {
 				MongoConfig: mdto,
 			},
 		}, nil
+	} else if c.GcpCloudStorageConfig != nil {
+		gdto, err := c.GcpCloudStorageConfig.ToDto()
+		if err != nil {
+			return nil, err
+		}
+		return &mgmtv1alpha1.ConnectionConfig{
+			Config: &mgmtv1alpha1.ConnectionConfig_GcpCloudstorageConfig{
+				GcpCloudstorageConfig: gdto,
+			},
+		}, nil
 	}
 	return nil, errors.ErrUnsupported
 }
@@ -214,6 +225,12 @@ func (c *ConnectionConfig) FromDto(dto *mgmtv1alpha1.ConnectionConfig) error {
 		if err != nil {
 			return err
 		}
+	case *mgmtv1alpha1.ConnectionConfig_GcpCloudstorageConfig:
+		c.GcpCloudStorageConfig = &GcpCloudStorageConfig{}
+		err := c.GcpCloudStorageConfig.FromDto(config.GcpCloudstorageConfig)
+		if err != nil {
+			return err
+		}
 	default:
 		return fmt.Errorf("unable to convert to ConnectionConfig from DTO ConnectionConfig, type not supported: %T", config)
 	}
@@ -266,6 +283,30 @@ func (m *MongoConnectionConfig) FromDto(dto *mgmtv1alpha1.MongoConnectionConfig)
 	return nil
 }
 
+type GcpCloudStorageConfig struct {
+	Bucket     string  `json:"bucket"`
+	PathPrefix *string `json:"pathPrefix,omitempty"`
+
+	ServiceAccountCredentials *string `json:"serviceAccountCredentials,omitempty"`
+}
+
+func (g *GcpCloudStorageConfig) ToDto() (*mgmtv1alpha1.GcpCloudStorageConnectionConfig, error) {
+	return &mgmtv1alpha1.GcpCloudStorageConnectionConfig{
+		Bucket:                    g.Bucket,
+		PathPrefix:                g.PathPrefix,
+		ServiceAccountCredentials: g.ServiceAccountCredentials,
+	}, nil
+}
+func (g *GcpCloudStorageConfig) FromDto(dto *mgmtv1alpha1.GcpCloudStorageConnectionConfig) error {
+	if dto == nil {
+		return errors.New("dto was nil, expected *mgmtv1alpha1.GcpCloudStorageConnectionConfig")
+	}
+	g.Bucket = dto.Bucket
+	g.PathPrefix = dto.PathPrefix
+	g.ServiceAccountCredentials = dto.ServiceAccountCredentials
+	return nil
+}
+
 type PostgresConnectionConfig struct {
 	Connection        *PostgresConnection `json:"connection,omitempty"`
 	Url               *string             `json:"url,omitempty"`