From 02ca0988d0a6aaa71d2be9d4e47029f1216377a7 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 16 Nov 2023 13:32:17 +0530 Subject: [PATCH] s3 peer ui v0 --- flow/connectors/core.go | 8 +- ui/app/api/peers/route.ts | 6 + ui/app/mirrors/create/helpers/common.ts | 2 +- ui/app/mirrors/create/page.tsx | 1 - ui/app/peers/create/[peerType]/handlers.ts | 17 ++ .../peers/create/[peerType]/helpers/common.ts | 3 + ui/app/peers/create/[peerType]/helpers/s3.ts | 54 ++-- ui/app/peers/create/[peerType]/page.tsx | 14 +- ui/app/peers/create/[peerType]/schema.ts | 4 +- ui/components/S3Form.tsx | 260 +++++++++++------- 10 files changed, 243 insertions(+), 126 deletions(-) diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 0612514dd7..4c393ca48d 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -230,8 +230,12 @@ func GetConnector(ctx context.Context, peer *protos.Peer) (Connector, error) { return nil, fmt.Errorf("missing sqlserver config for %s peer %s", peer.Type.String(), peer.Name) } return connsqlserver.NewSQLServerConnector(ctx, sqlServerConfig) - // case protos.DBType_S3: - // return conns3.NewS3Connector(ctx, config.GetS3Config()) + case protos.DBType_S3: + s3Config := peer.GetS3Config() + if s3Config == nil { + return nil, fmt.Errorf("missing s3 config for %s peer %s", peer.Type.String(), peer.Name) + } + return conns3.NewS3Connector(ctx, s3Config) // case protos.DBType_EVENTHUB: // return connsqlserver.NewSQLServerConnector(ctx, config.GetSqlserverConfig()) default: diff --git a/ui/app/api/peers/route.ts b/ui/app/api/peers/route.ts index ff790da762..ca74ba0127 100644 --- a/ui/app/api/peers/route.ts +++ b/ui/app/api/peers/route.ts @@ -52,6 +52,12 @@ const constructPeer = ( type: DBType.BIGQUERY, bigqueryConfig: config as BigqueryConfig, }; + case 'S3': + return { + name, + type: DBType.S3, + s3Config: config as S3Config, + }; default: return; } diff --git a/ui/app/mirrors/create/helpers/common.ts b/ui/app/mirrors/create/helpers/common.ts index 86a8e04430..02f548eeba 100644 --- a/ui/app/mirrors/create/helpers/common.ts +++ b/ui/app/mirrors/create/helpers/common.ts @@ -59,7 +59,7 @@ export const blankQRepSetting = { waitBetweenBatchesSeconds: 30, writeMode: undefined, stagingPath: '', - numRowsPerPartition: 0, + numRowsPerPartition: 100000, setupWatermarkTableOnDestination: false, dstTableFullResync: false, }; diff --git a/ui/app/mirrors/create/page.tsx b/ui/app/mirrors/create/page.tsx index d2027f1dd8..b443cdd760 100644 --- a/ui/app/mirrors/create/page.tsx +++ b/ui/app/mirrors/create/page.tsx @@ -63,7 +63,6 @@ export default function CreateMirrors() { const [config, setConfig] = useState(blankCDCSetting); const [peers, setPeers] = useState([]); const [rows, setRows] = useState([]); - const [validSource, setValidSource] = useState(false); const [sourceSchema, setSourceSchema] = useState('public'); const [qrepQuery, setQrepQuery] = useState(`-- Here's a sample template: diff --git a/ui/app/peers/create/[peerType]/handlers.ts b/ui/app/peers/create/[peerType]/handlers.ts index ee215431c0..d4fcc72986 100644 --- a/ui/app/peers/create/[peerType]/handlers.ts +++ b/ui/app/peers/create/[peerType]/handlers.ts @@ -3,6 +3,7 @@ import { UCreatePeerResponse, UValidatePeerResponse, } from '@/app/dto/PeersDTO'; +import { S3Config } from '@/grpc_generated/peers'; import { Dispatch, SetStateAction } from 'react'; import { bqSchema, pgSchema, s3Schema, sfSchema } from './schema'; @@ -17,6 +18,15 @@ const validateFields = ( setMessage({ ok: false, msg: 'Peer name is required' }); return false; } + + if (type === 'S3') { + const s3Valid = S3Validation(config as S3Config); + if (s3Valid.length > 0) { + setMessage({ ok: false, msg: s3Valid }); + return false; + } + } + let validationErr: string | undefined; switch (type) { case 'POSTGRES': @@ -74,6 +84,13 @@ export const handleValidate = async ( setLoading(false); }; +const S3Validation = (config: S3Config): string => { + if (!config.secretAccessKey && !config.accessKeyId && !config.roleArn) { + return 'Either both access key and secret or role ARN is required'; + } + return ''; +}; + // API call to create peer export const handleCreate = async ( type: string, diff --git a/ui/app/peers/create/[peerType]/helpers/common.ts b/ui/app/peers/create/[peerType]/helpers/common.ts index c08b0035ee..62b2bb26c2 100644 --- a/ui/app/peers/create/[peerType]/helpers/common.ts +++ b/ui/app/peers/create/[peerType]/helpers/common.ts @@ -2,6 +2,7 @@ import { PeerConfig } from '@/app/dto/PeersDTO'; import { PeerSetter } from '@/components/ConfigForm'; import { blankBigquerySetting } from './bq'; import { blankPostgresSetting } from './pg'; +import { blankS3Setting } from './s3'; import { blankSnowflakeSetting } from './sf'; export interface PeerSetting { @@ -22,6 +23,8 @@ export const getBlankSetting = (dbType: string): PeerConfig => { return blankSnowflakeSetting; case 'BIGQUERY': return blankBigquerySetting; + case 'S3': + return blankS3Setting; default: return blankPostgresSetting; } diff --git a/ui/app/peers/create/[peerType]/helpers/s3.ts b/ui/app/peers/create/[peerType]/helpers/s3.ts index 6469fa6231..c085d3df74 100644 --- a/ui/app/peers/create/[peerType]/helpers/s3.ts +++ b/ui/app/peers/create/[peerType]/helpers/s3.ts @@ -6,54 +6,60 @@ export const s3Setting: PeerSetting[] = [ label: 'Bucket URL', stateHandler: (value, setter) => setter((curr) => ({ ...curr, url: value })), - tips: 'The URL of the S3/GCS bucket. It begins with s3://', + tips: 'The URL of your existing S3/GCS bucket along with a prefix of your choice. It begins with s3://', + helpfulLink: + 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html#accessing-a-bucket-using-S3-format', + default: 's3:///', }, { label: 'Access Key ID', stateHandler: (value, setter) => - setter((curr) => ({ ...curr, accessKeyID: parseInt(value, 10) })), - tips: 'Specifies the TCP/IP port or local Unix domain socket file extension on which postgres is listening for connections from client applications.', + setter((curr) => ({ ...curr, accessKeyId: value })), + optional: true, + tips: 'The AWS access key ID associated with your account. In case of GCS, this is the HMAC access key ID.', + helpfulLink: + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html', }, { label: 'Secret Access Key', stateHandler: (value, setter) => setter((curr) => ({ ...curr, secretAccessKey: value })), - tips: 'Specify the user that we should use to connect to this host.', - helpfulLink: 'https://www.postgresql.org/docs/8.0/user-manag.html', - }, - { - label: 'Role ARN', - stateHandler: (value, setter) => - setter((curr) => ({ ...curr, roleArn: value })), - type: 'password', - tips: 'Password associated with the user you provided.', - helpfulLink: 'https://www.postgresql.org/docs/current/auth-password.html', + tips: 'The AWS secret access key associated with your account. In case of GCS, this is the HMAC secret.', + helpfulLink: + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html', optional: true, }, { label: 'Region', stateHandler: (value, setter) => setter((curr) => ({ ...curr, region: value })), - tips: 'Specify which database to associate with this peer.', - helpfulLink: - 'https://www.postgresql.org/docs/current/sql-createdatabase.html', - optional: true, + tips: 'The region where your bucket is located. For example, us-east-1. In case of GCS, this will be set to auto, which detects where your bucket it.', }, { - label: 'Endpoint', + label: 'Role ARN', stateHandler: (value, setter) => - setter((curr) => ({ ...curr, endpoint: value })), + setter((curr) => ({ ...curr, roleArn: value })), + type: 'password', + tips: 'You may set this instead of the access key ID and secret.', + helpfulLink: + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-arns', optional: true, - tips: '', }, ]; -export const blankS3Config: S3Config = { - url: '', +export const blankS3Setting: S3Config = { + url: 's3:///', accessKeyId: undefined, secretAccessKey: undefined, roleArn: undefined, region: undefined, - endpoint: undefined, - metadataDb: undefined, + endpoint: '', + metadataDb: { + host: '', + port: 5432, + user: 'postgres', + password: '', + database: 'postgres', + transactionSnapshot: '', + }, }; diff --git a/ui/app/peers/create/[peerType]/page.tsx b/ui/app/peers/create/[peerType]/page.tsx index 9f86c9d80e..cbefbfad8b 100644 --- a/ui/app/peers/create/[peerType]/page.tsx +++ b/ui/app/peers/create/[peerType]/page.tsx @@ -5,7 +5,7 @@ import S3ConfigForm from '@/components/S3Form'; import { Button } from '@/lib/Button'; import { ButtonGroup } from '@/lib/ButtonGroup'; import { Label } from '@/lib/Label'; -import { LayoutMain, RowWithTextField } from '@/lib/Layout'; +import { RowWithTextField } from '@/lib/Layout'; import { Panel } from '@/lib/Panel'; import { TextField } from '@/lib/TextField'; import { Tooltip } from '@/lib/Tooltip'; @@ -58,7 +58,15 @@ export default function CreateConfig({ }; return ( - +
- +
); } diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts index 07ea3fa43d..466484b850 100644 --- a/ui/app/peers/create/[peerType]/schema.ts +++ b/ui/app/peers/create/[peerType]/schema.ts @@ -193,8 +193,8 @@ export const s3Schema = z.object({ required_error: 'URL is required', }) .min(1, { message: 'URL must be non-empty' }) - .refine((url) => url.startsWith('s3:/'), { - message: 'URL must start with s3:/', + .refine((url) => url.startsWith('s3://'), { + message: 'URL must start with s3://', }), accessKeyId: z .string({ diff --git a/ui/components/S3Form.tsx b/ui/components/S3Form.tsx index 4ca83214e5..cbd78ff7ce 100644 --- a/ui/components/S3Form.tsx +++ b/ui/components/S3Form.tsx @@ -1,13 +1,14 @@ 'use client'; import { PeerConfig } from '@/app/dto/PeersDTO'; +import { postgresSetting } from '@/app/peers/create/[peerType]/helpers/pg'; import { - blankPostgresSetting, - postgresSetting, -} from '@/app/peers/create/[peerType]/helpers/pg'; -import { s3Setting } from '@/app/peers/create/[peerType]/helpers/s3'; + blankS3Setting, + s3Setting, +} from '@/app/peers/create/[peerType]/helpers/s3'; import { PostgresConfig } from '@/grpc_generated/peers'; import { Label } from '@/lib/Label'; -import { RowWithTextField } from '@/lib/Layout'; +import { RowWithRadiobutton, RowWithTextField } from '@/lib/Layout'; +import { RadioButton, RadioButtonGroup } from '@/lib/RadioButtonGroup'; import { TextField } from '@/lib/TextField'; import { Tooltip } from '@/lib/Tooltip'; import { useEffect, useState } from 'react'; @@ -18,65 +19,116 @@ interface S3Props { setter: PeerSetter; } const S3ConfigForm = ({ setter }: S3Props) => { - const [metadataDB, setMetadataDB] = - useState(blankPostgresSetting); + const [metadataDB, setMetadataDB] = useState( + blankS3Setting.metadataDb! + ); + const [storageType, setStorageType] = useState<'S3' | 'GCS'>('S3'); + const displayCondition = (label: string) => { + return !( + (label === 'Region' || label === 'Role ARN') && + storageType === 'GCS' + ); + }; useEffect(() => { + const endpoint = storageType === 'S3' ? '' : 'storage.googleapis.com'; setter((prev) => { return { ...prev, metadataDb: metadataDB as PostgresConfig, + endpoint, }; }); - }, [metadataDB]); + + if (storageType === 'GCS') { + setter((prev) => { + return { + ...prev, + region: 'auto', + }; + }); + } + }, [metadataDB, storageType, setter]); + return ( - <> +
+ + setStorageType(val as 'S3' | 'GCS')} + > + S3} + action={} + /> + GCS} + action={} + /> + {s3Setting.map((setting, index) => { - return ( - - {setting.label}{' '} - {!setting.optional && ( - - - - )} - - } - action={ -
- ) => - setting.stateHandler(e.target.value, setter) - } - /> - {setting.tips && ( - - )} -
- } - /> - ); + if (displayCondition(setting.label)) + return ( + + {setting.label}{' '} + {!setting.optional && ( + + + + )} + + } + action={ +
+ ) => + setting.stateHandler(e.target.value, setter) + } + /> + {setting.tips && ( + + )} +
+ } + /> + ); })} - - {postgresSetting.map((setting, index) => { - - {setting.label}{' '} - - - - - } - action={ -
- ) => - setting.stateHandler(e.target.value, setMetadataDB) - } - /> - {setting.tips && ( - - )} -
- } - />; - })} - + } + action={ +
+ ) => + pgSetting.stateHandler(e.target.value, setMetadataDB) + } + defaultValue={ + (metadataDB as PostgresConfig)[ + pgSetting.label.toLowerCase() as keyof PostgresConfig + ] || '' + } + /> + {pgSetting.tips && ( + + )} +
+ } + /> + ) + )} +
); };