diff --git a/flow/cmd/handler.go b/flow/cmd/handler.go index 416800f02f..13ce8115b1 100644 --- a/flow/cmd/handler.go +++ b/flow/cmd/handler.go @@ -559,7 +559,13 @@ func (h *FlowRequestHandler) CreatePeer( } sqlServerConfig := sqlServerConfigObject.SqlserverConfig encodedConfig, encodingErr = proto.Marshal(sqlServerConfig) - + case protos.DBType_S3: + s3ConfigObject, ok := config.(*protos.Peer_S3Config) + if !ok { + return wrongConfigResponse, nil + } + s3Config := s3ConfigObject.S3Config + encodedConfig, encodingErr = proto.Marshal(s3Config) default: return wrongConfigResponse, nil } diff --git a/ui/app/dto/PeersDTO.ts b/ui/app/dto/PeersDTO.ts index 9954aceaa7..0e6dfef818 100644 --- a/ui/app/dto/PeersDTO.ts +++ b/ui/app/dto/PeersDTO.ts @@ -1,6 +1,7 @@ import { BigqueryConfig, PostgresConfig, + S3Config, SnowflakeConfig, } from '@/grpc_generated/peers'; @@ -31,7 +32,11 @@ export type UDropPeerResponse = { errorMessage: string; }; -export type PeerConfig = PostgresConfig | SnowflakeConfig | BigqueryConfig; +export type PeerConfig = + | PostgresConfig + | SnowflakeConfig + | BigqueryConfig + | S3Config; export type CatalogPeer = { id: number; name: string; diff --git a/ui/app/peers/create/[peerType]/handlers.ts b/ui/app/peers/create/[peerType]/handlers.ts index f51e44aff7..ee215431c0 100644 --- a/ui/app/peers/create/[peerType]/handlers.ts +++ b/ui/app/peers/create/[peerType]/handlers.ts @@ -4,7 +4,7 @@ import { UValidatePeerResponse, } from '@/app/dto/PeersDTO'; import { Dispatch, SetStateAction } from 'react'; -import { bqSchema, pgSchema, sfSchema } from './schema'; +import { bqSchema, pgSchema, s3Schema, sfSchema } from './schema'; // Frontend form validation const validateFields = ( @@ -31,6 +31,10 @@ const validateFields = ( const bqConfig = bqSchema.safeParse(config); if (!bqConfig.success) validationErr = bqConfig.error.issues[0].message; break; + case 'S3': + const s3Config = s3Schema.safeParse(config); + if (!s3Config.success) validationErr = s3Config.error.issues[0].message; + break; default: validationErr = 'Unsupported peer type ' + type; } diff --git a/ui/app/peers/create/[peerType]/helpers/s3.ts b/ui/app/peers/create/[peerType]/helpers/s3.ts new file mode 100644 index 0000000000..6469fa6231 --- /dev/null +++ b/ui/app/peers/create/[peerType]/helpers/s3.ts @@ -0,0 +1,59 @@ +import { S3Config } from '@/grpc_generated/peers'; +import { PeerSetting } from './common'; + +export const s3Setting: PeerSetting[] = [ + { + label: 'Bucket URL', + stateHandler: (value, setter) => + setter((curr) => ({ ...curr, url: value })), + tips: 'The URL of the S3/GCS bucket. It begins with s3://', + }, + { + label: 'Access Key ID', + stateHandler: (value, setter) => + setter((curr) => ({ ...curr, accessKeyID: parseInt(value, 10) })), + tips: 'Specifies the TCP/IP port or local Unix domain socket file extension on which postgres is listening for connections from client applications.', + }, + { + label: 'Secret Access Key', + stateHandler: (value, setter) => + setter((curr) => ({ ...curr, secretAccessKey: value })), + tips: 'Specify the user that we should use to connect to this host.', + helpfulLink: 'https://www.postgresql.org/docs/8.0/user-manag.html', + }, + { + label: 'Role ARN', + stateHandler: (value, setter) => + setter((curr) => ({ ...curr, roleArn: value })), + type: 'password', + tips: 'Password associated with the user you provided.', + helpfulLink: 'https://www.postgresql.org/docs/current/auth-password.html', + optional: true, + }, + { + label: 'Region', + stateHandler: (value, setter) => + setter((curr) => ({ ...curr, region: value })), + tips: 'Specify which database to associate with this peer.', + helpfulLink: + 'https://www.postgresql.org/docs/current/sql-createdatabase.html', + optional: true, + }, + { + label: 'Endpoint', + stateHandler: (value, setter) => + setter((curr) => ({ ...curr, endpoint: value })), + optional: true, + tips: '', + }, +]; + +export const blankS3Config: S3Config = { + url: '', + accessKeyId: undefined, + secretAccessKey: undefined, + roleArn: undefined, + region: undefined, + endpoint: undefined, + metadataDb: undefined, +}; diff --git a/ui/app/peers/create/[peerType]/page.tsx b/ui/app/peers/create/[peerType]/page.tsx index 6bceac8b18..9f86c9d80e 100644 --- a/ui/app/peers/create/[peerType]/page.tsx +++ b/ui/app/peers/create/[peerType]/page.tsx @@ -1,6 +1,7 @@ 'use client'; import { PeerConfig } from '@/app/dto/PeersDTO'; import BQConfig from '@/components/BigqueryConfig'; +import S3ConfigForm from '@/components/S3Form'; import { Button } from '@/lib/Button'; import { ButtonGroup } from '@/lib/ButtonGroup'; import { Label } from '@/lib/Label'; @@ -45,6 +46,8 @@ export default function CreateConfig({ return configForm(snowflakeSetting); case 'BIGQUERY': return ; + case 'S3': + return ; default: return <>; } diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts index 69da7b1c1b..07ea3fa43d 100644 --- a/ui/app/peers/create/[peerType]/schema.ts +++ b/ui/app/peers/create/[peerType]/schema.ts @@ -185,3 +185,43 @@ export const bqSchema = z.object({ 'Dataset ID must only contain numbers, letters, and underscores' ), }); + +export const s3Schema = z.object({ + url: z + .string({ + invalid_type_error: 'URL must be a string', + required_error: 'URL is required', + }) + .min(1, { message: 'URL must be non-empty' }) + .refine((url) => url.startsWith('s3:/'), { + message: 'URL must start with s3:/', + }), + accessKeyId: z + .string({ + invalid_type_error: 'Access Key ID must be a string', + required_error: 'Access Key ID is required', + }) + .min(1, { message: 'Access Key ID must be non-empty' }), + secretAccessKey: z + .string({ + invalid_type_error: 'Secret Access Key must be a string', + required_error: 'Secret Access Key is required', + }) + .min(1, { message: 'Secret Access Key must be non-empty' }), + roleArn: z + .string({ + invalid_type_error: 'Role ARN must be a string', + }) + .optional(), + region: z + .string({ + invalid_type_error: 'Region must be a string', + }) + .optional(), + endpoint: z + .string({ + invalid_type_error: 'Endpoint must be a string', + }) + .optional(), + metadataDb: pgSchema, +}); diff --git a/ui/components/PeerComponent.tsx b/ui/components/PeerComponent.tsx index ab956992e2..7378e91635 100644 --- a/ui/components/PeerComponent.tsx +++ b/ui/components/PeerComponent.tsx @@ -15,6 +15,9 @@ export const DBTypeToImageMapping = (peerType: DBType | string) => { case DBType.BIGQUERY: case 'BIGQUERY': return '/svgs/bq.svg'; + case DBType.S3: + case 'S3': + return '/svgs/aws.svg'; case DBType.EVENTHUB_GROUP: case DBType.EVENTHUB: return '/svgs/ms.svg'; diff --git a/ui/components/S3Form.tsx b/ui/components/S3Form.tsx new file mode 100644 index 0000000000..4ca83214e5 --- /dev/null +++ b/ui/components/S3Form.tsx @@ -0,0 +1,132 @@ +'use client'; +import { PeerConfig } from '@/app/dto/PeersDTO'; +import { + blankPostgresSetting, + postgresSetting, +} from '@/app/peers/create/[peerType]/helpers/pg'; +import { s3Setting } from '@/app/peers/create/[peerType]/helpers/s3'; +import { PostgresConfig } from '@/grpc_generated/peers'; +import { Label } from '@/lib/Label'; +import { RowWithTextField } from '@/lib/Layout'; +import { TextField } from '@/lib/TextField'; +import { Tooltip } from '@/lib/Tooltip'; +import { useEffect, useState } from 'react'; +import { PeerSetter } from './ConfigForm'; +import { InfoPopover } from './InfoPopover'; + +interface S3Props { + setter: PeerSetter; +} +const S3ConfigForm = ({ setter }: S3Props) => { + const [metadataDB, setMetadataDB] = + useState(blankPostgresSetting); + useEffect(() => { + setter((prev) => { + return { + ...prev, + metadataDb: metadataDB as PostgresConfig, + }; + }); + }, [metadataDB]); + return ( + <> + {s3Setting.map((setting, index) => { + return ( + + {setting.label}{' '} + {!setting.optional && ( + + + + )} + + } + action={ +
+ ) => + setting.stateHandler(e.target.value, setter) + } + /> + {setting.tips && ( + + )} +
+ } + /> + ); + })} + + + + {postgresSetting.map((setting, index) => { + + {setting.label}{' '} + + + + + } + action={ +
+ ) => + setting.stateHandler(e.target.value, setMetadataDB) + } + /> + {setting.tips && ( + + )} +
+ } + />; + })} + + ); +}; + +export default S3ConfigForm; diff --git a/ui/components/SelectSource.tsx b/ui/components/SelectSource.tsx index 695174a2fe..bc911f8fa5 100644 --- a/ui/components/SelectSource.tsx +++ b/ui/components/SelectSource.tsx @@ -28,7 +28,10 @@ export default function SelectSource({ .filter( (value): value is string => typeof value === 'string' && - (value === 'POSTGRES' || value === 'SNOWFLAKE' || value === 'BIGQUERY') + (value === 'POSTGRES' || + value === 'SNOWFLAKE' || + value === 'BIGQUERY' || + value === 'S3') ) .map((value) => ({ label: value, value })); @@ -37,7 +40,7 @@ export default function SelectSource({ placeholder='Select a source' options={dbTypes} defaultValue={dbTypes.find((opt) => opt.value === peerType)} - onChange={(val, action) => val && setPeerType(val.value)} + onChange={(val, _) => val && setPeerType(val.value)} formatOptionLabel={SourceLabel} /> );