diff --git a/flow/connectors/external_metadata/store.go b/flow/connectors/external_metadata/store.go index 8693dabb58..f1b8c25079 100644 --- a/flow/connectors/external_metadata/store.go +++ b/flow/connectors/external_metadata/store.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/PeerDB-io/peer-flow/connectors/utils" + cc "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/jackc/pgx/v5/pgxpool" log "github.com/sirupsen/logrus" @@ -23,18 +24,30 @@ type PostgresMetadataStore struct { func NewPostgresMetadataStore(ctx context.Context, pgConfig *protos.PostgresConfig, schemaName string) (*PostgresMetadataStore, error) { - connectionString := utils.GetPGConnectionString(pgConfig) - pool, err := pgxpool.New(ctx, connectionString) - if err != nil { - log.Errorf("failed to create connection pool: %v", err) - return nil, err + var storePool *pgxpool.Pool + var poolErr error + if pgConfig == nil { + storePool, poolErr = cc.GetCatalogConnectionPoolFromEnv() + if poolErr != nil { + return nil, fmt.Errorf("failed to create catalog connection pool: %v", poolErr) + } + + log.Info("obtained catalog connection pool for metadata store") + } else { + connectionString := utils.GetPGConnectionString(pgConfig) + storePool, poolErr = pgxpool.New(ctx, connectionString) + if poolErr != nil { + log.Errorf("failed to create connection pool: %v", poolErr) + return nil, poolErr + } + + log.Info("created connection pool for metadata store") } - log.Info("created connection pool for metadata store") return &PostgresMetadataStore{ ctx: ctx, config: pgConfig, - pool: pool, + pool: storePool, schemaName: schemaName, }, nil } diff --git a/ui/app/peers/create/[peerType]/helpers/s3.ts b/ui/app/peers/create/[peerType]/helpers/s3.ts index caf93acc22..1cd0f38d61 100644 --- a/ui/app/peers/create/[peerType]/helpers/s3.ts +++ b/ui/app/peers/create/[peerType]/helpers/s3.ts @@ -45,6 +45,15 @@ export const s3Setting: PeerSetting[] = [ }, ]; +export const blankMetadata = { + host: '', + port: 5432, + user: 'postgres', + password: '', + database: 'postgres', + transactionSnapshot: '', +}; + export const blankS3Setting: S3Config = { url: 's3:///', accessKeyId: undefined, @@ -52,12 +61,7 @@ export const blankS3Setting: S3Config = { roleArn: undefined, region: undefined, endpoint: '', - metadataDb: { - host: '', - port: 5432, - user: 'postgres', - password: '', - database: 'postgres', - transactionSnapshot: '', - }, + // For Storage peers created in UI + // we use catalog as the metadata DB + metadataDb: blankMetadata, }; diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts index ecbe5b63c3..01a82386e2 100644 --- a/ui/app/peers/create/[peerType]/schema.ts +++ b/ui/app/peers/create/[peerType]/schema.ts @@ -234,5 +234,5 @@ export const s3Schema = z.object({ invalid_type_error: 'Endpoint must be a string', }) .optional(), - metadataDb: pgSchema, + metadataDb: pgSchema.optional(), }); diff --git a/ui/components/S3Form.tsx b/ui/components/S3Form.tsx index cbd78ff7ce..2a3f005671 100644 --- a/ui/components/S3Form.tsx +++ b/ui/components/S3Form.tsx @@ -9,6 +9,7 @@ import { PostgresConfig } from '@/grpc_generated/peers'; import { Label } from '@/lib/Label'; import { RowWithRadiobutton, RowWithTextField } from '@/lib/Layout'; import { RadioButton, RadioButtonGroup } from '@/lib/RadioButtonGroup'; +import { Switch } from '@/lib/Switch'; import { TextField } from '@/lib/TextField'; import { Tooltip } from '@/lib/Tooltip'; import { useEffect, useState } from 'react'; @@ -19,6 +20,7 @@ interface S3Props { setter: PeerSetter; } const S3ConfigForm = ({ setter }: S3Props) => { + const [showMetadata, setShowMetadata] = useState(false); const [metadataDB, setMetadataDB] = useState( blankS3Setting.metadataDb! ); @@ -34,7 +36,7 @@ const S3ConfigForm = ({ setter }: S3Props) => { setter((prev) => { return { ...prev, - metadataDb: metadataDB as PostgresConfig, + metadataDb: showMetadata ? (metadataDB as PostgresConfig) : undefined, endpoint, }; }); @@ -47,7 +49,7 @@ const S3ConfigForm = ({ setter }: S3Props) => { }; }); } - }, [metadataDB, storageType, setter]); + }, [metadataDB, storageType, setter, showMetadata]); return (
@@ -143,62 +145,65 @@ const S3ConfigForm = ({ setter }: S3Props) => { For S3/GCS storage peers, PeerDB uses an external PostgreSQL database to store metadata (last sync state) for mirrors.

- More information on creation of storage peers in PeerDB{' '} - - here. - + By default, PeerDB will use its internal Catalog as the metadata + database. +

+

+ You can also choose to use your own PostgreSQL database: - {postgresSetting.map( - (pgSetting, index) => - pgSetting.label !== 'Transaction Snapshot' && ( - - {pgSetting.label}{' '} - + + setShowMetadata(state)} /> +
+ {showMetadata && + postgresSetting.map( + (pgSetting, index) => + pgSetting.label !== 'Transaction Snapshot' && ( + + {pgSetting.label}{' '} + + + + + } + action={ +
- - - - } - action={ -
- ) => - pgSetting.stateHandler(e.target.value, setMetadataDB) - } - defaultValue={ - (metadataDB as PostgresConfig)[ - pgSetting.label.toLowerCase() as keyof PostgresConfig - ] || '' - } - /> - {pgSetting.tips && ( - ) => + pgSetting.stateHandler(e.target.value, setMetadataDB) + } + defaultValue={ + (metadataDB as PostgresConfig)[ + pgSetting.label.toLowerCase() as keyof PostgresConfig + ] || '' + } /> - )} -
- } - /> - ) - )} + {pgSetting.tips && ( + + )} +
+ } + /> + ) + )} ); };