Use catalog as external store (#680)

If metadataDB is set to empty string/undefined, we now use catalog as the metadata postgres instance for S3/GCS and Eventhub CDC <img width="1479" alt="Screenshot 2023-11-17 at 11 14 29 PM" src="https://github.com/PeerDB-io/peerdb/assets/65964360/4bed58cd-e472-48e7-8671-79481bcb7957">
PeerDB-io · Nov 17, 2023 · 8453899 · 8453899
1 parent 2b944a3
commit 8453899
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 71 deletions.
diff --git a/flow/connectors/external_metadata/store.go b/flow/connectors/external_metadata/store.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
+	cc "github.com/PeerDB-io/peer-flow/connectors/utils/catalog"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/jackc/pgx/v5/pgxpool"
 	log "github.com/sirupsen/logrus"
@@ -23,18 +24,30 @@ type PostgresMetadataStore struct {
 
 func NewPostgresMetadataStore(ctx context.Context, pgConfig *protos.PostgresConfig,
 	schemaName string) (*PostgresMetadataStore, error) {
-	connectionString := utils.GetPGConnectionString(pgConfig)
-	pool, err := pgxpool.New(ctx, connectionString)
-	if err != nil {
-		log.Errorf("failed to create connection pool: %v", err)
-		return nil, err
+	var storePool *pgxpool.Pool
+	var poolErr error
+	if pgConfig == nil {
+		storePool, poolErr = cc.GetCatalogConnectionPoolFromEnv()
+		if poolErr != nil {
+			return nil, fmt.Errorf("failed to create catalog connection pool: %v", poolErr)
+		}
+
+		log.Info("obtained catalog connection pool for metadata store")
+	} else {
+		connectionString := utils.GetPGConnectionString(pgConfig)
+		storePool, poolErr = pgxpool.New(ctx, connectionString)
+		if poolErr != nil {
+			log.Errorf("failed to create connection pool: %v", poolErr)
+			return nil, poolErr
+		}
+
+		log.Info("created connection pool for metadata store")
 	}
-	log.Info("created connection pool for metadata store")
 
 	return &PostgresMetadataStore{
 		ctx:        ctx,
 		config:     pgConfig,
-		pool:       pool,
+		pool:       storePool,
 		schemaName: schemaName,
 	}, nil
 }

diff --git a/ui/app/peers/create/[peerType]/helpers/s3.ts b/ui/app/peers/create/[peerType]/helpers/s3.ts
@@ -45,19 +45,23 @@ export const s3Setting: PeerSetting[] = [
   },
 ];
 
+export const blankMetadata = {
+  host: '',
+  port: 5432,
+  user: 'postgres',
+  password: '',
+  database: 'postgres',
+  transactionSnapshot: '',
+};
+
 export const blankS3Setting: S3Config = {
   url: 's3://<bucket_name>/<prefix_name>',
   accessKeyId: undefined,
   secretAccessKey: undefined,
   roleArn: undefined,
   region: undefined,
   endpoint: '',
-  metadataDb: {
-    host: '',
-    port: 5432,
-    user: 'postgres',
-    password: '',
-    database: 'postgres',
-    transactionSnapshot: '',
-  },
+  // For Storage peers created in UI
+  // we use catalog as the metadata DB
+  metadataDb: blankMetadata,
 };
diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts
@@ -234,5 +234,5 @@ export const s3Schema = z.object({
       invalid_type_error: 'Endpoint must be a string',
     })
     .optional(),
-  metadataDb: pgSchema,
+  metadataDb: pgSchema.optional(),
 });
diff --git a/ui/components/S3Form.tsx b/ui/components/S3Form.tsx
@@ -9,6 +9,7 @@ import { PostgresConfig } from '@/grpc_generated/peers';
 import { Label } from '@/lib/Label';
 import { RowWithRadiobutton, RowWithTextField } from '@/lib/Layout';
 import { RadioButton, RadioButtonGroup } from '@/lib/RadioButtonGroup';
+import { Switch } from '@/lib/Switch';
 import { TextField } from '@/lib/TextField';
 import { Tooltip } from '@/lib/Tooltip';
 import { useEffect, useState } from 'react';
@@ -19,6 +20,7 @@ interface S3Props {
   setter: PeerSetter;
 }
 const S3ConfigForm = ({ setter }: S3Props) => {
+  const [showMetadata, setShowMetadata] = useState<boolean>(false);
   const [metadataDB, setMetadataDB] = useState<PeerConfig>(
     blankS3Setting.metadataDb!
   );
@@ -34,7 +36,7 @@ const S3ConfigForm = ({ setter }: S3Props) => {
     setter((prev) => {
       return {
         ...prev,
-        metadataDb: metadataDB as PostgresConfig,
+        metadataDb: showMetadata ? (metadataDB as PostgresConfig) : undefined,
         endpoint,
       };
     });
@@ -47,7 +49,7 @@ const S3ConfigForm = ({ setter }: S3Props) => {
         };
       });
     }
-  }, [metadataDB, storageType, setter]);
+  }, [metadataDB, storageType, setter, showMetadata]);
 
   return (
     <div>
@@ -143,62 +145,65 @@ const S3ConfigForm = ({ setter }: S3Props) => {
         For S3/GCS storage peers, PeerDB uses an external PostgreSQL database to
         store metadata (last sync state) for mirrors.
         <br></br>
-        More information on creation of storage peers in PeerDB{' '}
-        <a
-          style={{ color: 'teal' }}
-          href='https://docs.peerdb.io/sql/commands/create-peer#storage-peers-s3-and-gcs'
-        >
-          here.
-        </a>
+        By default, PeerDB will use its internal Catalog as the metadata
+        database.
+        <br></br>
+        <br></br>
+        You can also choose to use your own PostgreSQL database:
       </Label>
-      {postgresSetting.map(
-        (pgSetting, index) =>
-          pgSetting.label !== 'Transaction Snapshot' && (
-            <RowWithTextField
-              key={index}
-              label={
-                <Label>
-                  {pgSetting.label}{' '}
-                  <Tooltip
-                    style={{ width: '100%' }}
-                    content={'This is a required field.'}
+      <div style={{ width: '50%', display: 'flex', alignItems: 'center' }}>
+        <Label variant='subheadline'>Use my own metadata detabase</Label>
+        <Switch onCheckedChange={(state) => setShowMetadata(state)} />
+      </div>
+      {showMetadata &&
+        postgresSetting.map(
+          (pgSetting, index) =>
+            pgSetting.label !== 'Transaction Snapshot' && (
+              <RowWithTextField
+                key={index}
+                label={
+                  <Label>
+                    {pgSetting.label}{' '}
+                    <Tooltip
+                      style={{ width: '100%' }}
+                      content={'This is a required field.'}
+                    >
+                      <Label colorName='lowContrast' colorSet='destructive'>
+                        *
+                      </Label>
+                    </Tooltip>
+                  </Label>
+                }
+                action={
+                  <div
+                    style={{
+                      display: 'flex',
+                      flexDirection: 'row',
+                      alignItems: 'center',
+                    }}
                   >
-                    <Label colorName='lowContrast' colorSet='destructive'>
-                      *
-                    </Label>
-                  </Tooltip>
-                </Label>
-              }
-              action={
-                <div
-                  style={{
-                    display: 'flex',
-                    flexDirection: 'row',
-                    alignItems: 'center',
-                  }}
-                >
-                  <TextField
-                    variant='simple'
-                    onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
-                      pgSetting.stateHandler(e.target.value, setMetadataDB)
-                    }
-                    defaultValue={
-                      (metadataDB as PostgresConfig)[
-                        pgSetting.label.toLowerCase() as keyof PostgresConfig
-                      ] || ''
-                    }
-                  />
-                  {pgSetting.tips && (
-                    <InfoPopover
-                      tips={pgSetting.tips}
-                      link={pgSetting.helpfulLink}
+                    <TextField
+                      variant='simple'
+                      onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
+                        pgSetting.stateHandler(e.target.value, setMetadataDB)
+                      }
+                      defaultValue={
+                        (metadataDB as PostgresConfig)[
+                          pgSetting.label.toLowerCase() as keyof PostgresConfig
+                        ] || ''
+                      }
                     />
-                  )}
-                </div>
-              }
-            />
-          )
-      )}
+                    {pgSetting.tips && (
+                      <InfoPopover
+                        tips={pgSetting.tips}
+                        link={pgSetting.helpfulLink}
+                      />
+                    )}
+                  </div>
+                }
+              />
+            )
+        )}
     </div>
   );
 };