peerdb takes care of external metadata store

PeerDB-io · Nov 17, 2023 · f50aaa0 · f50aaa0
1 parent 66566ce
commit f50aaa0
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 106 deletions.
diff --git a/flow/connectors/external_metadata/store.go b/flow/connectors/external_metadata/store.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
+	cc "github.com/PeerDB-io/peer-flow/connectors/utils/catalog"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/jackc/pgx/v5/pgxpool"
 	log "github.com/sirupsen/logrus"
@@ -23,18 +24,30 @@ type PostgresMetadataStore struct {
 
 func NewPostgresMetadataStore(ctx context.Context, pgConfig *protos.PostgresConfig,
 	schemaName string) (*PostgresMetadataStore, error) {
-	connectionString := utils.GetPGConnectionString(pgConfig)
-	pool, err := pgxpool.New(ctx, connectionString)
-	if err != nil {
-		log.Errorf("failed to create connection pool: %v", err)
-		return nil, err
+	var storePool *pgxpool.Pool
+	var poolErr error
+	if pgConfig == nil {
+		storePool, poolErr = cc.GetCatalogConnectionPoolFromEnv()
+		if poolErr != nil {
+			return nil, fmt.Errorf("failed to create catalog connection pool: %v", poolErr)
+		}
+
+		log.Info("obtained catalog connection pool for metadata store")
+	} else {
+		connectionString := utils.GetPGConnectionString(pgConfig)
+		storePool, poolErr = pgxpool.New(ctx, connectionString)
+		if poolErr != nil {
+			log.Errorf("failed to create connection pool: %v", poolErr)
+			return nil, poolErr
+		}
+
+		log.Info("created connection pool for metadata store")
 	}
-	log.Info("created connection pool for metadata store")
 
 	return &PostgresMetadataStore{
 		ctx:        ctx,
 		config:     pgConfig,
-		pool:       pool,
+		pool:       storePool,
 		schemaName: schemaName,
 	}, nil
 }

diff --git a/ui/app/peers/create/[peerType]/helpers/s3.ts b/ui/app/peers/create/[peerType]/helpers/s3.ts
@@ -52,12 +52,5 @@ export const blankS3Setting: S3Config = {
   roleArn: undefined,
   region: undefined,
   endpoint: '',
-  metadataDb: {
-    host: '',
-    port: 5432,
-    user: 'postgres',
-    password: '',
-    database: 'postgres',
-    transactionSnapshot: '',
-  },
+  metadataDb: undefined,
 };
diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts
@@ -234,5 +234,4 @@ export const s3Schema = z.object({
       invalid_type_error: 'Endpoint must be a string',
     })
     .optional(),
-  metadataDb: pgSchema,
 });
diff --git a/ui/components/S3Form.tsx b/ui/components/S3Form.tsx
@@ -1,11 +1,5 @@
 'use client';
-import { PeerConfig } from '@/app/dto/PeersDTO';
-import { postgresSetting } from '@/app/peers/create/[peerType]/helpers/pg';
-import {
-  blankS3Setting,
-  s3Setting,
-} from '@/app/peers/create/[peerType]/helpers/s3';
-import { PostgresConfig } from '@/grpc_generated/peers';
+import { s3Setting } from '@/app/peers/create/[peerType]/helpers/s3';
 import { Label } from '@/lib/Label';
 import { RowWithRadiobutton, RowWithTextField } from '@/lib/Layout';
 import { RadioButton, RadioButtonGroup } from '@/lib/RadioButtonGroup';
@@ -19,9 +13,6 @@ interface S3Props {
   setter: PeerSetter;
 }
 const S3ConfigForm = ({ setter }: S3Props) => {
-  const [metadataDB, setMetadataDB] = useState<PeerConfig>(
-    blankS3Setting.metadataDb!
-  );
   const [storageType, setStorageType] = useState<'S3' | 'GCS'>('S3');
   const displayCondition = (label: string) => {
     return !(
@@ -31,23 +22,15 @@ const S3ConfigForm = ({ setter }: S3Props) => {
   };
   useEffect(() => {
     const endpoint = storageType === 'S3' ? '' : 'storage.googleapis.com';
+    const region = storageType === 'S3' ? '' : 'auto';
     setter((prev) => {
       return {
         ...prev,
-        metadataDb: metadataDB as PostgresConfig,
         endpoint,
+        region,
       };
     });
-
-    if (storageType === 'GCS') {
-      setter((prev) => {
-        return {
-          ...prev,
-          region: 'auto',
-        };
-      });
-    }
-  }, [metadataDB, storageType, setter]);
+  }, [storageType, setter]);
 
   return (
     <div>
@@ -130,75 +113,6 @@ const S3ConfigForm = ({ setter }: S3Props) => {
             />
           );
       })}
-
-      <Label
-        as='label'
-        style={{ marginTop: '1rem' }}
-        variant='subheadline'
-        colorName='lowContrast'
-      >
-        Metadata Database
-      </Label>
-      <Label>
-        For S3/GCS storage peers, PeerDB uses an external PostgreSQL database to
-        store metadata (last sync state) for mirrors.
-        <br></br>
-        More information on creation of storage peers in PeerDB{' '}
-        <a
-          style={{ color: 'teal' }}
-          href='https://docs.peerdb.io/sql/commands/create-peer#storage-peers-s3-and-gcs'
-        >
-          here.
-        </a>
-      </Label>
-      {postgresSetting.map(
-        (pgSetting, index) =>
-          pgSetting.label !== 'Transaction Snapshot' && (
-            <RowWithTextField
-              key={index}
-              label={
-                <Label>
-                  {pgSetting.label}{' '}
-                  <Tooltip
-                    style={{ width: '100%' }}
-                    content={'This is a required field.'}
-                  >
-                    <Label colorName='lowContrast' colorSet='destructive'>
-                      *
-                    </Label>
-                  </Tooltip>
-                </Label>
-              }
-              action={
-                <div
-                  style={{
-                    display: 'flex',
-                    flexDirection: 'row',
-                    alignItems: 'center',
-                  }}
-                >
-                  <TextField
-                    variant='simple'
-                    onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
-                      pgSetting.stateHandler(e.target.value, setMetadataDB)
-                    }
-                    defaultValue={
-                      (metadataDB as PostgresConfig)[
-                        pgSetting.label.toLowerCase() as keyof PostgresConfig
-                      ] || ''
-                    }
-                  />
-                  {pgSetting.tips && (
-                    <InfoPopover
-                      tips={pgSetting.tips}
-                      link={pgSetting.helpfulLink}
-                    />
-                  )}
-                </div>
-              }
-            />
-          )
-      )}
     </div>
   );
 };