Merge branch 'main' into ui/peerdb-columns

PeerDB-io · Apr 30, 2024 · 8a1d66b · 8a1d66b
2 parents 88d5388 + 13921fb
commit 8a1d66b
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 0 deletions.
diff --git a/flow/connectors/clickhouse/qrep_avro_sync.go b/flow/connectors/clickhouse/qrep_avro_sync.go
@@ -128,6 +128,7 @@ func (s *ClickhouseAvroSyncMethod) SyncQRepRecords(
 	for _, col := range dstTableSchema {
 		colName := col.Name()
 		if strings.EqualFold(colName, config.SoftDeleteColName) ||
+			strings.EqualFold(colName, signColName) ||
 			strings.EqualFold(colName, config.SyncedAtColName) ||
 			strings.EqualFold(colName, versionColName) {
 			continue

diff --git a/ui/app/mirrors/create/handlers.ts b/ui/app/mirrors/create/handlers.ts
@@ -211,6 +211,17 @@ export const handleCreateQRep = async (
     return;
   }
 
+  if (
+    !xmin &&
+    config.writeMode?.writeType != QRepWriteType.QREP_WRITE_MODE_OVERWRITE &&
+    !(query.includes('{{.start}}') && query.includes('{{.end}}'))
+  ) {
+    notifyErr(
+      'Please include placeholders {{.start}} and {{.end}} in the query'
+    );
+    return;
+  }
+
   if (xmin == true) {
     config.watermarkColumn = 'xmin';
     config.query = `SELECT * FROM ${quotedWatermarkTable(

diff --git a/ui/app/mirrors/create/page.tsx b/ui/app/mirrors/create/page.tsx
@@ -191,6 +191,11 @@ export default function CreateMirrors() {
                 Write a query whose results will be replicated to a target
                 table.
                 <br></br>
+                For append and upsert modes, make sure the query{' '}
+                <b>includes the start and end placeholders</b> in the query.
+                PeerDB uses these placeholders for partitioning query results
+                for performance.
+                <br></br>
                 In most cases, you will require a watermark table and a
                 watermark column in that table.
                 <br></br>