Skip to content

Commit

Permalink
UI: Better Create QRep (#1643)
Browse files Browse the repository at this point in the history
- Check if query box is unfilled
- Add explanation text for the query box
- Set Create Destination Table default to true
- Remove Staging Path field
- Remove some dead code

<img width="1159" alt="Screenshot 2024-04-23 at 10 47 52 PM"
src="https://github.com/PeerDB-io/peerdb/assets/65964360/6e67a0a8-5fdb-4310-8b1f-c4bd28204c1d">
<img width="424" alt="Screenshot 2024-04-23 at 10 52 00 PM"
src="https://github.com/PeerDB-io/peerdb/assets/65964360/4f43a314-bf6c-4086-add3-8eecc51c1ecb">

---------

Co-authored-by: Kevin Biju <[email protected]>
Co-authored-by: Philip Dubé <[email protected]>
  • Loading branch information
3 people authored Apr 25, 2024
1 parent 9851e34 commit f988899
Show file tree
Hide file tree
Showing 6 changed files with 51 additions and 50 deletions.
7 changes: 6 additions & 1 deletion ui/app/mirrors/create/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
UTablesResponse,
} from '@/app/dto/PeersDTO';
import { notifyErr } from '@/app/utils/notify';
import QRepQueryTemplate from '@/app/utils/qreptemplate';
import { DBTypeToGoodText } from '@/components/PeerTypeComponent';
import {
FlowConnectionConfigs,
Expand Down Expand Up @@ -198,14 +199,18 @@ export const handleCreateQRep = async (
route: RouteCallback,
xmin?: boolean
) => {
console.log(config);
const flowNameValid = flowNameSchema.safeParse(flowJobName);
if (!flowNameValid.success) {
const flowNameErr = flowNameValid.error.issues[0].message;
notifyErr(flowNameErr);
return;
}

if (query === QRepQueryTemplate) {
notifyErr('Please fill in the query box');
return;
}

if (xmin == true) {
config.watermarkColumn = 'xmin';
config.query = `SELECT * FROM ${quotedWatermarkTable(
Expand Down
2 changes: 1 addition & 1 deletion ui/app/mirrors/create/helpers/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,6 @@ export const blankQRepSetting = {
writeMode: undefined,
stagingPath: '',
numRowsPerPartition: 100000,
setupWatermarkTableOnDestination: false,
setupWatermarkTableOnDestination: true,
dstTableFullResync: false,
};
28 changes: 4 additions & 24 deletions ui/app/mirrors/create/helpers/qrep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
import { MirrorSetting } from './common';
export const qrepSettings: MirrorSetting[] = [
{
label: 'Table',
label: 'Source Table',
stateHandler: (value, setter) =>
setter((curr: QRepConfig) => ({
...curr,
Expand All @@ -32,10 +32,11 @@ export const qrepSettings: MirrorSetting[] = [
stateHandler: (value, setter) =>
setter((curr: QRepConfig) => ({
...curr,
setupWatermarkTableOnDestination: (value as boolean) || false,
setupWatermarkTableOnDestination: value as boolean,
})),
tips: 'Specify if you want to create the watermark table on the destination as-is, can be used for some queries.',
type: 'switch',
default: true,
},
{
label: 'Destination Table Name',
Expand Down Expand Up @@ -70,17 +71,6 @@ export const qrepSettings: MirrorSetting[] = [
default: '4',
type: 'number',
},
{
label: 'Staging Path',
stateHandler: (value, setter) =>
setter((curr: QRepConfig) => ({
...curr,
stagingPath: (value as string) || '',
})),
tips: `You can specify staging path for sync mode AVRO. For Snowflake as destination peer:
If this starts with s3:// then it will be written to S3.
If nothing is specified then it will be written to local disk.`,
},
{
label: 'Write Type',
stateHandler: (value, setter) =>
Expand Down Expand Up @@ -127,7 +117,7 @@ export const qrepSettings: MirrorSetting[] = [
type: 'switch',
},
{
label: 'Wait Time Between Batches',
label: 'Sync Interval (Seconds)',
stateHandler: (value, setter) =>
setter((curr: QRepConfig) => ({
...curr,
Expand All @@ -137,14 +127,4 @@ export const qrepSettings: MirrorSetting[] = [
default: 30,
type: 'number',
},
// {
// label: 'Resync Destination Table',
// stateHandler: (value, setter) =>
// setter((curr: QRepConfig) => ({
// ...curr,
// dstTableFullResync:value as boolean
// })),
// tips: 'Perform a resync of the provided destination table',
// type: 'switch',
// },
];
51 changes: 34 additions & 17 deletions ui/app/mirrors/create/page.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
'use client';
import SelectTheme from '@/app/styles/select';
import QRepQueryTemplate from '@/app/utils/qreptemplate';
import { DBTypeToImageMapping } from '@/components/PeerComponent';
import { RequiredIndicator } from '@/components/RequiredIndicator';
import { QRepConfig } from '@/grpc_generated/flow';
Expand All @@ -13,6 +14,7 @@ import { ProgressCircle } from '@/lib/ProgressCircle';
import { TextField } from '@/lib/TextField';
import { Divider } from '@tremor/react';
import Image from 'next/image';
import Link from 'next/link';
import { useRouter, useSearchParams } from 'next/navigation';
import { useEffect, useState } from 'react';
import ReactSelect from 'react-select';
Expand Down Expand Up @@ -69,29 +71,14 @@ export default function CreateMirrors() {
const [config, setConfig] = useState<CDCConfig | QRepConfig>(blankCDCSetting);
const [peers, setPeers] = useState<Peer[]>([]);
const [rows, setRows] = useState<TableMapRow[]>([]);
const [qrepQuery, setQrepQuery] =
useState<string>(`-- Here's a sample template:
SELECT * FROM <table_name>
WHERE <watermark_column>
BETWEEN {{.start}} AND {{.end}}`);
const [qrepQuery, setQrepQuery] = useState<string>(QRepQueryTemplate);

useEffect(() => {
fetch('/api/peers', { cache: 'no-store' })
.then((res) => res.json())
.then((res) => {
setPeers(res);
});

if (mirrorType != MirrorType.CDC) {
if (mirrorType === MirrorType.XMin) {
setConfig((curr) => {
return { ...curr, setupWatermarkTableOnDestination: true };
});
} else
setConfig((curr) => {
return { ...curr, setupWatermarkTableOnDestination: false };
});
}
}, [mirrorType]);

let listMirrorsPage = () => {
Expand Down Expand Up @@ -192,7 +179,37 @@ export default function CreateMirrors() {
<Divider style={{ marginTop: '1rem', marginBottom: '1rem' }} />

{mirrorType === MirrorType.QRep && (
<PeerDBCodeEditor code={qrepQuery} setter={setQrepQuery} />
<div
style={{
display: 'flex',
flexDirection: 'column',
rowGap: '1rem',
}}
>
<Label variant='subheadline'>Replication Query</Label>
<Label>
Write a query whose results will be replicated to a target
table.
<br></br>
In most cases, you will require a watermark table and a
watermark column in that table.
<br></br>
<Link
style={{
color: 'teal',
cursor: 'pointer',
width: 'fit-content',
}}
target='_blank'
href={
'https://docs.peerdb.io/usecases/Streaming%20Query%20Replication/postgres-to-snowflake#step-2-set-up-mirror-to-transform-and-sync-data'
}
>
What does that mean ?
</Link>
</Label>
<PeerDBCodeEditor code={qrepQuery} setter={setQrepQuery} />
</div>
)}

{mirrorType && (
Expand Down
7 changes: 0 additions & 7 deletions ui/app/mirrors/create/qrep/qrep.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,6 @@ interface QRepConfigProps {
xmin?: boolean;
}

interface QRepConfigProps {
settings: MirrorSetting[];
mirrorConfig: QRepConfig;
setter: MirrorSetter;
xmin?: boolean;
}

const WriteModes = ['Append', 'Upsert', 'Overwrite'].map((value) => ({
label: value,
value,
Expand Down
6 changes: 6 additions & 0 deletions ui/app/utils/qreptemplate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
const QRepQueryTemplate = `-- Here is a sample template: fill in the table name and watermark column
SELECT * FROM <table_name>
WHERE <watermark_column>
BETWEEN {{.start}} AND {{.end}}`;

export default QRepQueryTemplate;

0 comments on commit f988899

Please sign in to comment.