diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/destinations/components/DestinationConnectionCard.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/destinations/components/DestinationConnectionCard.tsx index e6a3c55896..2dc213e6ab 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/destinations/components/DestinationConnectionCard.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/destinations/components/DestinationConnectionCard.tsx @@ -1,4 +1,5 @@ 'use client'; +import ConnectionSelectContent from '@/app/(mgmt)/[account]/new/job/connect/ConnectionSelectContent'; import DestinationOptionsForm from '@/components/jobs/Form/DestinationOptionsForm'; import { Button } from '@/components/ui/button'; import { Card, CardContent, CardFooter } from '@/components/ui/card'; @@ -13,11 +14,11 @@ import { import { Select, SelectContent, - SelectItem, SelectTrigger, SelectValue, } from '@/components/ui/select'; import { useToast } from '@/components/ui/use-toast'; +import { splitConnections } from '@/libs/utils'; import { getErrorMessage } from '@/util/util'; import { NewDestinationFormValues } from '@/yup-validations/jobs'; import { useMutation } from '@connectrpc/connect-query'; @@ -124,7 +125,9 @@ export default function DestinationConnectionCard({ form.control, jobSourceId ); - console.log(form.formState.errors, form.formState.isValid); + + const { postgres, mysql, s3, mongodb, gcpcs, dynamodb } = + splitConnections(connections); return (
@@ -153,18 +156,20 @@ export default function DestinationConnectionCard({ value={field.value} > - + - {availableConnections.map((connection) => ( - - {connection.name} - - ))} + @@ -188,6 +193,8 @@ export default function DestinationConnectionCard({ }); }} hideInitTableSchema={shouldHideInitTableSchema} + hideDynamoDbTableMappings={true} + destinationDetailsRecord={{}} // not used because we are hiding dynamodb table mappings /> diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/[id]/destination/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/[id]/destination/page.tsx index fda6a7fc97..9ef613fd6e 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/[id]/destination/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/[id]/destination/page.tsx @@ -1,5 +1,9 @@ 'use client'; -import { getConnectionIdFromSource } from '@/app/(mgmt)/[account]/jobs/[id]/source/components/util'; +import { + getConnectionIdFromSource, + getDestinationDetailsRecord, + isDynamoDBConnection, +} from '@/app/(mgmt)/[account]/jobs/[id]/source/components/util'; import { toJobDestinationOptions } from '@/app/(mgmt)/[account]/jobs/util'; import PageHeader from '@/components/headers/PageHeader'; import DestinationOptionsForm from '@/components/jobs/Form/DestinationOptionsForm'; @@ -18,32 +22,37 @@ import { import { Select, SelectContent, - SelectItem, SelectTrigger, SelectValue, } from '@/components/ui/select'; import { useToast } from '@/components/ui/use-toast'; +import { splitConnections } from '@/libs/utils'; import { getErrorMessage } from '@/util/util'; import { NewDestinationFormValues } from '@/yup-validations/jobs'; import { useMutation, useQuery } from '@connectrpc/connect-query'; import { yupResolver } from '@hookform/resolvers/yup'; -import { Connection, JobDestination } from '@neosync/sdk'; +import { + Connection, + GetConnectionSchemaMapsResponse, + JobDestination, +} from '@neosync/sdk'; import { createJobDestinationConnections, getConnections, + getConnectionSchemaMaps, getJob, } from '@neosync/sdk/connectquery'; import { Cross1Icon, PlusIcon } from '@radix-ui/react-icons'; import { useRouter } from 'next/navigation'; -import { ReactElement, useState } from 'react'; +import { ReactElement } from 'react'; import { useFieldArray, useForm } from 'react-hook-form'; import * as Yup from 'yup'; +import ConnectionSelectContent from '../../connect/ConnectionSelectContent'; -const FORM_SCHEMA = Yup.object({ - jobId: Yup.string().required(), +const FormValues = Yup.object({ destinations: Yup.array(NewDestinationFormValues).required(), }); -type FormValues = Yup.InferType; +type FormValues = Yup.InferType; export default function Page({ params }: PageProps): ReactElement { const id = params?.id ?? ''; @@ -60,43 +69,60 @@ export default function Page({ params }: PageProps): ReactElement { createJobDestinationConnections ); - const [currConnection, setCurrConnection] = useState< - Connection | undefined - >(); - const connections = connectionsData?.connections ?? []; - const destinationIds = new Set( + const destinationConnectionIds = new Set( data?.job?.destinations.map((d) => d.connectionId) ); const sourceConnectionId = getConnectionIdFromSource(data?.job?.source); const form = useForm({ - resolver: yupResolver(FORM_SCHEMA), + resolver: yupResolver(FormValues), defaultValues: { - jobId: id, destinations: [{ connectionId: '', destinationOptions: {} }], }, }); const availableConnections = connections.filter( - (c) => c.id != sourceConnectionId && !destinationIds?.has(c.id) + (c) => c.id != sourceConnectionId && !destinationConnectionIds?.has(c.id) + ); + const connRecord = connections.reduce( + (record, conn) => { + record[conn.id] = conn; + return record; + }, + {} as Record ); - const { fields, append, remove } = useFieldArray({ + const { append, remove } = useFieldArray({ control: form.control, name: 'destinations', }); - async function onSubmit(values: FormValues) { + const fields = form.watch('destinations'); + + // Contains a list of the new destinations to be added that are specifically dynamo db connections + const newDynamoDestConnections = fields + .map((field) => connRecord[field.connectionId]) + .filter((conn) => !!conn && isDynamoDBConnection(conn)); + + const { data: destinationConnectionSchemaMapsResp } = useQuery( + getConnectionSchemaMaps, + { + requests: newDynamoDestConnections.map((conn) => ({ + connectionId: conn.id, + })), + }, + { enabled: newDynamoDestConnections.length > 0 } + ); + + async function onSubmit(values: FormValues): Promise { try { + const connMap = new Map(connections.map((c) => [c.id, c])); const job = await createJobConnections({ jobId: id, destinations: values.destinations.map((d) => { return new JobDestination({ connectionId: d.connectionId, - options: toJobDestinationOptions( - d, - connections.find((c) => c.id === d.connectionId) - ), + options: toJobDestinationOptions(d, connMap.get(d.connectionId)), }); }), }); @@ -115,6 +141,11 @@ export default function Page({ params }: PageProps): ReactElement { } } + const { postgres, mysql, s3, mongodb, gcpcs, dynamodb } = + splitConnections(availableConnections); + const sourceConnection = connRecord[sourceConnectionId ?? ''] as + | Connection + | undefined; return (
@@ -130,10 +161,13 @@ export default function Page({ params }: PageProps): ReactElement {
- {fields.map((_, index) => { - const destOpts = form.watch( - `destinations.${index}.destinationOptions` - ); + {fields.map((f, index) => { + // not using the field here because it doesn't seem to always update when it needs to + const connId = f.connectionId; + const destOpts = f.destinationOptions; + const destConnection = connRecord[connId] as + | Connection + | undefined; return (
@@ -147,34 +181,55 @@ export default function Page({ params }: PageProps): ReactElement { @@ -199,7 +254,7 @@ export default function Page({ params }: PageProps): ReactElement {
{ form.setValue( @@ -212,6 +267,20 @@ export default function Page({ params }: PageProps): ReactElement { } ); }} + hideDynamoDbTableMappings={ + !isDynamoDBConnection( + destConnection ?? new Connection() + ) + } + destinationDetailsRecord={getDestinationDetailsRecord( + fields.map((field) => ({ + connectionId: field.connectionId, + id: field.connectionId, + })), + connRecord, + destinationConnectionSchemaMapsResp ?? + new GetConnectionSchemaMapsResponse() + )} />
); diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/aigenerate/single/connect/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/aigenerate/single/connect/page.tsx index 26030df545..fe19c2e676 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/aigenerate/single/connect/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/aigenerate/single/connect/page.tsx @@ -380,6 +380,8 @@ export default function Page({ searchParams }: PageProps): ReactElement { shouldValidate: true, }); }} + hideDynamoDbTableMappings={true} + destinationDetailsRecord={{}} // not used because we are hiding dynamodb table mappings />
diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/ConnectionSelectContent.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/ConnectionSelectContent.tsx index 2401ca7359..5a2cc86520 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/ConnectionSelectContent.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/ConnectionSelectContent.tsx @@ -12,7 +12,8 @@ interface Props { gcpcs?: Connection[]; dynamodb?: Connection[]; - newConnectionValue: string; + // Provide a value to include the new connection item + newConnectionValue?: string; } export default function ConnectionSelectContent(props: Props): ReactElement { const { @@ -53,16 +54,18 @@ export default function ConnectionSelectContent(props: Props): ReactElement { ) )} - -
- -

New Connection

-
-
+ {!!newConnectionValue && ( + +
+ +

New Connection

+
+
+ )} ); } diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/page.tsx index e0f4bedd82..be92c26724 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/connect/page.tsx @@ -449,6 +449,8 @@ export default function Page({ searchParams }: PageProps): ReactElement { } ); }} + hideDynamoDbTableMappings={true} + destinationDetailsRecord={{}} // not used beacause we are hiding dynamodb table mappings />
); diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/connect/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/connect/page.tsx index 0041f1aaf4..bfc99f457a 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/connect/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/generate/single/connect/page.tsx @@ -285,6 +285,8 @@ export default function Page({ searchParams }: PageProps): ReactElement { shouldValidate: true, }); }} + hideDynamoDbTableMappings={true} + destinationDetailsRecord={{}} // not used because we are hiding dynamodb table mappings /> diff --git a/frontend/apps/web/components/jobs/Form/DestinationOptionsForm.tsx b/frontend/apps/web/components/jobs/Form/DestinationOptionsForm.tsx index 7a94f41ae6..bcb505cb8c 100644 --- a/frontend/apps/web/components/jobs/Form/DestinationOptionsForm.tsx +++ b/frontend/apps/web/components/jobs/Form/DestinationOptionsForm.tsx @@ -4,6 +4,10 @@ import { Badge } from '@/components/ui/badge'; import { DestinationOptionsFormValues } from '@/yup-validations/jobs'; import { Connection } from '@neosync/sdk'; import { ReactElement } from 'react'; +import { DestinationDetails } from '../NosqlTable/TableMappings/Columns'; +import TableMappingsCard, { + Props as TableMappingsCardProps, +} from '../NosqlTable/TableMappings/TableMappingsCard'; interface DestinationOptionsProps { connection?: Connection; @@ -12,12 +16,21 @@ interface DestinationOptionsProps { setValue(newVal: DestinationOptionsFormValues): void; hideInitTableSchema?: boolean; + hideDynamoDbTableMappings?: boolean; + destinationDetailsRecord: Record; } export default function DestinationOptionsForm( props: DestinationOptionsProps ): ReactElement { - const { connection, value, setValue, hideInitTableSchema } = props; + const { + connection, + value, + setValue, + hideInitTableSchema, + hideDynamoDbTableMappings, + destinationDetailsRecord, + } = props; if (!connection) { return <>; @@ -200,7 +213,44 @@ export default function DestinationOptionsForm( case 'gcpCloudstorageConfig': return <>; case 'dynamodbConfig': - return <>; + return ( + { + if (tm.sourceTable === req.souceName) { + tm.destinationTable = req.tableName; + } + }); + } + setValue({ + ...value, + dynamodb: { + ...value.dynamodb, + tableMappings: tableMappings, + }, + }); + }, + }} + /> + ); default: return (
@@ -210,3 +260,21 @@ export default function DestinationOptionsForm( ); } } + +interface DynamoDbOptionsProps { + hideDynamoDbTableMappings: boolean; + + tableMappingsProps: TableMappingsCardProps; +} + +function DynamoDbOptions(props: DynamoDbOptionsProps): ReactElement { + const { hideDynamoDbTableMappings, tableMappingsProps } = props; + + return ( +
+ {!hideDynamoDbTableMappings && ( + + )} +
+ ); +} diff --git a/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx b/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx index 667ad0f4fe..1506515ea2 100644 --- a/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx +++ b/frontend/apps/web/components/jobs/NosqlTable/NosqlTable.tsx @@ -56,11 +56,9 @@ import { SchemaConstraintHandler } from '../SchemaTable/schema-constraint-handle import { TransformerHandler } from '../SchemaTable/transformer-handler'; import { DestinationDetails, - getTableMappingsColumns, OnTableMappingUpdateRequest, - TableMappingRow, } from './TableMappings/Columns'; -import TableMappingsTable from './TableMappings/TableMappingsTable'; +import TableMappingsCard from './TableMappings/TableMappingsCard'; import { DataTableRowActions } from './data-table-row-actions'; interface Props { @@ -169,62 +167,6 @@ export default function NosqlTable(props: Props): ReactElement {
); } - -interface TableMappingsCardProps { - mappings: EditDestinationOptionsFormValues[]; - onUpdate(req: OnTableMappingUpdateRequest): void; - destinationDetailsRecord: Record; -} - -function TableMappingsCard(props: TableMappingsCardProps): ReactElement { - const { mappings, onUpdate, destinationDetailsRecord } = props; - const columns = useMemo( - () => getTableMappingsColumns({ destinationDetailsRecord, onUpdate }), - [destinationDetailsRecord, onUpdate] - ); - return ( - - -
-
- -
- DynamoDB Table Mappings - {/*
{isValidating ? : null}
*/} -
- - Map a table from source to destination. As tables are added in the - form above, they will dynamically be added to this section. A mapping - is required to denote which table each source table should be synced - to for each corresponding DynamoDB destination. - -
- - - -
- ); -} - -function toTableMappingRows( - mappings: EditDestinationOptionsFormValues[] -): TableMappingRow[] { - return mappings.flatMap((mapping) => { - return ( - mapping.dynamodb?.tableMappings.map((tm): TableMappingRow => { - return { - destinationId: mapping.destinationId, - sourceTable: tm.sourceTable, - destinationTable: tm.destinationTable, - }; - }) ?? [] - ); - }); -} - interface AddNewRecordProps { collections: string[]; onSubmit(values: AddNewNosqlRecordFormValues): void; diff --git a/frontend/apps/web/components/jobs/NosqlTable/TableMappings/TableMappingsCard.tsx b/frontend/apps/web/components/jobs/NosqlTable/TableMappings/TableMappingsCard.tsx new file mode 100644 index 0000000000..a9aec50ceb --- /dev/null +++ b/frontend/apps/web/components/jobs/NosqlTable/TableMappings/TableMappingsCard.tsx @@ -0,0 +1,71 @@ +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { EditDestinationOptionsFormValues } from '@/yup-validations/jobs'; +import { TableIcon } from '@radix-ui/react-icons'; +import { ReactElement, useMemo } from 'react'; +import { + DestinationDetails, + getTableMappingsColumns, + OnTableMappingUpdateRequest, + TableMappingRow, +} from './Columns'; +import TableMappingsTable from './TableMappingsTable'; + +export interface Props { + mappings: EditDestinationOptionsFormValues[]; + onUpdate(req: OnTableMappingUpdateRequest): void; + destinationDetailsRecord: Record; +} + +export default function TableMappingsCard(props: Props): ReactElement { + const { mappings, onUpdate, destinationDetailsRecord } = props; + const columns = useMemo( + () => getTableMappingsColumns({ destinationDetailsRecord, onUpdate }), + [destinationDetailsRecord, onUpdate] + ); + return ( + + +
+
+ +
+ DynamoDB Table Mappings +
+ + Map a table from source to destination. As tables are added in the + form above, they will dynamically be added to this section. A mapping + is required to denote which table each source table should be synced + to for each corresponding DynamoDB destination. + +
+ + + +
+ ); +} + +function toTableMappingRows( + mappings: EditDestinationOptionsFormValues[] +): TableMappingRow[] { + return mappings.flatMap((mapping) => { + return ( + mapping.dynamodb?.tableMappings.map((tm): TableMappingRow => { + return { + destinationId: mapping.destinationId, + sourceTable: tm.sourceTable, + destinationTable: tm.destinationTable, + }; + }) ?? [] + ); + }); +}