From efc9f451d56095f60bdc4483f88d838f2f52beaa Mon Sep 17 00:00:00 2001 From: Kevin K Biju <52661649+heavycrystal@users.noreply.github.com> Date: Mon, 6 Nov 2023 18:30:36 +0000 Subject: [PATCH] full table resync for Snowflake QRep (#617) --- flow/activities/flowable.go | 24 +- flow/cmd/handler.go | 11 +- flow/connectors/snowflake/snowflake.go | 56 +- flow/e2e/test_utils.go | 8 +- flow/generated/protos/flow.pb.go | 943 +++++++++++++++--------- flow/workflows/qrep_flow.go | 94 ++- flow/workflows/snapshot_flow.go | 10 +- nexus/analyzer/src/qrep.rs | 5 + nexus/flow-rs/src/grpc.rs | 2 + nexus/pt/src/peerdb_flow.rs | 32 + nexus/pt/src/peerdb_flow.serde.rs | 381 ++++++++++ protos/flow.proto | 22 + ui/app/mirrors/create/helpers/common.ts | 1 + ui/grpc_generated/flow.ts | 394 +++++++++- 14 files changed, 1608 insertions(+), 375 deletions(-) diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index 6d209a364e..2c4e1cbb71 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -723,7 +723,8 @@ func (a *FlowableActivity) QRepWaitUntilNewRows(ctx context.Context, return nil } -func (a *FlowableActivity) RenameTables(ctx context.Context, config *protos.RenameTablesInput) (*protos.RenameTablesOutput, error) { +func (a *FlowableActivity) RenameTables(ctx context.Context, config *protos.RenameTablesInput) ( + *protos.RenameTablesOutput, error) { dstConn, err := connectors.GetCDCSyncConnector(ctx, config.Peer) if err != nil { return nil, fmt.Errorf("failed to get connector: %w", err) @@ -742,3 +743,24 @@ func (a *FlowableActivity) RenameTables(ctx context.Context, config *protos.Rena return sfConn.RenameTables(config) } + +func (a *FlowableActivity) CreateTablesFromExisting(ctx context.Context, req *protos.CreateTablesFromExistingInput) ( + *protos.CreateTablesFromExistingOutput, error) { + dstConn, err := connectors.GetCDCSyncConnector(ctx, req.Peer) + if err != nil { + return nil, fmt.Errorf("failed to get connector: %w", err) + } + defer connectors.CloseConnector(dstConn) + + // check if destination is snowflake, if not error out + if req.Peer.Type != protos.DBType_SNOWFLAKE { + return nil, fmt.Errorf("create tables from existing is only supported on snowflake") + } + + sfConn, ok := dstConn.(*connsnowflake.SnowflakeConnector) + if !ok { + return nil, fmt.Errorf("failed to cast connector to snowflake connector") + } + + return sfConn.CreateTablesFromExisting(req) +} diff --git a/flow/cmd/handler.go b/flow/cmd/handler.go index aa47626b69..4cdf44df8e 100644 --- a/flow/cmd/handler.go +++ b/flow/cmd/handler.go @@ -208,11 +208,6 @@ func (h *FlowRequestHandler) removeFlowEntryInCatalog( func (h *FlowRequestHandler) CreateQRepFlow( ctx context.Context, req *protos.CreateQRepFlowRequest) (*protos.CreateQRepFlowResponse, error) { - lastPartition := &protos.QRepPartition{ - PartitionId: "not-applicable-partition", - Range: nil, - } - cfg := req.QrepConfig workflowID := fmt.Sprintf("%s-qrepflow-%s", cfg.FlowJobName, uuid.New()) workflowOptions := client.StartWorkflowOptions{ @@ -225,14 +220,14 @@ func (h *FlowRequestHandler) CreateQRepFlow( return nil, fmt.Errorf("unable to create flow job entry: %w", err) } } - numPartitionsProcessed := 0 + + state := peerflow.NewQRepFlowState() _, err := h.temporalClient.ExecuteWorkflow( ctx, // context workflowOptions, // workflow start options peerflow.QRepFlowWorkflow, // workflow function cfg, // workflow input - lastPartition, // last partition - numPartitionsProcessed, // number of partitions processed + state, ) if err != nil { return nil, fmt.Errorf("unable to start QRepFlow workflow: %w", err) diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go index 3fe717dd40..c31620f82d 100644 --- a/flow/connectors/snowflake/snowflake.go +++ b/flow/connectors/snowflake/snowflake.go @@ -492,7 +492,7 @@ func (c *SnowflakeConnector) SyncRecords(req *model.SyncRecordsRequest) (*model. var res *model.SyncResponse if req.SyncMode == protos.QRepSyncMode_QREP_SYNC_MODE_STORAGE_AVRO { - log.Infof("sync mode is for flow %s is AVRO", req.FlowJobName) + log.Infof("sync mode for flow %s is AVRO", req.FlowJobName) res, err = c.syncRecordsViaAvro(req, rawTableIdentifier, syncBatchID) if err != nil { return nil, err @@ -516,7 +516,7 @@ func (c *SnowflakeConnector) SyncRecords(req *model.SyncRecordsRequest) (*model. }() if req.SyncMode == protos.QRepSyncMode_QREP_SYNC_MODE_MULTI_INSERT { - log.Infof("sync mode is for flow %s is MULTI_INSERT", req.FlowJobName) + log.Infof("sync mode for flow %s is MULTI_INSERT", req.FlowJobName) res, err = c.syncRecordsViaSQL(req, rawTableIdentifier, syncBatchID, syncRecordsTx) if err != nil { return nil, err @@ -1154,6 +1154,14 @@ func (c *SnowflakeConnector) RenameTables(req *protos.RenameTablesInput) (*proto if err != nil { return nil, fmt.Errorf("unable to begin transaction for rename tables: %w", err) } + defer func() { + deferErr := renameTablesTx.Rollback() + if deferErr != sql.ErrTxDone && deferErr != nil { + log.WithFields(log.Fields{ + "flowName": req.FlowJobName, + }).Errorf("unexpected error rolling back transaction for renaming tables: %v", err) + } + }() for _, renameRequest := range req.RenameTableOptions { src := renameRequest.CurrentName @@ -1193,3 +1201,47 @@ func (c *SnowflakeConnector) RenameTables(req *protos.RenameTablesInput) (*proto FlowJobName: req.FlowJobName, }, nil } + +func (c *SnowflakeConnector) CreateTablesFromExisting(req *protos.CreateTablesFromExistingInput) ( + *protos.CreateTablesFromExistingOutput, error) { + createTablesFromExistingTx, err := c.database.BeginTx(c.ctx, nil) + if err != nil { + return nil, fmt.Errorf("unable to begin transaction for rename tables: %w", err) + } + defer func() { + deferErr := createTablesFromExistingTx.Rollback() + if deferErr != sql.ErrTxDone && deferErr != nil { + log.WithFields(log.Fields{ + "flowName": req.FlowJobName, + }).Errorf("unexpected error rolling back transaction for creating tables: %v", err) + } + }() + + for newTable, existingTable := range req.NewToExistingTableMapping { + log.WithFields(log.Fields{ + "flowName": req.FlowJobName, + }).Infof("creating table '%s' similar to '%s'", newTable, existingTable) + + activity.RecordHeartbeat(c.ctx, fmt.Sprintf("creating table '%s' similar to '%s'", newTable, existingTable)) + + // rename the src table to dst + _, err = createTablesFromExistingTx.ExecContext(c.ctx, + fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s LIKE %s", newTable, existingTable)) + if err != nil { + return nil, fmt.Errorf("unable to create table %s: %w", newTable, err) + } + + log.WithFields(log.Fields{ + "flowName": req.FlowJobName, + }).Infof("successfully created table '%s'", newTable) + } + + err = createTablesFromExistingTx.Commit() + if err != nil { + return nil, fmt.Errorf("unable to commit transaction for creating tables: %w", err) + } + + return &protos.CreateTablesFromExistingOutput{ + FlowJobName: req.FlowJobName, + }, nil +} diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index 4ea032bf3d..0abf3311d6 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -301,12 +301,8 @@ func CreateQRepWorkflowConfig( } func RunQrepFlowWorkflow(env *testsuite.TestWorkflowEnvironment, config *protos.QRepConfig) { - lastPartition := &protos.QRepPartition{ - PartitionId: "not-applicable-partition", - Range: nil, - } - numPartitionsProcessed := 0 - env.ExecuteWorkflow(peerflow.QRepFlowWorkflow, config, lastPartition, numPartitionsProcessed) + state := peerflow.NewQRepFlowState() + env.ExecuteWorkflow(peerflow.QRepFlowWorkflow, config, state) } func GetOwnersSchema() *model.QRecordSchema { diff --git a/flow/generated/protos/flow.pb.go b/flow/generated/protos/flow.pb.go index 0dffeecc71..1c38efbd13 100644 --- a/flow/generated/protos/flow.pb.go +++ b/flow/generated/protos/flow.pb.go @@ -395,6 +395,8 @@ type FlowConnectionConfigs struct { PushBatchSize int64 `protobuf:"varint,21,opt,name=push_batch_size,json=pushBatchSize,proto3" json:"push_batch_size,omitempty"` PushParallelism int64 `protobuf:"varint,22,opt,name=push_parallelism,json=pushParallelism,proto3" json:"push_parallelism,omitempty"` // if true, then the flow will be resynced + // create new tables with "_resync" suffix, perform initial load and then swap the new tables with the old ones + // to be used after the old mirror is dropped Resync bool `protobuf:"varint,23,opt,name=resync,proto3" json:"resync,omitempty"` } @@ -756,6 +758,116 @@ func (x *RenameTablesOutput) GetFlowJobName() string { return "" } +type CreateTablesFromExistingInput struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FlowJobName string `protobuf:"bytes,1,opt,name=flow_job_name,json=flowJobName,proto3" json:"flow_job_name,omitempty"` + Peer *Peer `protobuf:"bytes,2,opt,name=peer,proto3" json:"peer,omitempty"` + NewToExistingTableMapping map[string]string `protobuf:"bytes,3,rep,name=new_to_existing_table_mapping,json=newToExistingTableMapping,proto3" json:"new_to_existing_table_mapping,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *CreateTablesFromExistingInput) Reset() { + *x = CreateTablesFromExistingInput{} + if protoimpl.UnsafeEnabled { + mi := &file_flow_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateTablesFromExistingInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateTablesFromExistingInput) ProtoMessage() {} + +func (x *CreateTablesFromExistingInput) ProtoReflect() protoreflect.Message { + mi := &file_flow_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateTablesFromExistingInput.ProtoReflect.Descriptor instead. +func (*CreateTablesFromExistingInput) Descriptor() ([]byte, []int) { + return file_flow_proto_rawDescGZIP(), []int{8} +} + +func (x *CreateTablesFromExistingInput) GetFlowJobName() string { + if x != nil { + return x.FlowJobName + } + return "" +} + +func (x *CreateTablesFromExistingInput) GetPeer() *Peer { + if x != nil { + return x.Peer + } + return nil +} + +func (x *CreateTablesFromExistingInput) GetNewToExistingTableMapping() map[string]string { + if x != nil { + return x.NewToExistingTableMapping + } + return nil +} + +type CreateTablesFromExistingOutput struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FlowJobName string `protobuf:"bytes,2,opt,name=flow_job_name,json=flowJobName,proto3" json:"flow_job_name,omitempty"` +} + +func (x *CreateTablesFromExistingOutput) Reset() { + *x = CreateTablesFromExistingOutput{} + if protoimpl.UnsafeEnabled { + mi := &file_flow_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateTablesFromExistingOutput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateTablesFromExistingOutput) ProtoMessage() {} + +func (x *CreateTablesFromExistingOutput) ProtoReflect() protoreflect.Message { + mi := &file_flow_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateTablesFromExistingOutput.ProtoReflect.Descriptor instead. +func (*CreateTablesFromExistingOutput) Descriptor() ([]byte, []int) { + return file_flow_proto_rawDescGZIP(), []int{9} +} + +func (x *CreateTablesFromExistingOutput) GetFlowJobName() string { + if x != nil { + return x.FlowJobName + } + return "" +} + type SyncFlowOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -768,7 +880,7 @@ type SyncFlowOptions struct { func (x *SyncFlowOptions) Reset() { *x = SyncFlowOptions{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[8] + mi := &file_flow_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -781,7 +893,7 @@ func (x *SyncFlowOptions) String() string { func (*SyncFlowOptions) ProtoMessage() {} func (x *SyncFlowOptions) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[8] + mi := &file_flow_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -794,7 +906,7 @@ func (x *SyncFlowOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use SyncFlowOptions.ProtoReflect.Descriptor instead. func (*SyncFlowOptions) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{8} + return file_flow_proto_rawDescGZIP(), []int{10} } func (x *SyncFlowOptions) GetBatchSize() int32 { @@ -822,7 +934,7 @@ type NormalizeFlowOptions struct { func (x *NormalizeFlowOptions) Reset() { *x = NormalizeFlowOptions{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[9] + mi := &file_flow_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -835,7 +947,7 @@ func (x *NormalizeFlowOptions) String() string { func (*NormalizeFlowOptions) ProtoMessage() {} func (x *NormalizeFlowOptions) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[9] + mi := &file_flow_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -848,7 +960,7 @@ func (x *NormalizeFlowOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use NormalizeFlowOptions.ProtoReflect.Descriptor instead. func (*NormalizeFlowOptions) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{9} + return file_flow_proto_rawDescGZIP(), []int{11} } func (x *NormalizeFlowOptions) GetBatchSize() int32 { @@ -870,7 +982,7 @@ type LastSyncState struct { func (x *LastSyncState) Reset() { *x = LastSyncState{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[10] + mi := &file_flow_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -883,7 +995,7 @@ func (x *LastSyncState) String() string { func (*LastSyncState) ProtoMessage() {} func (x *LastSyncState) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[10] + mi := &file_flow_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -896,7 +1008,7 @@ func (x *LastSyncState) ProtoReflect() protoreflect.Message { // Deprecated: Use LastSyncState.ProtoReflect.Descriptor instead. func (*LastSyncState) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{10} + return file_flow_proto_rawDescGZIP(), []int{12} } func (x *LastSyncState) GetCheckpoint() int64 { @@ -927,7 +1039,7 @@ type StartFlowInput struct { func (x *StartFlowInput) Reset() { *x = StartFlowInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[11] + mi := &file_flow_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -940,7 +1052,7 @@ func (x *StartFlowInput) String() string { func (*StartFlowInput) ProtoMessage() {} func (x *StartFlowInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[11] + mi := &file_flow_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -953,7 +1065,7 @@ func (x *StartFlowInput) ProtoReflect() protoreflect.Message { // Deprecated: Use StartFlowInput.ProtoReflect.Descriptor instead. func (*StartFlowInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{11} + return file_flow_proto_rawDescGZIP(), []int{13} } func (x *StartFlowInput) GetLastSyncState() *LastSyncState { @@ -995,7 +1107,7 @@ type StartNormalizeInput struct { func (x *StartNormalizeInput) Reset() { *x = StartNormalizeInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[12] + mi := &file_flow_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1008,7 +1120,7 @@ func (x *StartNormalizeInput) String() string { func (*StartNormalizeInput) ProtoMessage() {} func (x *StartNormalizeInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[12] + mi := &file_flow_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1021,7 +1133,7 @@ func (x *StartNormalizeInput) ProtoReflect() protoreflect.Message { // Deprecated: Use StartNormalizeInput.ProtoReflect.Descriptor instead. func (*StartNormalizeInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{12} + return file_flow_proto_rawDescGZIP(), []int{14} } func (x *StartNormalizeInput) GetFlowConnectionConfigs() *FlowConnectionConfigs { @@ -1043,7 +1155,7 @@ type GetLastSyncedIDInput struct { func (x *GetLastSyncedIDInput) Reset() { *x = GetLastSyncedIDInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[13] + mi := &file_flow_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1056,7 +1168,7 @@ func (x *GetLastSyncedIDInput) String() string { func (*GetLastSyncedIDInput) ProtoMessage() {} func (x *GetLastSyncedIDInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[13] + mi := &file_flow_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1069,7 +1181,7 @@ func (x *GetLastSyncedIDInput) ProtoReflect() protoreflect.Message { // Deprecated: Use GetLastSyncedIDInput.ProtoReflect.Descriptor instead. func (*GetLastSyncedIDInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{13} + return file_flow_proto_rawDescGZIP(), []int{15} } func (x *GetLastSyncedIDInput) GetPeerConnectionConfig() *Peer { @@ -1099,7 +1211,7 @@ type EnsurePullabilityInput struct { func (x *EnsurePullabilityInput) Reset() { *x = EnsurePullabilityInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[14] + mi := &file_flow_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1112,7 +1224,7 @@ func (x *EnsurePullabilityInput) String() string { func (*EnsurePullabilityInput) ProtoMessage() {} func (x *EnsurePullabilityInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[14] + mi := &file_flow_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1125,7 +1237,7 @@ func (x *EnsurePullabilityInput) ProtoReflect() protoreflect.Message { // Deprecated: Use EnsurePullabilityInput.ProtoReflect.Descriptor instead. func (*EnsurePullabilityInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{14} + return file_flow_proto_rawDescGZIP(), []int{16} } func (x *EnsurePullabilityInput) GetPeerConnectionConfig() *Peer { @@ -1162,7 +1274,7 @@ type EnsurePullabilityBatchInput struct { func (x *EnsurePullabilityBatchInput) Reset() { *x = EnsurePullabilityBatchInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[15] + mi := &file_flow_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1175,7 +1287,7 @@ func (x *EnsurePullabilityBatchInput) String() string { func (*EnsurePullabilityBatchInput) ProtoMessage() {} func (x *EnsurePullabilityBatchInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[15] + mi := &file_flow_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1188,7 +1300,7 @@ func (x *EnsurePullabilityBatchInput) ProtoReflect() protoreflect.Message { // Deprecated: Use EnsurePullabilityBatchInput.ProtoReflect.Descriptor instead. func (*EnsurePullabilityBatchInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{15} + return file_flow_proto_rawDescGZIP(), []int{17} } func (x *EnsurePullabilityBatchInput) GetPeerConnectionConfig() *Peer { @@ -1223,7 +1335,7 @@ type PostgresTableIdentifier struct { func (x *PostgresTableIdentifier) Reset() { *x = PostgresTableIdentifier{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[16] + mi := &file_flow_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1236,7 +1348,7 @@ func (x *PostgresTableIdentifier) String() string { func (*PostgresTableIdentifier) ProtoMessage() {} func (x *PostgresTableIdentifier) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[16] + mi := &file_flow_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1249,7 +1361,7 @@ func (x *PostgresTableIdentifier) ProtoReflect() protoreflect.Message { // Deprecated: Use PostgresTableIdentifier.ProtoReflect.Descriptor instead. func (*PostgresTableIdentifier) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{16} + return file_flow_proto_rawDescGZIP(), []int{18} } func (x *PostgresTableIdentifier) GetRelId() uint32 { @@ -1273,7 +1385,7 @@ type TableIdentifier struct { func (x *TableIdentifier) Reset() { *x = TableIdentifier{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[17] + mi := &file_flow_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1286,7 +1398,7 @@ func (x *TableIdentifier) String() string { func (*TableIdentifier) ProtoMessage() {} func (x *TableIdentifier) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[17] + mi := &file_flow_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1299,7 +1411,7 @@ func (x *TableIdentifier) ProtoReflect() protoreflect.Message { // Deprecated: Use TableIdentifier.ProtoReflect.Descriptor instead. func (*TableIdentifier) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{17} + return file_flow_proto_rawDescGZIP(), []int{19} } func (m *TableIdentifier) GetTableIdentifier() isTableIdentifier_TableIdentifier { @@ -1337,7 +1449,7 @@ type EnsurePullabilityOutput struct { func (x *EnsurePullabilityOutput) Reset() { *x = EnsurePullabilityOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[18] + mi := &file_flow_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1350,7 +1462,7 @@ func (x *EnsurePullabilityOutput) String() string { func (*EnsurePullabilityOutput) ProtoMessage() {} func (x *EnsurePullabilityOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[18] + mi := &file_flow_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1363,7 +1475,7 @@ func (x *EnsurePullabilityOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use EnsurePullabilityOutput.ProtoReflect.Descriptor instead. func (*EnsurePullabilityOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{18} + return file_flow_proto_rawDescGZIP(), []int{20} } func (x *EnsurePullabilityOutput) GetTableIdentifier() *TableIdentifier { @@ -1384,7 +1496,7 @@ type EnsurePullabilityBatchOutput struct { func (x *EnsurePullabilityBatchOutput) Reset() { *x = EnsurePullabilityBatchOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[19] + mi := &file_flow_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1397,7 +1509,7 @@ func (x *EnsurePullabilityBatchOutput) String() string { func (*EnsurePullabilityBatchOutput) ProtoMessage() {} func (x *EnsurePullabilityBatchOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[19] + mi := &file_flow_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1410,7 +1522,7 @@ func (x *EnsurePullabilityBatchOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use EnsurePullabilityBatchOutput.ProtoReflect.Descriptor instead. func (*EnsurePullabilityBatchOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{19} + return file_flow_proto_rawDescGZIP(), []int{21} } func (x *EnsurePullabilityBatchOutput) GetTableIdentifierMapping() map[string]*TableIdentifier { @@ -1438,7 +1550,7 @@ type SetupReplicationInput struct { func (x *SetupReplicationInput) Reset() { *x = SetupReplicationInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[20] + mi := &file_flow_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1451,7 +1563,7 @@ func (x *SetupReplicationInput) String() string { func (*SetupReplicationInput) ProtoMessage() {} func (x *SetupReplicationInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[20] + mi := &file_flow_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1464,7 +1576,7 @@ func (x *SetupReplicationInput) ProtoReflect() protoreflect.Message { // Deprecated: Use SetupReplicationInput.ProtoReflect.Descriptor instead. func (*SetupReplicationInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{20} + return file_flow_proto_rawDescGZIP(), []int{22} } func (x *SetupReplicationInput) GetPeerConnectionConfig() *Peer { @@ -1528,7 +1640,7 @@ type SetupReplicationOutput struct { func (x *SetupReplicationOutput) Reset() { *x = SetupReplicationOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[21] + mi := &file_flow_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1541,7 +1653,7 @@ func (x *SetupReplicationOutput) String() string { func (*SetupReplicationOutput) ProtoMessage() {} func (x *SetupReplicationOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[21] + mi := &file_flow_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1554,7 +1666,7 @@ func (x *SetupReplicationOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use SetupReplicationOutput.ProtoReflect.Descriptor instead. func (*SetupReplicationOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{21} + return file_flow_proto_rawDescGZIP(), []int{23} } func (x *SetupReplicationOutput) GetSlotName() string { @@ -1585,7 +1697,7 @@ type CreateRawTableInput struct { func (x *CreateRawTableInput) Reset() { *x = CreateRawTableInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[22] + mi := &file_flow_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1598,7 +1710,7 @@ func (x *CreateRawTableInput) String() string { func (*CreateRawTableInput) ProtoMessage() {} func (x *CreateRawTableInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[22] + mi := &file_flow_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1611,7 +1723,7 @@ func (x *CreateRawTableInput) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateRawTableInput.ProtoReflect.Descriptor instead. func (*CreateRawTableInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{22} + return file_flow_proto_rawDescGZIP(), []int{24} } func (x *CreateRawTableInput) GetPeerConnectionConfig() *Peer { @@ -1653,7 +1765,7 @@ type CreateRawTableOutput struct { func (x *CreateRawTableOutput) Reset() { *x = CreateRawTableOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[23] + mi := &file_flow_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1666,7 +1778,7 @@ func (x *CreateRawTableOutput) String() string { func (*CreateRawTableOutput) ProtoMessage() {} func (x *CreateRawTableOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[23] + mi := &file_flow_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1679,7 +1791,7 @@ func (x *CreateRawTableOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateRawTableOutput.ProtoReflect.Descriptor instead. func (*CreateRawTableOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{23} + return file_flow_proto_rawDescGZIP(), []int{25} } func (x *CreateRawTableOutput) GetTableIdentifier() string { @@ -1705,7 +1817,7 @@ type TableSchema struct { func (x *TableSchema) Reset() { *x = TableSchema{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[24] + mi := &file_flow_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1718,7 +1830,7 @@ func (x *TableSchema) String() string { func (*TableSchema) ProtoMessage() {} func (x *TableSchema) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[24] + mi := &file_flow_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1731,7 +1843,7 @@ func (x *TableSchema) ProtoReflect() protoreflect.Message { // Deprecated: Use TableSchema.ProtoReflect.Descriptor instead. func (*TableSchema) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{24} + return file_flow_proto_rawDescGZIP(), []int{26} } func (x *TableSchema) GetTableIdentifier() string { @@ -1774,7 +1886,7 @@ type GetTableSchemaBatchInput struct { func (x *GetTableSchemaBatchInput) Reset() { *x = GetTableSchemaBatchInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[25] + mi := &file_flow_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1787,7 +1899,7 @@ func (x *GetTableSchemaBatchInput) String() string { func (*GetTableSchemaBatchInput) ProtoMessage() {} func (x *GetTableSchemaBatchInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[25] + mi := &file_flow_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1800,7 +1912,7 @@ func (x *GetTableSchemaBatchInput) ProtoReflect() protoreflect.Message { // Deprecated: Use GetTableSchemaBatchInput.ProtoReflect.Descriptor instead. func (*GetTableSchemaBatchInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{25} + return file_flow_proto_rawDescGZIP(), []int{27} } func (x *GetTableSchemaBatchInput) GetPeerConnectionConfig() *Peer { @@ -1828,7 +1940,7 @@ type GetTableSchemaBatchOutput struct { func (x *GetTableSchemaBatchOutput) Reset() { *x = GetTableSchemaBatchOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[26] + mi := &file_flow_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1841,7 +1953,7 @@ func (x *GetTableSchemaBatchOutput) String() string { func (*GetTableSchemaBatchOutput) ProtoMessage() {} func (x *GetTableSchemaBatchOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[26] + mi := &file_flow_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1854,7 +1966,7 @@ func (x *GetTableSchemaBatchOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use GetTableSchemaBatchOutput.ProtoReflect.Descriptor instead. func (*GetTableSchemaBatchOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{26} + return file_flow_proto_rawDescGZIP(), []int{28} } func (x *GetTableSchemaBatchOutput) GetTableNameSchemaMapping() map[string]*TableSchema { @@ -1877,7 +1989,7 @@ type SetupNormalizedTableInput struct { func (x *SetupNormalizedTableInput) Reset() { *x = SetupNormalizedTableInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[27] + mi := &file_flow_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1890,7 +2002,7 @@ func (x *SetupNormalizedTableInput) String() string { func (*SetupNormalizedTableInput) ProtoMessage() {} func (x *SetupNormalizedTableInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[27] + mi := &file_flow_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1903,7 +2015,7 @@ func (x *SetupNormalizedTableInput) ProtoReflect() protoreflect.Message { // Deprecated: Use SetupNormalizedTableInput.ProtoReflect.Descriptor instead. func (*SetupNormalizedTableInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{27} + return file_flow_proto_rawDescGZIP(), []int{29} } func (x *SetupNormalizedTableInput) GetPeerConnectionConfig() *Peer { @@ -1939,7 +2051,7 @@ type SetupNormalizedTableBatchInput struct { func (x *SetupNormalizedTableBatchInput) Reset() { *x = SetupNormalizedTableBatchInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[28] + mi := &file_flow_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1952,7 +2064,7 @@ func (x *SetupNormalizedTableBatchInput) String() string { func (*SetupNormalizedTableBatchInput) ProtoMessage() {} func (x *SetupNormalizedTableBatchInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[28] + mi := &file_flow_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1965,7 +2077,7 @@ func (x *SetupNormalizedTableBatchInput) ProtoReflect() protoreflect.Message { // Deprecated: Use SetupNormalizedTableBatchInput.ProtoReflect.Descriptor instead. func (*SetupNormalizedTableBatchInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{28} + return file_flow_proto_rawDescGZIP(), []int{30} } func (x *SetupNormalizedTableBatchInput) GetPeerConnectionConfig() *Peer { @@ -1994,7 +2106,7 @@ type SetupNormalizedTableOutput struct { func (x *SetupNormalizedTableOutput) Reset() { *x = SetupNormalizedTableOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[29] + mi := &file_flow_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2007,7 +2119,7 @@ func (x *SetupNormalizedTableOutput) String() string { func (*SetupNormalizedTableOutput) ProtoMessage() {} func (x *SetupNormalizedTableOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[29] + mi := &file_flow_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2020,7 +2132,7 @@ func (x *SetupNormalizedTableOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use SetupNormalizedTableOutput.ProtoReflect.Descriptor instead. func (*SetupNormalizedTableOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{29} + return file_flow_proto_rawDescGZIP(), []int{31} } func (x *SetupNormalizedTableOutput) GetTableIdentifier() string { @@ -2048,7 +2160,7 @@ type SetupNormalizedTableBatchOutput struct { func (x *SetupNormalizedTableBatchOutput) Reset() { *x = SetupNormalizedTableBatchOutput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[30] + mi := &file_flow_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2061,7 +2173,7 @@ func (x *SetupNormalizedTableBatchOutput) String() string { func (*SetupNormalizedTableBatchOutput) ProtoMessage() {} func (x *SetupNormalizedTableBatchOutput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[30] + mi := &file_flow_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2074,7 +2186,7 @@ func (x *SetupNormalizedTableBatchOutput) ProtoReflect() protoreflect.Message { // Deprecated: Use SetupNormalizedTableBatchOutput.ProtoReflect.Descriptor instead. func (*SetupNormalizedTableBatchOutput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{30} + return file_flow_proto_rawDescGZIP(), []int{32} } func (x *SetupNormalizedTableBatchOutput) GetTableExistsMapping() map[string]bool { @@ -2097,7 +2209,7 @@ type IntPartitionRange struct { func (x *IntPartitionRange) Reset() { *x = IntPartitionRange{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[31] + mi := &file_flow_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2110,7 +2222,7 @@ func (x *IntPartitionRange) String() string { func (*IntPartitionRange) ProtoMessage() {} func (x *IntPartitionRange) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[31] + mi := &file_flow_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2123,7 +2235,7 @@ func (x *IntPartitionRange) ProtoReflect() protoreflect.Message { // Deprecated: Use IntPartitionRange.ProtoReflect.Descriptor instead. func (*IntPartitionRange) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{31} + return file_flow_proto_rawDescGZIP(), []int{33} } func (x *IntPartitionRange) GetStart() int64 { @@ -2152,7 +2264,7 @@ type TimestampPartitionRange struct { func (x *TimestampPartitionRange) Reset() { *x = TimestampPartitionRange{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[32] + mi := &file_flow_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2165,7 +2277,7 @@ func (x *TimestampPartitionRange) String() string { func (*TimestampPartitionRange) ProtoMessage() {} func (x *TimestampPartitionRange) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[32] + mi := &file_flow_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2178,7 +2290,7 @@ func (x *TimestampPartitionRange) ProtoReflect() protoreflect.Message { // Deprecated: Use TimestampPartitionRange.ProtoReflect.Descriptor instead. func (*TimestampPartitionRange) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{32} + return file_flow_proto_rawDescGZIP(), []int{34} } func (x *TimestampPartitionRange) GetStart() *timestamppb.Timestamp { @@ -2207,7 +2319,7 @@ type TID struct { func (x *TID) Reset() { *x = TID{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[33] + mi := &file_flow_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2220,7 +2332,7 @@ func (x *TID) String() string { func (*TID) ProtoMessage() {} func (x *TID) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[33] + mi := &file_flow_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2233,7 +2345,7 @@ func (x *TID) ProtoReflect() protoreflect.Message { // Deprecated: Use TID.ProtoReflect.Descriptor instead. func (*TID) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{33} + return file_flow_proto_rawDescGZIP(), []int{35} } func (x *TID) GetBlockNumber() uint32 { @@ -2262,7 +2374,7 @@ type TIDPartitionRange struct { func (x *TIDPartitionRange) Reset() { *x = TIDPartitionRange{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[34] + mi := &file_flow_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2275,7 +2387,7 @@ func (x *TIDPartitionRange) String() string { func (*TIDPartitionRange) ProtoMessage() {} func (x *TIDPartitionRange) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[34] + mi := &file_flow_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2288,7 +2400,7 @@ func (x *TIDPartitionRange) ProtoReflect() protoreflect.Message { // Deprecated: Use TIDPartitionRange.ProtoReflect.Descriptor instead. func (*TIDPartitionRange) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{34} + return file_flow_proto_rawDescGZIP(), []int{36} } func (x *TIDPartitionRange) GetStart() *TID { @@ -2323,7 +2435,7 @@ type PartitionRange struct { func (x *PartitionRange) Reset() { *x = PartitionRange{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[35] + mi := &file_flow_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2336,7 +2448,7 @@ func (x *PartitionRange) String() string { func (*PartitionRange) ProtoMessage() {} func (x *PartitionRange) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[35] + mi := &file_flow_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2349,7 +2461,7 @@ func (x *PartitionRange) ProtoReflect() protoreflect.Message { // Deprecated: Use PartitionRange.ProtoReflect.Descriptor instead. func (*PartitionRange) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{35} + return file_flow_proto_rawDescGZIP(), []int{37} } func (m *PartitionRange) GetRange() isPartitionRange_Range { @@ -2414,7 +2526,7 @@ type QRepWriteMode struct { func (x *QRepWriteMode) Reset() { *x = QRepWriteMode{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[36] + mi := &file_flow_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2427,7 +2539,7 @@ func (x *QRepWriteMode) String() string { func (*QRepWriteMode) ProtoMessage() {} func (x *QRepWriteMode) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[36] + mi := &file_flow_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2440,7 +2552,7 @@ func (x *QRepWriteMode) ProtoReflect() protoreflect.Message { // Deprecated: Use QRepWriteMode.ProtoReflect.Descriptor instead. func (*QRepWriteMode) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{36} + return file_flow_proto_rawDescGZIP(), []int{38} } func (x *QRepWriteMode) GetWriteType() QRepWriteType { @@ -2492,12 +2604,15 @@ type QRepConfig struct { NumRowsPerPartition uint32 `protobuf:"varint,16,opt,name=num_rows_per_partition,json=numRowsPerPartition,proto3" json:"num_rows_per_partition,omitempty"` // Creates the watermark table on the destination as-is, can be used for some queries. SetupWatermarkTableOnDestination bool `protobuf:"varint,17,opt,name=setup_watermark_table_on_destination,json=setupWatermarkTableOnDestination,proto3" json:"setup_watermark_table_on_destination,omitempty"` + // create new tables with "_peerdb_resync" suffix, perform initial load and then swap the new table with the old ones + // to be used after the old mirror is dropped + DstTableFullResync bool `protobuf:"varint,18,opt,name=dst_table_full_resync,json=dstTableFullResync,proto3" json:"dst_table_full_resync,omitempty"` } func (x *QRepConfig) Reset() { *x = QRepConfig{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[37] + mi := &file_flow_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2510,7 +2625,7 @@ func (x *QRepConfig) String() string { func (*QRepConfig) ProtoMessage() {} func (x *QRepConfig) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[37] + mi := &file_flow_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2523,7 +2638,7 @@ func (x *QRepConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use QRepConfig.ProtoReflect.Descriptor instead. func (*QRepConfig) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{37} + return file_flow_proto_rawDescGZIP(), []int{39} } func (x *QRepConfig) GetFlowJobName() string { @@ -2645,6 +2760,13 @@ func (x *QRepConfig) GetSetupWatermarkTableOnDestination() bool { return false } +func (x *QRepConfig) GetDstTableFullResync() bool { + if x != nil { + return x.DstTableFullResync + } + return false +} + type QRepPartition struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2658,7 +2780,7 @@ type QRepPartition struct { func (x *QRepPartition) Reset() { *x = QRepPartition{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[38] + mi := &file_flow_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2671,7 +2793,7 @@ func (x *QRepPartition) String() string { func (*QRepPartition) ProtoMessage() {} func (x *QRepPartition) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[38] + mi := &file_flow_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2684,7 +2806,7 @@ func (x *QRepPartition) ProtoReflect() protoreflect.Message { // Deprecated: Use QRepPartition.ProtoReflect.Descriptor instead. func (*QRepPartition) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{38} + return file_flow_proto_rawDescGZIP(), []int{40} } func (x *QRepPartition) GetPartitionId() string { @@ -2720,7 +2842,7 @@ type QRepPartitionBatch struct { func (x *QRepPartitionBatch) Reset() { *x = QRepPartitionBatch{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[39] + mi := &file_flow_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2733,7 +2855,7 @@ func (x *QRepPartitionBatch) String() string { func (*QRepPartitionBatch) ProtoMessage() {} func (x *QRepPartitionBatch) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[39] + mi := &file_flow_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2746,7 +2868,7 @@ func (x *QRepPartitionBatch) ProtoReflect() protoreflect.Message { // Deprecated: Use QRepPartitionBatch.ProtoReflect.Descriptor instead. func (*QRepPartitionBatch) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{39} + return file_flow_proto_rawDescGZIP(), []int{41} } func (x *QRepPartitionBatch) GetBatchId() int32 { @@ -2774,7 +2896,7 @@ type QRepParitionResult struct { func (x *QRepParitionResult) Reset() { *x = QRepParitionResult{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[40] + mi := &file_flow_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2787,7 +2909,7 @@ func (x *QRepParitionResult) String() string { func (*QRepParitionResult) ProtoMessage() {} func (x *QRepParitionResult) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[40] + mi := &file_flow_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2800,7 +2922,7 @@ func (x *QRepParitionResult) ProtoReflect() protoreflect.Message { // Deprecated: Use QRepParitionResult.ProtoReflect.Descriptor instead. func (*QRepParitionResult) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{40} + return file_flow_proto_rawDescGZIP(), []int{42} } func (x *QRepParitionResult) GetPartitions() []*QRepPartition { @@ -2821,7 +2943,7 @@ type DropFlowInput struct { func (x *DropFlowInput) Reset() { *x = DropFlowInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[41] + mi := &file_flow_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2834,7 +2956,7 @@ func (x *DropFlowInput) String() string { func (*DropFlowInput) ProtoMessage() {} func (x *DropFlowInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[41] + mi := &file_flow_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2847,7 +2969,7 @@ func (x *DropFlowInput) ProtoReflect() protoreflect.Message { // Deprecated: Use DropFlowInput.ProtoReflect.Descriptor instead. func (*DropFlowInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{41} + return file_flow_proto_rawDescGZIP(), []int{43} } func (x *DropFlowInput) GetFlowName() string { @@ -2869,7 +2991,7 @@ type DeltaAddedColumn struct { func (x *DeltaAddedColumn) Reset() { *x = DeltaAddedColumn{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[42] + mi := &file_flow_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2882,7 +3004,7 @@ func (x *DeltaAddedColumn) String() string { func (*DeltaAddedColumn) ProtoMessage() {} func (x *DeltaAddedColumn) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[42] + mi := &file_flow_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2895,7 +3017,7 @@ func (x *DeltaAddedColumn) ProtoReflect() protoreflect.Message { // Deprecated: Use DeltaAddedColumn.ProtoReflect.Descriptor instead. func (*DeltaAddedColumn) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{42} + return file_flow_proto_rawDescGZIP(), []int{44} } func (x *DeltaAddedColumn) GetColumnName() string { @@ -2925,7 +3047,7 @@ type TableSchemaDelta struct { func (x *TableSchemaDelta) Reset() { *x = TableSchemaDelta{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[43] + mi := &file_flow_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2938,7 +3060,7 @@ func (x *TableSchemaDelta) String() string { func (*TableSchemaDelta) ProtoMessage() {} func (x *TableSchemaDelta) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[43] + mi := &file_flow_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2951,7 +3073,7 @@ func (x *TableSchemaDelta) ProtoReflect() protoreflect.Message { // Deprecated: Use TableSchemaDelta.ProtoReflect.Descriptor instead. func (*TableSchemaDelta) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{43} + return file_flow_proto_rawDescGZIP(), []int{45} } func (x *TableSchemaDelta) GetSrcTableName() string { @@ -2987,7 +3109,7 @@ type ReplayTableSchemaDeltaInput struct { func (x *ReplayTableSchemaDeltaInput) Reset() { *x = ReplayTableSchemaDeltaInput{} if protoimpl.UnsafeEnabled { - mi := &file_flow_proto_msgTypes[44] + mi := &file_flow_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3000,7 +3122,7 @@ func (x *ReplayTableSchemaDeltaInput) String() string { func (*ReplayTableSchemaDeltaInput) ProtoMessage() {} func (x *ReplayTableSchemaDeltaInput) ProtoReflect() protoreflect.Message { - mi := &file_flow_proto_msgTypes[44] + mi := &file_flow_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3013,7 +3135,7 @@ func (x *ReplayTableSchemaDeltaInput) ProtoReflect() protoreflect.Message { // Deprecated: Use ReplayTableSchemaDeltaInput.ProtoReflect.Descriptor instead. func (*ReplayTableSchemaDeltaInput) Descriptor() ([]byte, []int) { - return file_flow_proto_rawDescGZIP(), []int{44} + return file_flow_proto_rawDescGZIP(), []int{46} } func (x *ReplayTableSchemaDeltaInput) GetFlowConnectionConfigs() *FlowConnectionConfigs { @@ -3030,6 +3152,69 @@ func (x *ReplayTableSchemaDeltaInput) GetTableSchemaDeltas() []*TableSchemaDelta return nil } +type QRepFlowState struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + LastPartition *QRepPartition `protobuf:"bytes,1,opt,name=last_partition,json=lastPartition,proto3" json:"last_partition,omitempty"` + NumPartitionsProcessed uint64 `protobuf:"varint,2,opt,name=num_partitions_processed,json=numPartitionsProcessed,proto3" json:"num_partitions_processed,omitempty"` + NeedsResync bool `protobuf:"varint,3,opt,name=needs_resync,json=needsResync,proto3" json:"needs_resync,omitempty"` +} + +func (x *QRepFlowState) Reset() { + *x = QRepFlowState{} + if protoimpl.UnsafeEnabled { + mi := &file_flow_proto_msgTypes[47] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QRepFlowState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QRepFlowState) ProtoMessage() {} + +func (x *QRepFlowState) ProtoReflect() protoreflect.Message { + mi := &file_flow_proto_msgTypes[47] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QRepFlowState.ProtoReflect.Descriptor instead. +func (*QRepFlowState) Descriptor() ([]byte, []int) { + return file_flow_proto_rawDescGZIP(), []int{47} +} + +func (x *QRepFlowState) GetLastPartition() *QRepPartition { + if x != nil { + return x.LastPartition + } + return nil +} + +func (x *QRepFlowState) GetNumPartitionsProcessed() uint64 { + if x != nil { + return x.NumPartitionsProcessed + } + return 0 +} + +func (x *QRepFlowState) GetNeedsResync() bool { + if x != nil { + return x.NeedsResync + } + return false +} + var File_flow_proto protoreflect.FileDescriptor var file_flow_proto_rawDesc = []byte{ @@ -3186,6 +3371,31 @@ var file_flow_proto_rawDesc = []byte{ 0x61, 0x62, 0x6c, 0x65, 0x73, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x22, 0x0a, 0x0d, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x6c, 0x6f, 0x77, 0x4a, 0x6f, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x22, + 0xc7, 0x02, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, + 0x46, 0x72, 0x6f, 0x6d, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x12, 0x22, 0x0a, 0x0d, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x6c, 0x6f, 0x77, 0x4a, 0x6f, + 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x04, 0x70, 0x65, 0x65, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x70, 0x65, 0x65, + 0x72, 0x73, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x52, 0x04, 0x70, 0x65, 0x65, 0x72, 0x12, 0x8b, 0x01, + 0x0a, 0x1d, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x6f, 0x5f, 0x65, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, + 0x6c, 0x6f, 0x77, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, + 0x46, 0x72, 0x6f, 0x6d, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, + 0x74, 0x2e, 0x4e, 0x65, 0x77, 0x54, 0x6f, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x19, 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x1a, 0x4c, 0x0a, 0x1e, 0x4e, + 0x65, 0x77, 0x54, 0x6f, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x44, 0x0a, 0x1e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x45, 0x78, 0x69, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x22, 0x0a, 0x0d, 0x66, + 0x6c, 0x6f, 0x77, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x66, 0x6c, 0x6f, 0x77, 0x4a, 0x6f, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x8d, 0x02, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x46, 0x6c, 0x6f, 0x77, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x69, @@ -3529,7 +3739,7 @@ var file_flow_proto_rawDesc = []byte{ 0x52, 0x09, 0x77, 0x72, 0x69, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x75, 0x70, 0x73, 0x65, 0x72, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x75, 0x70, 0x73, 0x65, 0x72, 0x74, 0x4b, - 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x22, 0xe6, 0x06, 0x0a, 0x0a, 0x51, 0x52, + 0x65, 0x79, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x22, 0x99, 0x07, 0x0a, 0x0a, 0x51, 0x52, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x22, 0x0a, 0x0d, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x6c, 0x6f, 0x77, 0x4a, 0x6f, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x33, 0x0a, 0x0b, @@ -3584,79 +3794,93 @@ var file_flow_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x20, 0x73, 0x65, 0x74, 0x75, 0x70, 0x57, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x6e, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x97, 0x01, 0x0a, 0x0d, 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, - 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x61, - 0x6e, 0x67, 0x65, 0x52, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x66, 0x75, - 0x6c, 0x6c, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x66, 0x75, 0x6c, 0x6c, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x6b, 0x0a, 0x12, - 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x62, 0x61, 0x74, 0x63, 0x68, 0x49, 0x64, 0x12, 0x3a, 0x0a, - 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, - 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x50, 0x0a, 0x12, 0x51, 0x52, 0x65, - 0x70, 0x50, 0x61, 0x72, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, - 0x3a, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, - 0x77, 0x2e, 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2c, 0x0a, 0x0d, 0x44, - 0x72, 0x6f, 0x70, 0x46, 0x6c, 0x6f, 0x77, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x1b, 0x0a, 0x09, - 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x66, 0x6c, 0x6f, 0x77, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x54, 0x0a, 0x10, 0x44, 0x65, 0x6c, - 0x74, 0x61, 0x41, 0x64, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x1f, 0x0a, - 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, - 0xa2, 0x01, 0x0a, 0x10, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, - 0x65, 0x6c, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x72, 0x63, 0x5f, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x72, - 0x63, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x64, 0x73, - 0x74, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0c, 0x64, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x42, 0x0a, 0x0d, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, - 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x41, 0x64, 0x64, 0x65, 0x64, - 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x0c, 0x61, 0x64, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x73, 0x22, 0xc8, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x79, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x49, - 0x6e, 0x70, 0x75, 0x74, 0x12, 0x5a, 0x0a, 0x17, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, - 0x6c, 0x6f, 0x77, 0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x15, 0x66, 0x6c, 0x6f, 0x77, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, - 0x12, 0x4d, 0x0a, 0x13, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x52, 0x11, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x73, 0x2a, - 0x50, 0x0a, 0x0c, 0x51, 0x52, 0x65, 0x70, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x6f, 0x64, 0x65, 0x12, - 0x1f, 0x0a, 0x1b, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4d, 0x4f, 0x44, - 0x45, 0x5f, 0x4d, 0x55, 0x4c, 0x54, 0x49, 0x5f, 0x49, 0x4e, 0x53, 0x45, 0x52, 0x54, 0x10, 0x00, - 0x12, 0x1f, 0x0a, 0x1b, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4d, 0x4f, - 0x44, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, - 0x01, 0x2a, 0x66, 0x0a, 0x0d, 0x51, 0x52, 0x65, 0x70, 0x57, 0x72, 0x69, 0x74, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, - 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x1a, - 0x0a, 0x16, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x4d, 0x4f, 0x44, - 0x45, 0x5f, 0x55, 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x51, 0x52, - 0x45, 0x50, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x4f, 0x56, - 0x45, 0x52, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x02, 0x42, 0x76, 0x0a, 0x0f, 0x63, 0x6f, 0x6d, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x42, 0x09, 0x46, 0x6c, - 0x6f, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x10, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0xa2, 0x02, 0x03, 0x50, 0x58, - 0x58, 0xaa, 0x02, 0x0a, 0x50, 0x65, 0x65, 0x72, 0x64, 0x62, 0x46, 0x6c, 0x6f, 0x77, 0xca, 0x02, - 0x0a, 0x50, 0x65, 0x65, 0x72, 0x64, 0x62, 0x46, 0x6c, 0x6f, 0x77, 0xe2, 0x02, 0x16, 0x50, 0x65, - 0x65, 0x72, 0x64, 0x62, 0x46, 0x6c, 0x6f, 0x77, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0a, 0x50, 0x65, 0x65, 0x72, 0x64, 0x62, 0x46, 0x6c, 0x6f, - 0x77, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x15, 0x64, 0x73, 0x74, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, + 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x12, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x12, 0x64, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x75, 0x6c, 0x6c, 0x52, + 0x65, 0x73, 0x79, 0x6e, 0x63, 0x22, 0x97, 0x01, 0x0a, 0x0d, 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x05, 0x72, 0x61, + 0x6e, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, + 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x05, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x30, 0x0a, + 0x14, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x66, 0x75, 0x6c, + 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x6b, 0x0a, 0x12, 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x62, 0x61, 0x74, 0x63, 0x68, 0x49, 0x64, + 0x12, 0x3a, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, + 0x6f, 0x77, 0x2e, 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x50, 0x0a, 0x12, + 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, + 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x51, 0x52, 0x65, 0x70, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x2c, + 0x0a, 0x0d, 0x44, 0x72, 0x6f, 0x70, 0x46, 0x6c, 0x6f, 0x77, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x66, 0x6c, 0x6f, 0x77, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x54, 0x0a, 0x10, + 0x44, 0x65, 0x6c, 0x74, 0x61, 0x41, 0x64, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x22, 0xa2, 0x01, 0x0a, 0x10, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x72, 0x63, 0x5f, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x73, 0x72, 0x63, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, + 0x0e, 0x64, 0x73, 0x74, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x42, 0x0a, 0x0d, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x70, 0x65, 0x65, + 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x41, 0x64, + 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x52, 0x0c, 0x61, 0x64, 0x64, 0x65, 0x64, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x22, 0xc8, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x70, 0x6c, + 0x61, 0x79, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, 0x65, 0x6c, + 0x74, 0x61, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x5a, 0x0a, 0x17, 0x66, 0x6c, 0x6f, 0x77, 0x5f, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, + 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x46, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x15, 0x66, 0x6c, + 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x12, 0x4d, 0x0a, 0x13, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x52, + 0x11, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x44, 0x65, 0x6c, 0x74, + 0x61, 0x73, 0x22, 0xaf, 0x01, 0x0a, 0x0d, 0x51, 0x52, 0x65, 0x70, 0x46, 0x6c, 0x6f, 0x77, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, + 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x51, 0x52, 0x65, 0x70, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x18, 0x6e, 0x75, 0x6d, 0x5f, 0x70, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, + 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x6e, 0x75, 0x6d, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, + 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x79, 0x6e, + 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x52, 0x65, + 0x73, 0x79, 0x6e, 0x63, 0x2a, 0x50, 0x0a, 0x0c, 0x51, 0x52, 0x65, 0x70, 0x53, 0x79, 0x6e, 0x63, + 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x53, 0x59, 0x4e, + 0x43, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x4d, 0x55, 0x4c, 0x54, 0x49, 0x5f, 0x49, 0x4e, 0x53, + 0x45, 0x52, 0x54, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x53, 0x59, + 0x4e, 0x43, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, + 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x2a, 0x66, 0x0a, 0x0d, 0x51, 0x52, 0x65, 0x70, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x51, 0x52, 0x45, 0x50, 0x5f, + 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, + 0x44, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x57, 0x52, 0x49, 0x54, + 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x50, 0x53, 0x45, 0x52, 0x54, 0x10, 0x01, 0x12, + 0x1d, 0x0a, 0x19, 0x51, 0x52, 0x45, 0x50, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x4d, 0x4f, + 0x44, 0x45, 0x5f, 0x4f, 0x56, 0x45, 0x52, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x02, 0x42, 0x76, + 0x0a, 0x0f, 0x63, 0x6f, 0x6d, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x64, 0x62, 0x5f, 0x66, 0x6c, 0x6f, + 0x77, 0x42, 0x09, 0x46, 0x6c, 0x6f, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x10, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, + 0xa2, 0x02, 0x03, 0x50, 0x58, 0x58, 0xaa, 0x02, 0x0a, 0x50, 0x65, 0x65, 0x72, 0x64, 0x62, 0x46, + 0x6c, 0x6f, 0x77, 0xca, 0x02, 0x0a, 0x50, 0x65, 0x65, 0x72, 0x64, 0x62, 0x46, 0x6c, 0x6f, 0x77, + 0xe2, 0x02, 0x16, 0x50, 0x65, 0x65, 0x72, 0x64, 0x62, 0x46, 0x6c, 0x6f, 0x77, 0x5c, 0x47, 0x50, + 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0a, 0x50, 0x65, 0x65, 0x72, + 0x64, 0x62, 0x46, 0x6c, 0x6f, 0x77, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3672,7 +3896,7 @@ func file_flow_proto_rawDescGZIP() []byte { } var file_flow_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_flow_proto_msgTypes = make([]protoimpl.MessageInfo, 56) +var file_flow_proto_msgTypes = make([]protoimpl.MessageInfo, 60) var file_flow_proto_goTypes = []interface{}{ (QRepSyncMode)(0), // 0: peerdb_flow.QRepSyncMode (QRepWriteType)(0), // 1: peerdb_flow.QRepWriteType @@ -3684,126 +3908,133 @@ var file_flow_proto_goTypes = []interface{}{ (*RenameTableOption)(nil), // 7: peerdb_flow.RenameTableOption (*RenameTablesInput)(nil), // 8: peerdb_flow.RenameTablesInput (*RenameTablesOutput)(nil), // 9: peerdb_flow.RenameTablesOutput - (*SyncFlowOptions)(nil), // 10: peerdb_flow.SyncFlowOptions - (*NormalizeFlowOptions)(nil), // 11: peerdb_flow.NormalizeFlowOptions - (*LastSyncState)(nil), // 12: peerdb_flow.LastSyncState - (*StartFlowInput)(nil), // 13: peerdb_flow.StartFlowInput - (*StartNormalizeInput)(nil), // 14: peerdb_flow.StartNormalizeInput - (*GetLastSyncedIDInput)(nil), // 15: peerdb_flow.GetLastSyncedIDInput - (*EnsurePullabilityInput)(nil), // 16: peerdb_flow.EnsurePullabilityInput - (*EnsurePullabilityBatchInput)(nil), // 17: peerdb_flow.EnsurePullabilityBatchInput - (*PostgresTableIdentifier)(nil), // 18: peerdb_flow.PostgresTableIdentifier - (*TableIdentifier)(nil), // 19: peerdb_flow.TableIdentifier - (*EnsurePullabilityOutput)(nil), // 20: peerdb_flow.EnsurePullabilityOutput - (*EnsurePullabilityBatchOutput)(nil), // 21: peerdb_flow.EnsurePullabilityBatchOutput - (*SetupReplicationInput)(nil), // 22: peerdb_flow.SetupReplicationInput - (*SetupReplicationOutput)(nil), // 23: peerdb_flow.SetupReplicationOutput - (*CreateRawTableInput)(nil), // 24: peerdb_flow.CreateRawTableInput - (*CreateRawTableOutput)(nil), // 25: peerdb_flow.CreateRawTableOutput - (*TableSchema)(nil), // 26: peerdb_flow.TableSchema - (*GetTableSchemaBatchInput)(nil), // 27: peerdb_flow.GetTableSchemaBatchInput - (*GetTableSchemaBatchOutput)(nil), // 28: peerdb_flow.GetTableSchemaBatchOutput - (*SetupNormalizedTableInput)(nil), // 29: peerdb_flow.SetupNormalizedTableInput - (*SetupNormalizedTableBatchInput)(nil), // 30: peerdb_flow.SetupNormalizedTableBatchInput - (*SetupNormalizedTableOutput)(nil), // 31: peerdb_flow.SetupNormalizedTableOutput - (*SetupNormalizedTableBatchOutput)(nil), // 32: peerdb_flow.SetupNormalizedTableBatchOutput - (*IntPartitionRange)(nil), // 33: peerdb_flow.IntPartitionRange - (*TimestampPartitionRange)(nil), // 34: peerdb_flow.TimestampPartitionRange - (*TID)(nil), // 35: peerdb_flow.TID - (*TIDPartitionRange)(nil), // 36: peerdb_flow.TIDPartitionRange - (*PartitionRange)(nil), // 37: peerdb_flow.PartitionRange - (*QRepWriteMode)(nil), // 38: peerdb_flow.QRepWriteMode - (*QRepConfig)(nil), // 39: peerdb_flow.QRepConfig - (*QRepPartition)(nil), // 40: peerdb_flow.QRepPartition - (*QRepPartitionBatch)(nil), // 41: peerdb_flow.QRepPartitionBatch - (*QRepParitionResult)(nil), // 42: peerdb_flow.QRepParitionResult - (*DropFlowInput)(nil), // 43: peerdb_flow.DropFlowInput - (*DeltaAddedColumn)(nil), // 44: peerdb_flow.DeltaAddedColumn - (*TableSchemaDelta)(nil), // 45: peerdb_flow.TableSchemaDelta - (*ReplayTableSchemaDeltaInput)(nil), // 46: peerdb_flow.ReplayTableSchemaDeltaInput - nil, // 47: peerdb_flow.FlowConnectionConfigs.SrcTableIdNameMappingEntry - nil, // 48: peerdb_flow.FlowConnectionConfigs.TableNameSchemaMappingEntry - nil, // 49: peerdb_flow.SyncFlowOptions.RelationMessageMappingEntry - nil, // 50: peerdb_flow.StartFlowInput.RelationMessageMappingEntry - nil, // 51: peerdb_flow.EnsurePullabilityBatchOutput.TableIdentifierMappingEntry - nil, // 52: peerdb_flow.SetupReplicationInput.TableNameMappingEntry - nil, // 53: peerdb_flow.CreateRawTableInput.TableNameMappingEntry - nil, // 54: peerdb_flow.TableSchema.ColumnsEntry - nil, // 55: peerdb_flow.GetTableSchemaBatchOutput.TableNameSchemaMappingEntry - nil, // 56: peerdb_flow.SetupNormalizedTableBatchInput.TableNameSchemaMappingEntry - nil, // 57: peerdb_flow.SetupNormalizedTableBatchOutput.TableExistsMappingEntry - (*Peer)(nil), // 58: peerdb_peers.Peer - (*timestamppb.Timestamp)(nil), // 59: google.protobuf.Timestamp + (*CreateTablesFromExistingInput)(nil), // 10: peerdb_flow.CreateTablesFromExistingInput + (*CreateTablesFromExistingOutput)(nil), // 11: peerdb_flow.CreateTablesFromExistingOutput + (*SyncFlowOptions)(nil), // 12: peerdb_flow.SyncFlowOptions + (*NormalizeFlowOptions)(nil), // 13: peerdb_flow.NormalizeFlowOptions + (*LastSyncState)(nil), // 14: peerdb_flow.LastSyncState + (*StartFlowInput)(nil), // 15: peerdb_flow.StartFlowInput + (*StartNormalizeInput)(nil), // 16: peerdb_flow.StartNormalizeInput + (*GetLastSyncedIDInput)(nil), // 17: peerdb_flow.GetLastSyncedIDInput + (*EnsurePullabilityInput)(nil), // 18: peerdb_flow.EnsurePullabilityInput + (*EnsurePullabilityBatchInput)(nil), // 19: peerdb_flow.EnsurePullabilityBatchInput + (*PostgresTableIdentifier)(nil), // 20: peerdb_flow.PostgresTableIdentifier + (*TableIdentifier)(nil), // 21: peerdb_flow.TableIdentifier + (*EnsurePullabilityOutput)(nil), // 22: peerdb_flow.EnsurePullabilityOutput + (*EnsurePullabilityBatchOutput)(nil), // 23: peerdb_flow.EnsurePullabilityBatchOutput + (*SetupReplicationInput)(nil), // 24: peerdb_flow.SetupReplicationInput + (*SetupReplicationOutput)(nil), // 25: peerdb_flow.SetupReplicationOutput + (*CreateRawTableInput)(nil), // 26: peerdb_flow.CreateRawTableInput + (*CreateRawTableOutput)(nil), // 27: peerdb_flow.CreateRawTableOutput + (*TableSchema)(nil), // 28: peerdb_flow.TableSchema + (*GetTableSchemaBatchInput)(nil), // 29: peerdb_flow.GetTableSchemaBatchInput + (*GetTableSchemaBatchOutput)(nil), // 30: peerdb_flow.GetTableSchemaBatchOutput + (*SetupNormalizedTableInput)(nil), // 31: peerdb_flow.SetupNormalizedTableInput + (*SetupNormalizedTableBatchInput)(nil), // 32: peerdb_flow.SetupNormalizedTableBatchInput + (*SetupNormalizedTableOutput)(nil), // 33: peerdb_flow.SetupNormalizedTableOutput + (*SetupNormalizedTableBatchOutput)(nil), // 34: peerdb_flow.SetupNormalizedTableBatchOutput + (*IntPartitionRange)(nil), // 35: peerdb_flow.IntPartitionRange + (*TimestampPartitionRange)(nil), // 36: peerdb_flow.TimestampPartitionRange + (*TID)(nil), // 37: peerdb_flow.TID + (*TIDPartitionRange)(nil), // 38: peerdb_flow.TIDPartitionRange + (*PartitionRange)(nil), // 39: peerdb_flow.PartitionRange + (*QRepWriteMode)(nil), // 40: peerdb_flow.QRepWriteMode + (*QRepConfig)(nil), // 41: peerdb_flow.QRepConfig + (*QRepPartition)(nil), // 42: peerdb_flow.QRepPartition + (*QRepPartitionBatch)(nil), // 43: peerdb_flow.QRepPartitionBatch + (*QRepParitionResult)(nil), // 44: peerdb_flow.QRepParitionResult + (*DropFlowInput)(nil), // 45: peerdb_flow.DropFlowInput + (*DeltaAddedColumn)(nil), // 46: peerdb_flow.DeltaAddedColumn + (*TableSchemaDelta)(nil), // 47: peerdb_flow.TableSchemaDelta + (*ReplayTableSchemaDeltaInput)(nil), // 48: peerdb_flow.ReplayTableSchemaDeltaInput + (*QRepFlowState)(nil), // 49: peerdb_flow.QRepFlowState + nil, // 50: peerdb_flow.FlowConnectionConfigs.SrcTableIdNameMappingEntry + nil, // 51: peerdb_flow.FlowConnectionConfigs.TableNameSchemaMappingEntry + nil, // 52: peerdb_flow.CreateTablesFromExistingInput.NewToExistingTableMappingEntry + nil, // 53: peerdb_flow.SyncFlowOptions.RelationMessageMappingEntry + nil, // 54: peerdb_flow.StartFlowInput.RelationMessageMappingEntry + nil, // 55: peerdb_flow.EnsurePullabilityBatchOutput.TableIdentifierMappingEntry + nil, // 56: peerdb_flow.SetupReplicationInput.TableNameMappingEntry + nil, // 57: peerdb_flow.CreateRawTableInput.TableNameMappingEntry + nil, // 58: peerdb_flow.TableSchema.ColumnsEntry + nil, // 59: peerdb_flow.GetTableSchemaBatchOutput.TableNameSchemaMappingEntry + nil, // 60: peerdb_flow.SetupNormalizedTableBatchInput.TableNameSchemaMappingEntry + nil, // 61: peerdb_flow.SetupNormalizedTableBatchOutput.TableExistsMappingEntry + (*Peer)(nil), // 62: peerdb_peers.Peer + (*timestamppb.Timestamp)(nil), // 63: google.protobuf.Timestamp } var file_flow_proto_depIdxs = []int32{ 3, // 0: peerdb_flow.RelationMessage.columns:type_name -> peerdb_flow.RelationMessageColumn - 58, // 1: peerdb_flow.FlowConnectionConfigs.source:type_name -> peerdb_peers.Peer - 58, // 2: peerdb_flow.FlowConnectionConfigs.destination:type_name -> peerdb_peers.Peer - 26, // 3: peerdb_flow.FlowConnectionConfigs.table_schema:type_name -> peerdb_flow.TableSchema + 62, // 1: peerdb_flow.FlowConnectionConfigs.source:type_name -> peerdb_peers.Peer + 62, // 2: peerdb_flow.FlowConnectionConfigs.destination:type_name -> peerdb_peers.Peer + 28, // 3: peerdb_flow.FlowConnectionConfigs.table_schema:type_name -> peerdb_flow.TableSchema 5, // 4: peerdb_flow.FlowConnectionConfigs.table_mappings:type_name -> peerdb_flow.TableMapping - 47, // 5: peerdb_flow.FlowConnectionConfigs.src_table_id_name_mapping:type_name -> peerdb_flow.FlowConnectionConfigs.SrcTableIdNameMappingEntry - 48, // 6: peerdb_flow.FlowConnectionConfigs.table_name_schema_mapping:type_name -> peerdb_flow.FlowConnectionConfigs.TableNameSchemaMappingEntry - 58, // 7: peerdb_flow.FlowConnectionConfigs.metadata_peer:type_name -> peerdb_peers.Peer + 50, // 5: peerdb_flow.FlowConnectionConfigs.src_table_id_name_mapping:type_name -> peerdb_flow.FlowConnectionConfigs.SrcTableIdNameMappingEntry + 51, // 6: peerdb_flow.FlowConnectionConfigs.table_name_schema_mapping:type_name -> peerdb_flow.FlowConnectionConfigs.TableNameSchemaMappingEntry + 62, // 7: peerdb_flow.FlowConnectionConfigs.metadata_peer:type_name -> peerdb_peers.Peer 0, // 8: peerdb_flow.FlowConnectionConfigs.snapshot_sync_mode:type_name -> peerdb_flow.QRepSyncMode 0, // 9: peerdb_flow.FlowConnectionConfigs.cdc_sync_mode:type_name -> peerdb_flow.QRepSyncMode - 58, // 10: peerdb_flow.RenameTablesInput.peer:type_name -> peerdb_peers.Peer + 62, // 10: peerdb_flow.RenameTablesInput.peer:type_name -> peerdb_peers.Peer 7, // 11: peerdb_flow.RenameTablesInput.rename_table_options:type_name -> peerdb_flow.RenameTableOption - 49, // 12: peerdb_flow.SyncFlowOptions.relation_message_mapping:type_name -> peerdb_flow.SyncFlowOptions.RelationMessageMappingEntry - 59, // 13: peerdb_flow.LastSyncState.last_synced_at:type_name -> google.protobuf.Timestamp - 12, // 14: peerdb_flow.StartFlowInput.last_sync_state:type_name -> peerdb_flow.LastSyncState - 6, // 15: peerdb_flow.StartFlowInput.flow_connection_configs:type_name -> peerdb_flow.FlowConnectionConfigs - 10, // 16: peerdb_flow.StartFlowInput.sync_flow_options:type_name -> peerdb_flow.SyncFlowOptions - 50, // 17: peerdb_flow.StartFlowInput.relation_message_mapping:type_name -> peerdb_flow.StartFlowInput.RelationMessageMappingEntry - 6, // 18: peerdb_flow.StartNormalizeInput.flow_connection_configs:type_name -> peerdb_flow.FlowConnectionConfigs - 58, // 19: peerdb_flow.GetLastSyncedIDInput.peer_connection_config:type_name -> peerdb_peers.Peer - 58, // 20: peerdb_flow.EnsurePullabilityInput.peer_connection_config:type_name -> peerdb_peers.Peer - 58, // 21: peerdb_flow.EnsurePullabilityBatchInput.peer_connection_config:type_name -> peerdb_peers.Peer - 18, // 22: peerdb_flow.TableIdentifier.postgres_table_identifier:type_name -> peerdb_flow.PostgresTableIdentifier - 19, // 23: peerdb_flow.EnsurePullabilityOutput.table_identifier:type_name -> peerdb_flow.TableIdentifier - 51, // 24: peerdb_flow.EnsurePullabilityBatchOutput.table_identifier_mapping:type_name -> peerdb_flow.EnsurePullabilityBatchOutput.TableIdentifierMappingEntry - 58, // 25: peerdb_flow.SetupReplicationInput.peer_connection_config:type_name -> peerdb_peers.Peer - 52, // 26: peerdb_flow.SetupReplicationInput.table_name_mapping:type_name -> peerdb_flow.SetupReplicationInput.TableNameMappingEntry - 58, // 27: peerdb_flow.SetupReplicationInput.destination_peer:type_name -> peerdb_peers.Peer - 58, // 28: peerdb_flow.CreateRawTableInput.peer_connection_config:type_name -> peerdb_peers.Peer - 53, // 29: peerdb_flow.CreateRawTableInput.table_name_mapping:type_name -> peerdb_flow.CreateRawTableInput.TableNameMappingEntry - 0, // 30: peerdb_flow.CreateRawTableInput.cdc_sync_mode:type_name -> peerdb_flow.QRepSyncMode - 54, // 31: peerdb_flow.TableSchema.columns:type_name -> peerdb_flow.TableSchema.ColumnsEntry - 58, // 32: peerdb_flow.GetTableSchemaBatchInput.peer_connection_config:type_name -> peerdb_peers.Peer - 55, // 33: peerdb_flow.GetTableSchemaBatchOutput.table_name_schema_mapping:type_name -> peerdb_flow.GetTableSchemaBatchOutput.TableNameSchemaMappingEntry - 58, // 34: peerdb_flow.SetupNormalizedTableInput.peer_connection_config:type_name -> peerdb_peers.Peer - 26, // 35: peerdb_flow.SetupNormalizedTableInput.source_table_schema:type_name -> peerdb_flow.TableSchema - 58, // 36: peerdb_flow.SetupNormalizedTableBatchInput.peer_connection_config:type_name -> peerdb_peers.Peer - 56, // 37: peerdb_flow.SetupNormalizedTableBatchInput.table_name_schema_mapping:type_name -> peerdb_flow.SetupNormalizedTableBatchInput.TableNameSchemaMappingEntry - 57, // 38: peerdb_flow.SetupNormalizedTableBatchOutput.table_exists_mapping:type_name -> peerdb_flow.SetupNormalizedTableBatchOutput.TableExistsMappingEntry - 59, // 39: peerdb_flow.TimestampPartitionRange.start:type_name -> google.protobuf.Timestamp - 59, // 40: peerdb_flow.TimestampPartitionRange.end:type_name -> google.protobuf.Timestamp - 35, // 41: peerdb_flow.TIDPartitionRange.start:type_name -> peerdb_flow.TID - 35, // 42: peerdb_flow.TIDPartitionRange.end:type_name -> peerdb_flow.TID - 33, // 43: peerdb_flow.PartitionRange.int_range:type_name -> peerdb_flow.IntPartitionRange - 34, // 44: peerdb_flow.PartitionRange.timestamp_range:type_name -> peerdb_flow.TimestampPartitionRange - 36, // 45: peerdb_flow.PartitionRange.tid_range:type_name -> peerdb_flow.TIDPartitionRange - 1, // 46: peerdb_flow.QRepWriteMode.write_type:type_name -> peerdb_flow.QRepWriteType - 58, // 47: peerdb_flow.QRepConfig.source_peer:type_name -> peerdb_peers.Peer - 58, // 48: peerdb_flow.QRepConfig.destination_peer:type_name -> peerdb_peers.Peer - 0, // 49: peerdb_flow.QRepConfig.sync_mode:type_name -> peerdb_flow.QRepSyncMode - 38, // 50: peerdb_flow.QRepConfig.write_mode:type_name -> peerdb_flow.QRepWriteMode - 37, // 51: peerdb_flow.QRepPartition.range:type_name -> peerdb_flow.PartitionRange - 40, // 52: peerdb_flow.QRepPartitionBatch.partitions:type_name -> peerdb_flow.QRepPartition - 40, // 53: peerdb_flow.QRepParitionResult.partitions:type_name -> peerdb_flow.QRepPartition - 44, // 54: peerdb_flow.TableSchemaDelta.added_columns:type_name -> peerdb_flow.DeltaAddedColumn - 6, // 55: peerdb_flow.ReplayTableSchemaDeltaInput.flow_connection_configs:type_name -> peerdb_flow.FlowConnectionConfigs - 45, // 56: peerdb_flow.ReplayTableSchemaDeltaInput.table_schema_deltas:type_name -> peerdb_flow.TableSchemaDelta - 26, // 57: peerdb_flow.FlowConnectionConfigs.TableNameSchemaMappingEntry.value:type_name -> peerdb_flow.TableSchema - 4, // 58: peerdb_flow.SyncFlowOptions.RelationMessageMappingEntry.value:type_name -> peerdb_flow.RelationMessage - 4, // 59: peerdb_flow.StartFlowInput.RelationMessageMappingEntry.value:type_name -> peerdb_flow.RelationMessage - 19, // 60: peerdb_flow.EnsurePullabilityBatchOutput.TableIdentifierMappingEntry.value:type_name -> peerdb_flow.TableIdentifier - 26, // 61: peerdb_flow.GetTableSchemaBatchOutput.TableNameSchemaMappingEntry.value:type_name -> peerdb_flow.TableSchema - 26, // 62: peerdb_flow.SetupNormalizedTableBatchInput.TableNameSchemaMappingEntry.value:type_name -> peerdb_flow.TableSchema - 63, // [63:63] is the sub-list for method output_type - 63, // [63:63] is the sub-list for method input_type - 63, // [63:63] is the sub-list for extension type_name - 63, // [63:63] is the sub-list for extension extendee - 0, // [0:63] is the sub-list for field type_name + 62, // 12: peerdb_flow.CreateTablesFromExistingInput.peer:type_name -> peerdb_peers.Peer + 52, // 13: peerdb_flow.CreateTablesFromExistingInput.new_to_existing_table_mapping:type_name -> peerdb_flow.CreateTablesFromExistingInput.NewToExistingTableMappingEntry + 53, // 14: peerdb_flow.SyncFlowOptions.relation_message_mapping:type_name -> peerdb_flow.SyncFlowOptions.RelationMessageMappingEntry + 63, // 15: peerdb_flow.LastSyncState.last_synced_at:type_name -> google.protobuf.Timestamp + 14, // 16: peerdb_flow.StartFlowInput.last_sync_state:type_name -> peerdb_flow.LastSyncState + 6, // 17: peerdb_flow.StartFlowInput.flow_connection_configs:type_name -> peerdb_flow.FlowConnectionConfigs + 12, // 18: peerdb_flow.StartFlowInput.sync_flow_options:type_name -> peerdb_flow.SyncFlowOptions + 54, // 19: peerdb_flow.StartFlowInput.relation_message_mapping:type_name -> peerdb_flow.StartFlowInput.RelationMessageMappingEntry + 6, // 20: peerdb_flow.StartNormalizeInput.flow_connection_configs:type_name -> peerdb_flow.FlowConnectionConfigs + 62, // 21: peerdb_flow.GetLastSyncedIDInput.peer_connection_config:type_name -> peerdb_peers.Peer + 62, // 22: peerdb_flow.EnsurePullabilityInput.peer_connection_config:type_name -> peerdb_peers.Peer + 62, // 23: peerdb_flow.EnsurePullabilityBatchInput.peer_connection_config:type_name -> peerdb_peers.Peer + 20, // 24: peerdb_flow.TableIdentifier.postgres_table_identifier:type_name -> peerdb_flow.PostgresTableIdentifier + 21, // 25: peerdb_flow.EnsurePullabilityOutput.table_identifier:type_name -> peerdb_flow.TableIdentifier + 55, // 26: peerdb_flow.EnsurePullabilityBatchOutput.table_identifier_mapping:type_name -> peerdb_flow.EnsurePullabilityBatchOutput.TableIdentifierMappingEntry + 62, // 27: peerdb_flow.SetupReplicationInput.peer_connection_config:type_name -> peerdb_peers.Peer + 56, // 28: peerdb_flow.SetupReplicationInput.table_name_mapping:type_name -> peerdb_flow.SetupReplicationInput.TableNameMappingEntry + 62, // 29: peerdb_flow.SetupReplicationInput.destination_peer:type_name -> peerdb_peers.Peer + 62, // 30: peerdb_flow.CreateRawTableInput.peer_connection_config:type_name -> peerdb_peers.Peer + 57, // 31: peerdb_flow.CreateRawTableInput.table_name_mapping:type_name -> peerdb_flow.CreateRawTableInput.TableNameMappingEntry + 0, // 32: peerdb_flow.CreateRawTableInput.cdc_sync_mode:type_name -> peerdb_flow.QRepSyncMode + 58, // 33: peerdb_flow.TableSchema.columns:type_name -> peerdb_flow.TableSchema.ColumnsEntry + 62, // 34: peerdb_flow.GetTableSchemaBatchInput.peer_connection_config:type_name -> peerdb_peers.Peer + 59, // 35: peerdb_flow.GetTableSchemaBatchOutput.table_name_schema_mapping:type_name -> peerdb_flow.GetTableSchemaBatchOutput.TableNameSchemaMappingEntry + 62, // 36: peerdb_flow.SetupNormalizedTableInput.peer_connection_config:type_name -> peerdb_peers.Peer + 28, // 37: peerdb_flow.SetupNormalizedTableInput.source_table_schema:type_name -> peerdb_flow.TableSchema + 62, // 38: peerdb_flow.SetupNormalizedTableBatchInput.peer_connection_config:type_name -> peerdb_peers.Peer + 60, // 39: peerdb_flow.SetupNormalizedTableBatchInput.table_name_schema_mapping:type_name -> peerdb_flow.SetupNormalizedTableBatchInput.TableNameSchemaMappingEntry + 61, // 40: peerdb_flow.SetupNormalizedTableBatchOutput.table_exists_mapping:type_name -> peerdb_flow.SetupNormalizedTableBatchOutput.TableExistsMappingEntry + 63, // 41: peerdb_flow.TimestampPartitionRange.start:type_name -> google.protobuf.Timestamp + 63, // 42: peerdb_flow.TimestampPartitionRange.end:type_name -> google.protobuf.Timestamp + 37, // 43: peerdb_flow.TIDPartitionRange.start:type_name -> peerdb_flow.TID + 37, // 44: peerdb_flow.TIDPartitionRange.end:type_name -> peerdb_flow.TID + 35, // 45: peerdb_flow.PartitionRange.int_range:type_name -> peerdb_flow.IntPartitionRange + 36, // 46: peerdb_flow.PartitionRange.timestamp_range:type_name -> peerdb_flow.TimestampPartitionRange + 38, // 47: peerdb_flow.PartitionRange.tid_range:type_name -> peerdb_flow.TIDPartitionRange + 1, // 48: peerdb_flow.QRepWriteMode.write_type:type_name -> peerdb_flow.QRepWriteType + 62, // 49: peerdb_flow.QRepConfig.source_peer:type_name -> peerdb_peers.Peer + 62, // 50: peerdb_flow.QRepConfig.destination_peer:type_name -> peerdb_peers.Peer + 0, // 51: peerdb_flow.QRepConfig.sync_mode:type_name -> peerdb_flow.QRepSyncMode + 40, // 52: peerdb_flow.QRepConfig.write_mode:type_name -> peerdb_flow.QRepWriteMode + 39, // 53: peerdb_flow.QRepPartition.range:type_name -> peerdb_flow.PartitionRange + 42, // 54: peerdb_flow.QRepPartitionBatch.partitions:type_name -> peerdb_flow.QRepPartition + 42, // 55: peerdb_flow.QRepParitionResult.partitions:type_name -> peerdb_flow.QRepPartition + 46, // 56: peerdb_flow.TableSchemaDelta.added_columns:type_name -> peerdb_flow.DeltaAddedColumn + 6, // 57: peerdb_flow.ReplayTableSchemaDeltaInput.flow_connection_configs:type_name -> peerdb_flow.FlowConnectionConfigs + 47, // 58: peerdb_flow.ReplayTableSchemaDeltaInput.table_schema_deltas:type_name -> peerdb_flow.TableSchemaDelta + 42, // 59: peerdb_flow.QRepFlowState.last_partition:type_name -> peerdb_flow.QRepPartition + 28, // 60: peerdb_flow.FlowConnectionConfigs.TableNameSchemaMappingEntry.value:type_name -> peerdb_flow.TableSchema + 4, // 61: peerdb_flow.SyncFlowOptions.RelationMessageMappingEntry.value:type_name -> peerdb_flow.RelationMessage + 4, // 62: peerdb_flow.StartFlowInput.RelationMessageMappingEntry.value:type_name -> peerdb_flow.RelationMessage + 21, // 63: peerdb_flow.EnsurePullabilityBatchOutput.TableIdentifierMappingEntry.value:type_name -> peerdb_flow.TableIdentifier + 28, // 64: peerdb_flow.GetTableSchemaBatchOutput.TableNameSchemaMappingEntry.value:type_name -> peerdb_flow.TableSchema + 28, // 65: peerdb_flow.SetupNormalizedTableBatchInput.TableNameSchemaMappingEntry.value:type_name -> peerdb_flow.TableSchema + 66, // [66:66] is the sub-list for method output_type + 66, // [66:66] is the sub-list for method input_type + 66, // [66:66] is the sub-list for extension type_name + 66, // [66:66] is the sub-list for extension extendee + 0, // [0:66] is the sub-list for field type_name } func init() { file_flow_proto_init() } @@ -3910,7 +4141,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SyncFlowOptions); i { + switch v := v.(*CreateTablesFromExistingInput); i { case 0: return &v.state case 1: @@ -3922,7 +4153,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NormalizeFlowOptions); i { + switch v := v.(*CreateTablesFromExistingOutput); i { case 0: return &v.state case 1: @@ -3934,7 +4165,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LastSyncState); i { + switch v := v.(*SyncFlowOptions); i { case 0: return &v.state case 1: @@ -3946,7 +4177,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartFlowInput); i { + switch v := v.(*NormalizeFlowOptions); i { case 0: return &v.state case 1: @@ -3958,7 +4189,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartNormalizeInput); i { + switch v := v.(*LastSyncState); i { case 0: return &v.state case 1: @@ -3970,7 +4201,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetLastSyncedIDInput); i { + switch v := v.(*StartFlowInput); i { case 0: return &v.state case 1: @@ -3982,7 +4213,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EnsurePullabilityInput); i { + switch v := v.(*StartNormalizeInput); i { case 0: return &v.state case 1: @@ -3994,7 +4225,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EnsurePullabilityBatchInput); i { + switch v := v.(*GetLastSyncedIDInput); i { case 0: return &v.state case 1: @@ -4006,7 +4237,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PostgresTableIdentifier); i { + switch v := v.(*EnsurePullabilityInput); i { case 0: return &v.state case 1: @@ -4018,7 +4249,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TableIdentifier); i { + switch v := v.(*EnsurePullabilityBatchInput); i { case 0: return &v.state case 1: @@ -4030,7 +4261,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EnsurePullabilityOutput); i { + switch v := v.(*PostgresTableIdentifier); i { case 0: return &v.state case 1: @@ -4042,7 +4273,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EnsurePullabilityBatchOutput); i { + switch v := v.(*TableIdentifier); i { case 0: return &v.state case 1: @@ -4054,7 +4285,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetupReplicationInput); i { + switch v := v.(*EnsurePullabilityOutput); i { case 0: return &v.state case 1: @@ -4066,7 +4297,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetupReplicationOutput); i { + switch v := v.(*EnsurePullabilityBatchOutput); i { case 0: return &v.state case 1: @@ -4078,7 +4309,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateRawTableInput); i { + switch v := v.(*SetupReplicationInput); i { case 0: return &v.state case 1: @@ -4090,7 +4321,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateRawTableOutput); i { + switch v := v.(*SetupReplicationOutput); i { case 0: return &v.state case 1: @@ -4102,7 +4333,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TableSchema); i { + switch v := v.(*CreateRawTableInput); i { case 0: return &v.state case 1: @@ -4114,7 +4345,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetTableSchemaBatchInput); i { + switch v := v.(*CreateRawTableOutput); i { case 0: return &v.state case 1: @@ -4126,7 +4357,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetTableSchemaBatchOutput); i { + switch v := v.(*TableSchema); i { case 0: return &v.state case 1: @@ -4138,7 +4369,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetupNormalizedTableInput); i { + switch v := v.(*GetTableSchemaBatchInput); i { case 0: return &v.state case 1: @@ -4150,7 +4381,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetupNormalizedTableBatchInput); i { + switch v := v.(*GetTableSchemaBatchOutput); i { case 0: return &v.state case 1: @@ -4162,7 +4393,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetupNormalizedTableOutput); i { + switch v := v.(*SetupNormalizedTableInput); i { case 0: return &v.state case 1: @@ -4174,7 +4405,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetupNormalizedTableBatchOutput); i { + switch v := v.(*SetupNormalizedTableBatchInput); i { case 0: return &v.state case 1: @@ -4186,7 +4417,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IntPartitionRange); i { + switch v := v.(*SetupNormalizedTableOutput); i { case 0: return &v.state case 1: @@ -4198,7 +4429,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimestampPartitionRange); i { + switch v := v.(*SetupNormalizedTableBatchOutput); i { case 0: return &v.state case 1: @@ -4210,7 +4441,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TID); i { + switch v := v.(*IntPartitionRange); i { case 0: return &v.state case 1: @@ -4222,7 +4453,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TIDPartitionRange); i { + switch v := v.(*TimestampPartitionRange); i { case 0: return &v.state case 1: @@ -4234,7 +4465,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PartitionRange); i { + switch v := v.(*TID); i { case 0: return &v.state case 1: @@ -4246,7 +4477,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QRepWriteMode); i { + switch v := v.(*TIDPartitionRange); i { case 0: return &v.state case 1: @@ -4258,7 +4489,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QRepConfig); i { + switch v := v.(*PartitionRange); i { case 0: return &v.state case 1: @@ -4270,7 +4501,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QRepPartition); i { + switch v := v.(*QRepWriteMode); i { case 0: return &v.state case 1: @@ -4282,7 +4513,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QRepPartitionBatch); i { + switch v := v.(*QRepConfig); i { case 0: return &v.state case 1: @@ -4294,7 +4525,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QRepParitionResult); i { + switch v := v.(*QRepPartition); i { case 0: return &v.state case 1: @@ -4306,7 +4537,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropFlowInput); i { + switch v := v.(*QRepPartitionBatch); i { case 0: return &v.state case 1: @@ -4318,7 +4549,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeltaAddedColumn); i { + switch v := v.(*QRepParitionResult); i { case 0: return &v.state case 1: @@ -4330,7 +4561,7 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TableSchemaDelta); i { + switch v := v.(*DropFlowInput); i { case 0: return &v.state case 1: @@ -4342,6 +4573,30 @@ func file_flow_proto_init() { } } file_flow_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeltaAddedColumn); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_flow_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TableSchemaDelta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_flow_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ReplayTableSchemaDeltaInput); i { case 0: return &v.state @@ -4353,11 +4608,23 @@ func file_flow_proto_init() { return nil } } + file_flow_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*QRepFlowState); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } - file_flow_proto_msgTypes[17].OneofWrappers = []interface{}{ + file_flow_proto_msgTypes[19].OneofWrappers = []interface{}{ (*TableIdentifier_PostgresTableIdentifier)(nil), } - file_flow_proto_msgTypes[35].OneofWrappers = []interface{}{ + file_flow_proto_msgTypes[37].OneofWrappers = []interface{}{ (*PartitionRange_IntRange)(nil), (*PartitionRange_TimestampRange)(nil), (*PartitionRange_TidRange)(nil), @@ -4368,7 +4635,7 @@ func file_flow_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_flow_proto_rawDesc, NumEnums: 2, - NumMessages: 56, + NumMessages: 60, NumExtensions: 0, NumServices: 0, }, diff --git a/flow/workflows/qrep_flow.go b/flow/workflows/qrep_flow.go index 9a5a8641e8..2be1856664 100644 --- a/flow/workflows/qrep_flow.go +++ b/flow/workflows/qrep_flow.go @@ -3,6 +3,7 @@ package peerflow import ( "fmt" + "strings" "time" "github.com/PeerDB-io/peer-flow/generated/protos" @@ -21,6 +22,18 @@ type QRepFlowExecution struct { runUUID string } +// returns a new empty PeerFlowState +func NewQRepFlowState() *protos.QRepFlowState { + return &protos.QRepFlowState{ + LastPartition: &protos.QRepPartition{ + PartitionId: "not-applicable-partition", + Range: nil, + }, + NumPartitionsProcessed: 0, + NeedsResync: true, + } +} + // NewQRepFlowExecution creates a new instance of QRepFlowExecution. func NewQRepFlowExecution(ctx workflow.Context, config *protos.QRepConfig, runUUID string) *QRepFlowExecution { return &QRepFlowExecution{ @@ -254,11 +267,60 @@ func (q *QRepFlowExecution) waitForNewRows(ctx workflow.Context, lastPartition * return nil } +func (q *QRepFlowExecution) handleTableCreationForResync(ctx workflow.Context, state *protos.QRepFlowState) error { + if state.NeedsResync && q.config.DstTableFullResync { + renamedTableIdentifier := fmt.Sprintf("%s_peerdb_resync", q.config.DestinationTableIdentifier) + createTablesFromExistingCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 10 * time.Minute, + HeartbeatTimeout: 2 * time.Minute, + }) + createTablesFromExistingFuture := workflow.ExecuteActivity( + createTablesFromExistingCtx, flowable.CreateTablesFromExisting, &protos.CreateTablesFromExistingInput{ + FlowJobName: q.config.FlowJobName, + Peer: q.config.DestinationPeer, + NewToExistingTableMapping: map[string]string{ + renamedTableIdentifier: q.config.DestinationTableIdentifier, + }, + }) + if err := createTablesFromExistingFuture.Get(createTablesFromExistingCtx, nil); err != nil { + return fmt.Errorf("failed to create table for mirror resync: %w", err) + } + q.config.DestinationTableIdentifier = renamedTableIdentifier + } + return nil +} + +func (q *QRepFlowExecution) handleTableRenameForResync(ctx workflow.Context, state *protos.QRepFlowState) error { + if state.NeedsResync && q.config.DstTableFullResync { + oldTableIdentifier := strings.TrimSuffix(q.config.DestinationTableIdentifier, "_peerdb_resync") + renameOpts := &protos.RenameTablesInput{} + renameOpts.FlowJobName = q.config.FlowJobName + renameOpts.Peer = q.config.DestinationPeer + renameOpts.RenameTableOptions = []*protos.RenameTableOption{ + { + CurrentName: q.config.DestinationTableIdentifier, + NewName: oldTableIdentifier, + }, + } + + renameTablesCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 30 * time.Minute, + HeartbeatTimeout: 5 * time.Minute, + }) + renameTablesFuture := workflow.ExecuteActivity(renameTablesCtx, flowable.RenameTables, renameOpts) + if err := renameTablesFuture.Get(renameTablesCtx, nil); err != nil { + return fmt.Errorf("failed to execute rename tables activity: %w", err) + } + q.config.DestinationTableIdentifier = oldTableIdentifier + } + state.NeedsResync = true + return nil +} + func QRepFlowWorkflow( ctx workflow.Context, config *protos.QRepConfig, - lastPartition *protos.QRepPartition, - numPartitionsProcessed int, + state *protos.QRepFlowState, ) error { // The structure of this workflow is as follows: // 1. Start the loop to continuously run the replication flow. @@ -289,8 +351,8 @@ func QRepFlowWorkflow( }) // register a query to get the number of partitions processed - err := workflow.SetQueryHandler(ctx, "num-partitions-processed", func() (int, error) { - return numPartitionsProcessed, nil + err := workflow.SetQueryHandler(ctx, "num-partitions-processed", func() (uint64, error) { + return state.NumPartitionsProcessed, nil }) if err != nil { return fmt.Errorf("failed to register query handler: %w", err) @@ -314,13 +376,18 @@ func QRepFlowWorkflow( } q.logger.Info("metadata tables setup for peer flow - ", config.FlowJobName) + err = q.handleTableCreationForResync(ctx, state) + if err != nil { + return err + } + err = q.SetupWatermarkTableOnDestination(ctx) if err != nil { return fmt.Errorf("failed to setup watermark table: %w", err) } logger.Info("fetching partitions to replicate for peer flow - ", config.FlowJobName) - partitions, err := q.GetPartitions(ctx, lastPartition) + partitions, err := q.GetPartitions(ctx, state.LastPartition) if err != nil { return fmt.Errorf("failed to get partitions: %w", err) } @@ -340,11 +407,16 @@ func QRepFlowWorkflow( return nil } + err = q.handleTableRenameForResync(ctx, state) + if err != nil { + return err + } + q.logger.Info("partitions processed - ", len(partitions.Partitions)) - numPartitionsProcessed += len(partitions.Partitions) + state.NumPartitionsProcessed += uint64(len(partitions.Partitions)) if len(partitions.Partitions) > 0 { - lastPartition = partitions.Partitions[len(partitions.Partitions)-1] + state.LastPartition = partitions.Partitions[len(partitions.Partitions)-1] } s.AddDefault(func() {}) @@ -356,17 +428,17 @@ func QRepFlowWorkflow( } // sleep for a while and continue the workflow - err = q.waitForNewRows(ctx, lastPartition) + err = q.waitForNewRows(ctx, state.LastPartition) if err != nil { return err } workflow.GetLogger(ctx).Info("Continuing as new workflow", - "Last Partition", lastPartition, - "Number of Partitions Processed", numPartitionsProcessed) + "Last Partition", state.LastPartition, + "Number of Partitions Processed", state.NumPartitionsProcessed) // Continue the workflow with new state - return workflow.NewContinueAsNewError(ctx, QRepFlowWorkflow, config, lastPartition, numPartitionsProcessed) + return workflow.NewContinueAsNewError(ctx, QRepFlowWorkflow, config, state) } // QRepPartitionWorkflow replicate a partition batch diff --git a/flow/workflows/snapshot_flow.go b/flow/workflows/snapshot_flow.go index fab06d3ce1..d04d34bf44 100644 --- a/flow/workflows/snapshot_flow.go +++ b/flow/workflows/snapshot_flow.go @@ -116,11 +116,6 @@ func (s *SnapshotFlowExecution) cloneTable( TaskQueue: shared.PeerFlowTaskQueue, }) - lastPartition := &protos.QRepPartition{ - PartitionId: "not-applicable-partition", - Range: nil, - } - // we know that the source is postgres as setup replication output is non-nil // only for postgres sourcePostgres := s.config.Source @@ -170,9 +165,8 @@ func (s *SnapshotFlowExecution) cloneTable( }, } - numPartitionsProcessed := 0 - - boundSelector.SpawnChild(childCtx, QRepFlowWorkflow, config, lastPartition, numPartitionsProcessed) + state := NewQRepFlowState() + boundSelector.SpawnChild(childCtx, QRepFlowWorkflow, config, state) return nil } diff --git a/nexus/analyzer/src/qrep.rs b/nexus/analyzer/src/qrep.rs index 7107d2728c..8445247da9 100644 --- a/nexus/analyzer/src/qrep.rs +++ b/nexus/analyzer/src/qrep.rs @@ -95,6 +95,11 @@ lazy_static::lazy_static! { name: "setup_watermark_table_on_destination", default_value: false, required: false + }, + QRepOptionType::Boolean { + name: "dst_table_full_resync", + default_value: false, + required: false }] }; } diff --git a/nexus/flow-rs/src/grpc.rs b/nexus/flow-rs/src/grpc.rs index 502fba6f3a..6adae529b7 100644 --- a/nexus/flow-rs/src/grpc.rs +++ b/nexus/flow-rs/src/grpc.rs @@ -294,6 +294,8 @@ impl FlowGrpcClient { cfg.initial_copy_only = *v; } else if key == "setup_watermark_table_on_destination" { cfg.setup_watermark_table_on_destination = *v; + } else if key == "dst_table_full_resync" { + cfg.dst_table_full_resync = *v; } else { return anyhow::Result::Err(anyhow::anyhow!("invalid bool option {}", key)); } diff --git a/nexus/pt/src/peerdb_flow.rs b/nexus/pt/src/peerdb_flow.rs index 9407c796da..326ca71950 100644 --- a/nexus/pt/src/peerdb_flow.rs +++ b/nexus/pt/src/peerdb_flow.rs @@ -90,6 +90,8 @@ pub struct FlowConnectionConfigs { #[prost(int64, tag="22")] pub push_parallelism: i64, /// if true, then the flow will be resynced + /// create new tables with "_resync" suffix, perform initial load and then swap the new tables with the old ones + /// to be used after the old mirror is dropped #[prost(bool, tag="23")] pub resync: bool, } @@ -119,6 +121,22 @@ pub struct RenameTablesOutput { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateTablesFromExistingInput { + #[prost(string, tag="1")] + pub flow_job_name: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub peer: ::core::option::Option, + #[prost(map="string, string", tag="3")] + pub new_to_existing_table_mapping: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateTablesFromExistingOutput { + #[prost(string, tag="2")] + pub flow_job_name: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct SyncFlowOptions { #[prost(int32, tag="1")] pub batch_size: i32, @@ -435,6 +453,10 @@ pub struct QRepConfig { /// Creates the watermark table on the destination as-is, can be used for some queries. #[prost(bool, tag="17")] pub setup_watermark_table_on_destination: bool, + /// create new tables with "_peerdb_resync" suffix, perform initial load and then swap the new table with the old ones + /// to be used after the old mirror is dropped + #[prost(bool, tag="18")] + pub dst_table_full_resync: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -492,6 +514,16 @@ pub struct ReplayTableSchemaDeltaInput { #[prost(message, repeated, tag="2")] pub table_schema_deltas: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct QRepFlowState { + #[prost(message, optional, tag="1")] + pub last_partition: ::core::option::Option, + #[prost(uint64, tag="2")] + pub num_partitions_processed: u64, + #[prost(bool, tag="3")] + pub needs_resync: bool, +} /// protos for qrep #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/nexus/pt/src/peerdb_flow.serde.rs b/nexus/pt/src/peerdb_flow.serde.rs index 36acb19f86..725845fe18 100644 --- a/nexus/pt/src/peerdb_flow.serde.rs +++ b/nexus/pt/src/peerdb_flow.serde.rs @@ -249,6 +249,235 @@ impl<'de> serde::Deserialize<'de> for CreateRawTableOutput { deserializer.deserialize_struct("peerdb_flow.CreateRawTableOutput", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for CreateTablesFromExistingInput { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.flow_job_name.is_empty() { + len += 1; + } + if self.peer.is_some() { + len += 1; + } + if !self.new_to_existing_table_mapping.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("peerdb_flow.CreateTablesFromExistingInput", len)?; + if !self.flow_job_name.is_empty() { + struct_ser.serialize_field("flowJobName", &self.flow_job_name)?; + } + if let Some(v) = self.peer.as_ref() { + struct_ser.serialize_field("peer", v)?; + } + if !self.new_to_existing_table_mapping.is_empty() { + struct_ser.serialize_field("newToExistingTableMapping", &self.new_to_existing_table_mapping)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for CreateTablesFromExistingInput { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "flow_job_name", + "flowJobName", + "peer", + "new_to_existing_table_mapping", + "newToExistingTableMapping", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + FlowJobName, + Peer, + NewToExistingTableMapping, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "flowJobName" | "flow_job_name" => Ok(GeneratedField::FlowJobName), + "peer" => Ok(GeneratedField::Peer), + "newToExistingTableMapping" | "new_to_existing_table_mapping" => Ok(GeneratedField::NewToExistingTableMapping), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = CreateTablesFromExistingInput; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct peerdb_flow.CreateTablesFromExistingInput") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut flow_job_name__ = None; + let mut peer__ = None; + let mut new_to_existing_table_mapping__ = None; + while let Some(k) = map.next_key()? { + match k { + GeneratedField::FlowJobName => { + if flow_job_name__.is_some() { + return Err(serde::de::Error::duplicate_field("flowJobName")); + } + flow_job_name__ = Some(map.next_value()?); + } + GeneratedField::Peer => { + if peer__.is_some() { + return Err(serde::de::Error::duplicate_field("peer")); + } + peer__ = map.next_value()?; + } + GeneratedField::NewToExistingTableMapping => { + if new_to_existing_table_mapping__.is_some() { + return Err(serde::de::Error::duplicate_field("newToExistingTableMapping")); + } + new_to_existing_table_mapping__ = Some( + map.next_value::>()? + ); + } + GeneratedField::__SkipField__ => { + let _ = map.next_value::()?; + } + } + } + Ok(CreateTablesFromExistingInput { + flow_job_name: flow_job_name__.unwrap_or_default(), + peer: peer__, + new_to_existing_table_mapping: new_to_existing_table_mapping__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("peerdb_flow.CreateTablesFromExistingInput", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for CreateTablesFromExistingOutput { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.flow_job_name.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("peerdb_flow.CreateTablesFromExistingOutput", len)?; + if !self.flow_job_name.is_empty() { + struct_ser.serialize_field("flowJobName", &self.flow_job_name)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for CreateTablesFromExistingOutput { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "flow_job_name", + "flowJobName", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + FlowJobName, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "flowJobName" | "flow_job_name" => Ok(GeneratedField::FlowJobName), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = CreateTablesFromExistingOutput; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct peerdb_flow.CreateTablesFromExistingOutput") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut flow_job_name__ = None; + while let Some(k) = map.next_key()? { + match k { + GeneratedField::FlowJobName => { + if flow_job_name__.is_some() { + return Err(serde::de::Error::duplicate_field("flowJobName")); + } + flow_job_name__ = Some(map.next_value()?); + } + GeneratedField::__SkipField__ => { + let _ = map.next_value::()?; + } + } + } + Ok(CreateTablesFromExistingOutput { + flow_job_name: flow_job_name__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("peerdb_flow.CreateTablesFromExistingOutput", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for DeltaAddedColumn { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -2368,6 +2597,9 @@ impl serde::Serialize for QRepConfig { if self.setup_watermark_table_on_destination { len += 1; } + if self.dst_table_full_resync { + len += 1; + } let mut struct_ser = serializer.serialize_struct("peerdb_flow.QRepConfig", len)?; if !self.flow_job_name.is_empty() { struct_ser.serialize_field("flowJobName", &self.flow_job_name)?; @@ -2422,6 +2654,9 @@ impl serde::Serialize for QRepConfig { if self.setup_watermark_table_on_destination { struct_ser.serialize_field("setupWatermarkTableOnDestination", &self.setup_watermark_table_on_destination)?; } + if self.dst_table_full_resync { + struct_ser.serialize_field("dstTableFullResync", &self.dst_table_full_resync)?; + } struct_ser.end() } } @@ -2465,6 +2700,8 @@ impl<'de> serde::Deserialize<'de> for QRepConfig { "numRowsPerPartition", "setup_watermark_table_on_destination", "setupWatermarkTableOnDestination", + "dst_table_full_resync", + "dstTableFullResync", ]; #[allow(clippy::enum_variant_names)] @@ -2486,6 +2723,7 @@ impl<'de> serde::Deserialize<'de> for QRepConfig { StagingPath, NumRowsPerPartition, SetupWatermarkTableOnDestination, + DstTableFullResync, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -2525,6 +2763,7 @@ impl<'de> serde::Deserialize<'de> for QRepConfig { "stagingPath" | "staging_path" => Ok(GeneratedField::StagingPath), "numRowsPerPartition" | "num_rows_per_partition" => Ok(GeneratedField::NumRowsPerPartition), "setupWatermarkTableOnDestination" | "setup_watermark_table_on_destination" => Ok(GeneratedField::SetupWatermarkTableOnDestination), + "dstTableFullResync" | "dst_table_full_resync" => Ok(GeneratedField::DstTableFullResync), _ => Ok(GeneratedField::__SkipField__), } } @@ -2561,6 +2800,7 @@ impl<'de> serde::Deserialize<'de> for QRepConfig { let mut staging_path__ = None; let mut num_rows_per_partition__ = None; let mut setup_watermark_table_on_destination__ = None; + let mut dst_table_full_resync__ = None; while let Some(k) = map.next_key()? { match k { GeneratedField::FlowJobName => { @@ -2675,6 +2915,12 @@ impl<'de> serde::Deserialize<'de> for QRepConfig { } setup_watermark_table_on_destination__ = Some(map.next_value()?); } + GeneratedField::DstTableFullResync => { + if dst_table_full_resync__.is_some() { + return Err(serde::de::Error::duplicate_field("dstTableFullResync")); + } + dst_table_full_resync__ = Some(map.next_value()?); + } GeneratedField::__SkipField__ => { let _ = map.next_value::()?; } @@ -2698,12 +2944,147 @@ impl<'de> serde::Deserialize<'de> for QRepConfig { staging_path: staging_path__.unwrap_or_default(), num_rows_per_partition: num_rows_per_partition__.unwrap_or_default(), setup_watermark_table_on_destination: setup_watermark_table_on_destination__.unwrap_or_default(), + dst_table_full_resync: dst_table_full_resync__.unwrap_or_default(), }) } } deserializer.deserialize_struct("peerdb_flow.QRepConfig", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for QRepFlowState { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.last_partition.is_some() { + len += 1; + } + if self.num_partitions_processed != 0 { + len += 1; + } + if self.needs_resync { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("peerdb_flow.QRepFlowState", len)?; + if let Some(v) = self.last_partition.as_ref() { + struct_ser.serialize_field("lastPartition", v)?; + } + if self.num_partitions_processed != 0 { + struct_ser.serialize_field("numPartitionsProcessed", ToString::to_string(&self.num_partitions_processed).as_str())?; + } + if self.needs_resync { + struct_ser.serialize_field("needsResync", &self.needs_resync)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for QRepFlowState { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "last_partition", + "lastPartition", + "num_partitions_processed", + "numPartitionsProcessed", + "needs_resync", + "needsResync", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + LastPartition, + NumPartitionsProcessed, + NeedsResync, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "lastPartition" | "last_partition" => Ok(GeneratedField::LastPartition), + "numPartitionsProcessed" | "num_partitions_processed" => Ok(GeneratedField::NumPartitionsProcessed), + "needsResync" | "needs_resync" => Ok(GeneratedField::NeedsResync), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = QRepFlowState; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct peerdb_flow.QRepFlowState") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut last_partition__ = None; + let mut num_partitions_processed__ = None; + let mut needs_resync__ = None; + while let Some(k) = map.next_key()? { + match k { + GeneratedField::LastPartition => { + if last_partition__.is_some() { + return Err(serde::de::Error::duplicate_field("lastPartition")); + } + last_partition__ = map.next_value()?; + } + GeneratedField::NumPartitionsProcessed => { + if num_partitions_processed__.is_some() { + return Err(serde::de::Error::duplicate_field("numPartitionsProcessed")); + } + num_partitions_processed__ = + Some(map.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::NeedsResync => { + if needs_resync__.is_some() { + return Err(serde::de::Error::duplicate_field("needsResync")); + } + needs_resync__ = Some(map.next_value()?); + } + GeneratedField::__SkipField__ => { + let _ = map.next_value::()?; + } + } + } + Ok(QRepFlowState { + last_partition: last_partition__, + num_partitions_processed: num_partitions_processed__.unwrap_or_default(), + needs_resync: needs_resync__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("peerdb_flow.QRepFlowState", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for QRepParitionResult { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/protos/flow.proto b/protos/flow.proto index d01aa7fd41..4f31235ca2 100644 --- a/protos/flow.proto +++ b/protos/flow.proto @@ -64,6 +64,8 @@ message FlowConnectionConfigs { int64 push_parallelism = 22; // if true, then the flow will be resynced + // create new tables with "_resync" suffix, perform initial load and then swap the new tables with the old ones + // to be used after the old mirror is dropped bool resync = 23; } @@ -82,6 +84,16 @@ message RenameTablesOutput { string flow_job_name = 1; } +message CreateTablesFromExistingInput { + string flow_job_name = 1; + peerdb_peers.Peer peer = 2; + map new_to_existing_table_mapping = 3; +} + +message CreateTablesFromExistingOutput { + string flow_job_name = 2; +} + message SyncFlowOptions { int32 batch_size = 1; map relation_message_mapping = 2; @@ -296,6 +308,10 @@ message QRepConfig { // Creates the watermark table on the destination as-is, can be used for some queries. bool setup_watermark_table_on_destination = 17; + + // create new tables with "_peerdb_resync" suffix, perform initial load and then swap the new table with the old ones + // to be used after the old mirror is dropped + bool dst_table_full_resync = 18; } message QRepPartition { @@ -332,3 +348,9 @@ message ReplayTableSchemaDeltaInput { FlowConnectionConfigs flow_connection_configs = 1; repeated TableSchemaDelta table_schema_deltas = 2; } + +message QRepFlowState { + QRepPartition last_partition = 1; + uint64 num_partitions_processed = 2; + bool needs_resync = 3; +} \ No newline at end of file diff --git a/ui/app/mirrors/create/helpers/common.ts b/ui/app/mirrors/create/helpers/common.ts index 01fb3bd5d2..f09ef1fbcf 100644 --- a/ui/app/mirrors/create/helpers/common.ts +++ b/ui/app/mirrors/create/helpers/common.ts @@ -63,4 +63,5 @@ export const blankQRepSetting: QRepConfig = { stagingPath: '', numRowsPerPartition: 0, setupWatermarkTableOnDestination: false, + dstTableFullResync: false, }; diff --git a/ui/grpc_generated/flow.ts b/ui/grpc_generated/flow.ts index 2999a1aabb..6cc5e8a0e1 100644 --- a/ui/grpc_generated/flow.ts +++ b/ui/grpc_generated/flow.ts @@ -133,7 +133,11 @@ export interface FlowConnectionConfigs { /** the below two are for eventhub only */ pushBatchSize: number; pushParallelism: number; - /** if true, then the flow will be resynced */ + /** + * if true, then the flow will be resynced + * create new tables with "_resync" suffix, perform initial load and then swap the new tables with the old ones + * to be used after the old mirror is dropped + */ resync: boolean; } @@ -162,6 +166,21 @@ export interface RenameTablesOutput { flowJobName: string; } +export interface CreateTablesFromExistingInput { + flowJobName: string; + peer: Peer | undefined; + newToExistingTableMapping: { [key: string]: string }; +} + +export interface CreateTablesFromExistingInput_NewToExistingTableMappingEntry { + key: string; + value: string; +} + +export interface CreateTablesFromExistingOutput { + flowJobName: string; +} + export interface SyncFlowOptions { batchSize: number; relationMessageMapping: { [key: number]: RelationMessage }; @@ -401,6 +420,11 @@ export interface QRepConfig { numRowsPerPartition: number; /** Creates the watermark table on the destination as-is, can be used for some queries. */ setupWatermarkTableOnDestination: boolean; + /** + * create new tables with "_peerdb_resync" suffix, perform initial load and then swap the new table with the old ones + * to be used after the old mirror is dropped + */ + dstTableFullResync: boolean; } export interface QRepPartition { @@ -438,6 +462,12 @@ export interface ReplayTableSchemaDeltaInput { tableSchemaDeltas: TableSchemaDelta[]; } +export interface QRepFlowState { + lastPartition: QRepPartition | undefined; + numPartitionsProcessed: number; + needsResync: boolean; +} + function createBaseTableNameMapping(): TableNameMapping { return { sourceTableName: "", destinationTableName: "" }; } @@ -1637,6 +1667,261 @@ export const RenameTablesOutput = { }, }; +function createBaseCreateTablesFromExistingInput(): CreateTablesFromExistingInput { + return { flowJobName: "", peer: undefined, newToExistingTableMapping: {} }; +} + +export const CreateTablesFromExistingInput = { + encode(message: CreateTablesFromExistingInput, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.flowJobName !== "") { + writer.uint32(10).string(message.flowJobName); + } + if (message.peer !== undefined) { + Peer.encode(message.peer, writer.uint32(18).fork()).ldelim(); + } + Object.entries(message.newToExistingTableMapping).forEach(([key, value]) => { + CreateTablesFromExistingInput_NewToExistingTableMappingEntry.encode( + { key: key as any, value }, + writer.uint32(26).fork(), + ).ldelim(); + }); + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): CreateTablesFromExistingInput { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseCreateTablesFromExistingInput(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.flowJobName = reader.string(); + continue; + case 2: + if (tag !== 18) { + break; + } + + message.peer = Peer.decode(reader, reader.uint32()); + continue; + case 3: + if (tag !== 26) { + break; + } + + const entry3 = CreateTablesFromExistingInput_NewToExistingTableMappingEntry.decode(reader, reader.uint32()); + if (entry3.value !== undefined) { + message.newToExistingTableMapping[entry3.key] = entry3.value; + } + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): CreateTablesFromExistingInput { + return { + flowJobName: isSet(object.flowJobName) ? String(object.flowJobName) : "", + peer: isSet(object.peer) ? Peer.fromJSON(object.peer) : undefined, + newToExistingTableMapping: isObject(object.newToExistingTableMapping) + ? Object.entries(object.newToExistingTableMapping).reduce<{ [key: string]: string }>((acc, [key, value]) => { + acc[key] = String(value); + return acc; + }, {}) + : {}, + }; + }, + + toJSON(message: CreateTablesFromExistingInput): unknown { + const obj: any = {}; + if (message.flowJobName !== "") { + obj.flowJobName = message.flowJobName; + } + if (message.peer !== undefined) { + obj.peer = Peer.toJSON(message.peer); + } + if (message.newToExistingTableMapping) { + const entries = Object.entries(message.newToExistingTableMapping); + if (entries.length > 0) { + obj.newToExistingTableMapping = {}; + entries.forEach(([k, v]) => { + obj.newToExistingTableMapping[k] = v; + }); + } + } + return obj; + }, + + create, I>>(base?: I): CreateTablesFromExistingInput { + return CreateTablesFromExistingInput.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>( + object: I, + ): CreateTablesFromExistingInput { + const message = createBaseCreateTablesFromExistingInput(); + message.flowJobName = object.flowJobName ?? ""; + message.peer = (object.peer !== undefined && object.peer !== null) ? Peer.fromPartial(object.peer) : undefined; + message.newToExistingTableMapping = Object.entries(object.newToExistingTableMapping ?? {}).reduce< + { [key: string]: string } + >((acc, [key, value]) => { + if (value !== undefined) { + acc[key] = String(value); + } + return acc; + }, {}); + return message; + }, +}; + +function createBaseCreateTablesFromExistingInput_NewToExistingTableMappingEntry(): CreateTablesFromExistingInput_NewToExistingTableMappingEntry { + return { key: "", value: "" }; +} + +export const CreateTablesFromExistingInput_NewToExistingTableMappingEntry = { + encode( + message: CreateTablesFromExistingInput_NewToExistingTableMappingEntry, + writer: _m0.Writer = _m0.Writer.create(), + ): _m0.Writer { + if (message.key !== "") { + writer.uint32(10).string(message.key); + } + if (message.value !== "") { + writer.uint32(18).string(message.value); + } + return writer; + }, + + decode( + input: _m0.Reader | Uint8Array, + length?: number, + ): CreateTablesFromExistingInput_NewToExistingTableMappingEntry { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseCreateTablesFromExistingInput_NewToExistingTableMappingEntry(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.key = reader.string(); + continue; + case 2: + if (tag !== 18) { + break; + } + + message.value = reader.string(); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): CreateTablesFromExistingInput_NewToExistingTableMappingEntry { + return { key: isSet(object.key) ? String(object.key) : "", value: isSet(object.value) ? String(object.value) : "" }; + }, + + toJSON(message: CreateTablesFromExistingInput_NewToExistingTableMappingEntry): unknown { + const obj: any = {}; + if (message.key !== "") { + obj.key = message.key; + } + if (message.value !== "") { + obj.value = message.value; + } + return obj; + }, + + create, I>>( + base?: I, + ): CreateTablesFromExistingInput_NewToExistingTableMappingEntry { + return CreateTablesFromExistingInput_NewToExistingTableMappingEntry.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>( + object: I, + ): CreateTablesFromExistingInput_NewToExistingTableMappingEntry { + const message = createBaseCreateTablesFromExistingInput_NewToExistingTableMappingEntry(); + message.key = object.key ?? ""; + message.value = object.value ?? ""; + return message; + }, +}; + +function createBaseCreateTablesFromExistingOutput(): CreateTablesFromExistingOutput { + return { flowJobName: "" }; +} + +export const CreateTablesFromExistingOutput = { + encode(message: CreateTablesFromExistingOutput, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.flowJobName !== "") { + writer.uint32(18).string(message.flowJobName); + } + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): CreateTablesFromExistingOutput { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseCreateTablesFromExistingOutput(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 2: + if (tag !== 18) { + break; + } + + message.flowJobName = reader.string(); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): CreateTablesFromExistingOutput { + return { flowJobName: isSet(object.flowJobName) ? String(object.flowJobName) : "" }; + }, + + toJSON(message: CreateTablesFromExistingOutput): unknown { + const obj: any = {}; + if (message.flowJobName !== "") { + obj.flowJobName = message.flowJobName; + } + return obj; + }, + + create, I>>(base?: I): CreateTablesFromExistingOutput { + return CreateTablesFromExistingOutput.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>( + object: I, + ): CreateTablesFromExistingOutput { + const message = createBaseCreateTablesFromExistingOutput(); + message.flowJobName = object.flowJobName ?? ""; + return message; + }, +}; + function createBaseSyncFlowOptions(): SyncFlowOptions { return { batchSize: 0, relationMessageMapping: {} }; } @@ -4879,6 +5164,7 @@ function createBaseQRepConfig(): QRepConfig { stagingPath: "", numRowsPerPartition: 0, setupWatermarkTableOnDestination: false, + dstTableFullResync: false, }; } @@ -4935,6 +5221,9 @@ export const QRepConfig = { if (message.setupWatermarkTableOnDestination === true) { writer.uint32(136).bool(message.setupWatermarkTableOnDestination); } + if (message.dstTableFullResync === true) { + writer.uint32(144).bool(message.dstTableFullResync); + } return writer; }, @@ -5064,6 +5353,13 @@ export const QRepConfig = { message.setupWatermarkTableOnDestination = reader.bool(); continue; + case 18: + if (tag !== 144) { + break; + } + + message.dstTableFullResync = reader.bool(); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -5096,6 +5392,7 @@ export const QRepConfig = { setupWatermarkTableOnDestination: isSet(object.setupWatermarkTableOnDestination) ? Boolean(object.setupWatermarkTableOnDestination) : false, + dstTableFullResync: isSet(object.dstTableFullResync) ? Boolean(object.dstTableFullResync) : false, }; }, @@ -5152,6 +5449,9 @@ export const QRepConfig = { if (message.setupWatermarkTableOnDestination === true) { obj.setupWatermarkTableOnDestination = message.setupWatermarkTableOnDestination; } + if (message.dstTableFullResync === true) { + obj.dstTableFullResync = message.dstTableFullResync; + } return obj; }, @@ -5183,6 +5483,7 @@ export const QRepConfig = { message.stagingPath = object.stagingPath ?? ""; message.numRowsPerPartition = object.numRowsPerPartition ?? 0; message.setupWatermarkTableOnDestination = object.setupWatermarkTableOnDestination ?? false; + message.dstTableFullResync = object.dstTableFullResync ?? false; return message; }, }; @@ -5714,6 +6015,97 @@ export const ReplayTableSchemaDeltaInput = { }, }; +function createBaseQRepFlowState(): QRepFlowState { + return { lastPartition: undefined, numPartitionsProcessed: 0, needsResync: false }; +} + +export const QRepFlowState = { + encode(message: QRepFlowState, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.lastPartition !== undefined) { + QRepPartition.encode(message.lastPartition, writer.uint32(10).fork()).ldelim(); + } + if (message.numPartitionsProcessed !== 0) { + writer.uint32(16).uint64(message.numPartitionsProcessed); + } + if (message.needsResync === true) { + writer.uint32(24).bool(message.needsResync); + } + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): QRepFlowState { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseQRepFlowState(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.lastPartition = QRepPartition.decode(reader, reader.uint32()); + continue; + case 2: + if (tag !== 16) { + break; + } + + message.numPartitionsProcessed = longToNumber(reader.uint64() as Long); + continue; + case 3: + if (tag !== 24) { + break; + } + + message.needsResync = reader.bool(); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): QRepFlowState { + return { + lastPartition: isSet(object.lastPartition) ? QRepPartition.fromJSON(object.lastPartition) : undefined, + numPartitionsProcessed: isSet(object.numPartitionsProcessed) ? Number(object.numPartitionsProcessed) : 0, + needsResync: isSet(object.needsResync) ? Boolean(object.needsResync) : false, + }; + }, + + toJSON(message: QRepFlowState): unknown { + const obj: any = {}; + if (message.lastPartition !== undefined) { + obj.lastPartition = QRepPartition.toJSON(message.lastPartition); + } + if (message.numPartitionsProcessed !== 0) { + obj.numPartitionsProcessed = Math.round(message.numPartitionsProcessed); + } + if (message.needsResync === true) { + obj.needsResync = message.needsResync; + } + return obj; + }, + + create, I>>(base?: I): QRepFlowState { + return QRepFlowState.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): QRepFlowState { + const message = createBaseQRepFlowState(); + message.lastPartition = (object.lastPartition !== undefined && object.lastPartition !== null) + ? QRepPartition.fromPartial(object.lastPartition) + : undefined; + message.numPartitionsProcessed = object.numPartitionsProcessed ?? 0; + message.needsResync = object.needsResync ?? false; + return message; + }, +}; + declare const self: any | undefined; declare const window: any | undefined; declare const global: any | undefined;