diff --git a/go/test/endtoend/vreplication/movetables_mirrortraffic_test.go b/go/test/endtoend/vreplication/movetables_mirrortraffic_test.go index dfab72d9582..54e648de6b4 100644 --- a/go/test/endtoend/vreplication/movetables_mirrortraffic_test.go +++ b/go/test/endtoend/vreplication/movetables_mirrortraffic_test.go @@ -35,7 +35,7 @@ func testMoveTablesMirrorTraffic(t *testing.T, flavor workflowFlavor) { _ = setupMinimalCustomerKeyspace(t) - mt := newMoveTables(vc, &moveTablesWorkflow{ + mtwf := &moveTablesWorkflow{ workflowInfo: &workflowInfo{ vc: vc, workflowName: workflowName, @@ -44,22 +44,45 @@ func testMoveTablesMirrorTraffic(t *testing.T, flavor workflowFlavor) { sourceKeyspace: sourceKeyspace, tables: "customer,loadtest,customer2", mirrorFlags: []string{"--percent", "25"}, - }, flavor) + } + mt := newMoveTables(vc, mtwf, flavor) + // Mirror rules do not exist by default. mt.Create() confirmNoMirrorRules(t) waitForWorkflowState(t, vc, ksWorkflow, binlogdatapb.VReplicationWorkflowState_Running.String()) + // Mirror rules can be created after a MoveTables workflow is created. mt.MirrorTraffic() confirmMirrorRulesExist(t) - expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{ topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA, topodatapb.TabletType_RDONLY, }, 25) + // Mirror rules can be adjusted after mirror rules are in place. + mtwf.mirrorFlags[1] = "50" + mt.MirrorTraffic() + confirmMirrorRulesExist(t) + expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{ + topodatapb.TabletType_PRIMARY, + topodatapb.TabletType_REPLICA, + topodatapb.TabletType_RDONLY, + }, 50) + + // Mirror rules can be adjusted multiple times after mirror rules are in + // place. + mtwf.mirrorFlags[1] = "75" + mt.MirrorTraffic() + confirmMirrorRulesExist(t) + expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{ + topodatapb.TabletType_PRIMARY, + topodatapb.TabletType_REPLICA, + topodatapb.TabletType_RDONLY, + }, 75) + lg := newLoadGenerator(t, vc) go func() { lg.start() @@ -69,10 +92,16 @@ func testMoveTablesMirrorTraffic(t *testing.T, flavor workflowFlavor) { mt.SwitchReads() confirmMirrorRulesExist(t) + // Mirror rules can be adjusted for writes after reads have been switched. + mtwf.mirrorFlags[1] = "100" + mtwf.mirrorFlags = append(mtwf.mirrorFlags, "--tablet-types", "primary") + mt.MirrorTraffic() + confirmMirrorRulesExist(t) expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{ topodatapb.TabletType_PRIMARY, - }, 25) + }, 100) + // Mirror rules are removed after writes are switched. mt.SwitchWrites() confirmNoMirrorRules(t) } diff --git a/go/test/endtoend/vreplication/vreplication_test_env.go b/go/test/endtoend/vreplication/vreplication_test_env.go index 751faf497ad..238242f0e65 100644 --- a/go/test/endtoend/vreplication/vreplication_test_env.go +++ b/go/test/endtoend/vreplication/vreplication_test_env.go @@ -17,7 +17,7 @@ limitations under the License. package vreplication var dryRunResultsSwitchWritesCustomerShard = []string{ - "Mirroring 0.000000 percent of traffic from keyspace product to keyspace customer for tablet types [PRIMARY]", + "Mirroring 0.00 percent of traffic from keyspace product to keyspace customer for tablet types [PRIMARY]", "Lock keyspace product", "Lock keyspace customer", "/Stop writes on keyspace product for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order]: [keyspace:product;shard:0;position:", @@ -36,7 +36,7 @@ var dryRunResultsSwitchWritesCustomerShard = []string{ } var dryRunResultsReadCustomerShard = []string{ - "Mirroring 0.000000 percent of traffic from keyspace product to keyspace customer for tablet types [RDONLY,REPLICA]", + "Mirroring 0.00 percent of traffic from keyspace product to keyspace customer for tablet types [RDONLY,REPLICA]", "Lock keyspace product", "Switch reads for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] to keyspace customer for tablet types [RDONLY,REPLICA]", "Routing rules for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] will be updated", diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 70fd969bbdd..0a00ee9a0bf 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -4091,8 +4091,20 @@ func (s *Server) WorkflowMirrorTraffic(ctx context.Context, req *vtctldatapb.Wor // Don't allow traffic to be mirrored if any traffic has been switched over // to the target keyspace. - if len(startState.RdonlyCellsSwitched) > 0 || len(startState.ReplicaCellsSwitched) > 0 || startState.WritesSwitched { - return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot mirror traffic for workflow %s at this time: traffic is switched", startState.Workflow) + var cannotSwitchTabletTypes []string + for _, tt := range req.TabletTypes { + if tt == topodatapb.TabletType_RDONLY && len(startState.RdonlyCellsSwitched) > 0 { + cannotSwitchTabletTypes = append(cannotSwitchTabletTypes, "rdonly") + } + if tt == topodatapb.TabletType_REPLICA && len(startState.ReplicaCellsSwitched) > 0 { + cannotSwitchTabletTypes = append(cannotSwitchTabletTypes, "replica") + } + if tt == topodatapb.TabletType_PRIMARY && startState.WritesSwitched { + cannotSwitchTabletTypes = append(cannotSwitchTabletTypes, "primary") + } + } + if len(cannotSwitchTabletTypes) > 0 { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot mirror [%s] traffic for workflow %s at this time: traffic for those tablet types is switched", strings.Join(cannotSwitchTabletTypes, ","), startState.Workflow) } if err := s.mirrorTraffic(ctx, req, ts, startState); err != nil { diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index fa81b7555fe..f2b9f6b2496 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -689,12 +689,12 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { DryRun: true, }, want: []string{ - fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", sourceKeyspaceName, targetKeyspaceName), + fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", sourceKeyspaceName, targetKeyspaceName), fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tablesStr, targetKeyspaceName), fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr), fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), - fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", sourceKeyspaceName, targetKeyspaceName), + fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", sourceKeyspaceName, targetKeyspaceName), fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]", @@ -730,12 +730,12 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { DryRun: true, }, want: []string{ - fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", targetKeyspaceName, sourceKeyspaceName), + fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", targetKeyspaceName, sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tablesStr, targetKeyspaceName), fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr), fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName), - fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", targetKeyspaceName, sourceKeyspaceName), + fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", targetKeyspaceName, sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]", @@ -891,7 +891,7 @@ func TestMirrorTraffic(t *testing.T) { wantMirrorRules: make(map[string]map[string]float32), }, { - name: "cannot mirror traffic after switch rdonly traffic", + name: "cannot mirror rdonly traffic after switch rdonly traffic", req: &vtctldatapb.WorkflowMirrorTrafficRequest{ Keyspace: targetKs, Workflow: workflow, @@ -902,11 +902,11 @@ func TestMirrorTraffic(t *testing.T) { fmt.Sprintf("%s.%s@rdonly", targetKs, table1): {fmt.Sprintf("%s.%s@rdonly", targetKs, table1)}, fmt.Sprintf("%s.%s@rdonly", targetKs, table2): {fmt.Sprintf("%s.%s@rdonly", targetKs, table2)}, }, - wantErr: "cannot mirror traffic for workflow src2target at this time: traffic is switched", + wantErr: "cannot mirror [rdonly] traffic for workflow src2target at this time: traffic for those tablet types is switched", wantMirrorRules: make(map[string]map[string]float32), }, { - name: "cannot mirror traffic after switch replica traffic", + name: "cannot mirror replica traffic after switch replica traffic", req: &vtctldatapb.WorkflowMirrorTrafficRequest{ Keyspace: targetKs, Workflow: workflow, @@ -917,11 +917,11 @@ func TestMirrorTraffic(t *testing.T) { fmt.Sprintf("%s.%s@replica", targetKs, table1): {fmt.Sprintf("%s.%s@replica", targetKs, table1)}, fmt.Sprintf("%s.%s@replica", targetKs, table2): {fmt.Sprintf("%s.%s@replica", targetKs, table2)}, }, - wantErr: "cannot mirror traffic for workflow src2target at this time: traffic is switched", + wantErr: "cannot mirror [replica] traffic for workflow src2target at this time: traffic for those tablet types is switched", wantMirrorRules: make(map[string]map[string]float32), }, { - name: "cannot mirror traffic after switch traffic", + name: "cannot mirror write traffic after switch traffic", req: &vtctldatapb.WorkflowMirrorTrafficRequest{ Keyspace: targetKs, Workflow: workflow, @@ -932,7 +932,7 @@ func TestMirrorTraffic(t *testing.T) { table1: {fmt.Sprintf("%s.%s", targetKs, table1)}, table2: {fmt.Sprintf("%s.%s", targetKs, table2)}, }, - wantErr: "cannot mirror traffic for workflow src2target at this time: traffic is switched", + wantErr: "cannot mirror [primary] traffic for workflow src2target at this time: traffic for those tablet types is switched", wantMirrorRules: make(map[string]map[string]float32), }, { diff --git a/go/vt/vtctl/workflow/switcher_dry_run.go b/go/vt/vtctl/workflow/switcher_dry_run.go index d96af1d276f..55c843b07cb 100644 --- a/go/vt/vtctl/workflow/switcher_dry_run.go +++ b/go/vt/vtctl/workflow/switcher_dry_run.go @@ -68,7 +68,7 @@ func (dr *switcherDryRun) mirrorTableTraffic(ctx context.Context, types []topoda for _, servedType := range types { tabletTypes = append(tabletTypes, servedType.String()) } - dr.drLog.Logf("Mirroring %f percent of traffic from keyspace %s to keyspace %s for tablet types [%s]", + dr.drLog.Logf("Mirroring %.2f percent of traffic from keyspace %s to keyspace %s for tablet types [%s]", percent, dr.ts.SourceKeyspaceName(), dr.ts.TargetKeyspaceName(), strings.Join(tabletTypes, ",")) return nil