Skip to content

Commit

Permalink
cr: test repeat calls to mirror traffic
Browse files Browse the repository at this point in the history 
Signed-off-by: Max Englander <[email protected]>
  • Loading branch information
@maxenglander
maxenglander committed Jun 21, 2024
1 parent 859b421 commit 71b9a00
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 19 deletions.
37 changes: 33 additions & 4 deletions go/test/endtoend/vreplication/movetables_mirrortraffic_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func testMoveTablesMirrorTraffic(t *testing.T, flavor workflowFlavor) {

_ = setupMinimalCustomerKeyspace(t)

mt := newMoveTables(vc, &moveTablesWorkflow{
mtwf := &moveTablesWorkflow{
workflowInfo: &workflowInfo{
vc: vc,
workflowName: workflowName,
Expand All @@ -44,22 +44,45 @@ func testMoveTablesMirrorTraffic(t *testing.T, flavor workflowFlavor) {
sourceKeyspace: sourceKeyspace,
tables: "customer,loadtest,customer2",
mirrorFlags: []string{"--percent", "25"},
}, flavor)
}
mt := newMoveTables(vc, mtwf, flavor)

// Mirror rules do not exist by default.
mt.Create()
confirmNoMirrorRules(t)

waitForWorkflowState(t, vc, ksWorkflow, binlogdatapb.VReplicationWorkflowState_Running.String())

// Mirror rules can be created after a MoveTables workflow is created.
mt.MirrorTraffic()
confirmMirrorRulesExist(t)

expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{
topodatapb.TabletType_PRIMARY,
topodatapb.TabletType_REPLICA,
topodatapb.TabletType_RDONLY,
}, 25)

// Mirror rules can be adjusted after mirror rules are in place.
mtwf.mirrorFlags[1] = "50"
mt.MirrorTraffic()
confirmMirrorRulesExist(t)
expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{
topodatapb.TabletType_PRIMARY,
topodatapb.TabletType_REPLICA,
topodatapb.TabletType_RDONLY,
}, 50)

// Mirror rules can be adjusted multiple times after mirror rules are in
// place.
mtwf.mirrorFlags[1] = "75"
mt.MirrorTraffic()
confirmMirrorRulesExist(t)
expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{
topodatapb.TabletType_PRIMARY,
topodatapb.TabletType_REPLICA,
topodatapb.TabletType_RDONLY,
}, 75)

lg := newLoadGenerator(t, vc)
go func() {
lg.start()
Expand All @@ -69,10 +92,16 @@ func testMoveTablesMirrorTraffic(t *testing.T, flavor workflowFlavor) {
mt.SwitchReads()
confirmMirrorRulesExist(t)

// Mirror rules can be adjusted for writes after reads have been switched.
mtwf.mirrorFlags[1] = "100"
mtwf.mirrorFlags = append(mtwf.mirrorFlags, "--tablet-types", "primary")
mt.MirrorTraffic()
confirmMirrorRulesExist(t)
expectMirrorRules(t, sourceKeyspace, targetKeyspace, tables, []topodatapb.TabletType{
topodatapb.TabletType_PRIMARY,
}, 25)
}, 100)

// Mirror rules are removed after writes are switched.
mt.SwitchWrites()
confirmNoMirrorRules(t)
}
Expand Down
4 changes: 2 additions & 2 deletions go/test/endtoend/vreplication/vreplication_test_env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.
package vreplication

var dryRunResultsSwitchWritesCustomerShard = []string{
"Mirroring 0.000000 percent of traffic from keyspace product to keyspace customer for tablet types [PRIMARY]",
"Mirroring 0.00 percent of traffic from keyspace product to keyspace customer for tablet types [PRIMARY]",
"Lock keyspace product",
"Lock keyspace customer",
"/Stop writes on keyspace product for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order]: [keyspace:product;shard:0;position:",
Expand All @@ -36,7 +36,7 @@ var dryRunResultsSwitchWritesCustomerShard = []string{
}

var dryRunResultsReadCustomerShard = []string{
"Mirroring 0.000000 percent of traffic from keyspace product to keyspace customer for tablet types [RDONLY,REPLICA]",
"Mirroring 0.00 percent of traffic from keyspace product to keyspace customer for tablet types [RDONLY,REPLICA]",
"Lock keyspace product",
"Switch reads for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] to keyspace customer for tablet types [RDONLY,REPLICA]",
"Routing rules for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] will be updated",
Expand Down
16 changes: 14 additions & 2 deletions go/vt/vtctl/workflow/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4091,8 +4091,20 @@ func (s *Server) WorkflowMirrorTraffic(ctx context.Context, req *vtctldatapb.Wor

// Don't allow traffic to be mirrored if any traffic has been switched over
// to the target keyspace.
if len(startState.RdonlyCellsSwitched) > 0 || len(startState.ReplicaCellsSwitched) > 0 || startState.WritesSwitched {
return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot mirror traffic for workflow %s at this time: traffic is switched", startState.Workflow)
var cannotSwitchTabletTypes []string
for _, tt := range req.TabletTypes {
if tt == topodatapb.TabletType_RDONLY && len(startState.RdonlyCellsSwitched) > 0 {
cannotSwitchTabletTypes = append(cannotSwitchTabletTypes, "rdonly")
}
if tt == topodatapb.TabletType_REPLICA && len(startState.ReplicaCellsSwitched) > 0 {
cannotSwitchTabletTypes = append(cannotSwitchTabletTypes, "replica")
}
if tt == topodatapb.TabletType_PRIMARY && startState.WritesSwitched {
cannotSwitchTabletTypes = append(cannotSwitchTabletTypes, "primary")
}
}
if len(cannotSwitchTabletTypes) > 0 {
return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot mirror [%s] traffic for workflow %s at this time: traffic for those tablet types is switched", strings.Join(cannotSwitchTabletTypes, ","), startState.Workflow)
}

if err := s.mirrorTraffic(ctx, req, ts, startState); err != nil {
Expand Down
20 changes: 10 additions & 10 deletions go/vt/vtctl/workflow/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -689,12 +689,12 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) {
DryRun: true,
},
want: []string{
fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", sourceKeyspaceName, targetKeyspaceName),
fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", sourceKeyspaceName, targetKeyspaceName),
fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName),
fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tablesStr, targetKeyspaceName),
fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr),
fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName),
fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", sourceKeyspaceName, targetKeyspaceName),
fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", sourceKeyspaceName, targetKeyspaceName),
fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName),
fmt.Sprintf("Lock keyspace %s", targetKeyspaceName),
fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]",
Expand Down Expand Up @@ -730,12 +730,12 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) {
DryRun: true,
},
want: []string{
fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", targetKeyspaceName, sourceKeyspaceName),
fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [REPLICA,RDONLY]", targetKeyspaceName, sourceKeyspaceName),
fmt.Sprintf("Lock keyspace %s", targetKeyspaceName),
fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tablesStr, targetKeyspaceName),
fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr),
fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName),
fmt.Sprintf("Mirroring 0.000000 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", targetKeyspaceName, sourceKeyspaceName),
fmt.Sprintf("Mirroring 0.00 percent of traffic from keyspace %s to keyspace %s for tablet types [PRIMARY]", targetKeyspaceName, sourceKeyspaceName),
fmt.Sprintf("Lock keyspace %s", targetKeyspaceName),
fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName),
fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]",
Expand Down Expand Up @@ -891,7 +891,7 @@ func TestMirrorTraffic(t *testing.T) {
wantMirrorRules: make(map[string]map[string]float32),
},
{
name: "cannot mirror traffic after switch rdonly traffic",
name: "cannot mirror rdonly traffic after switch rdonly traffic",
req: &vtctldatapb.WorkflowMirrorTrafficRequest{
Keyspace: targetKs,
Workflow: workflow,
Expand All @@ -902,11 +902,11 @@ func TestMirrorTraffic(t *testing.T) {
fmt.Sprintf("%s.%s@rdonly", targetKs, table1): {fmt.Sprintf("%s.%s@rdonly", targetKs, table1)},
fmt.Sprintf("%s.%s@rdonly", targetKs, table2): {fmt.Sprintf("%s.%s@rdonly", targetKs, table2)},
},
wantErr: "cannot mirror traffic for workflow src2target at this time: traffic is switched",
wantErr: "cannot mirror [rdonly] traffic for workflow src2target at this time: traffic for those tablet types is switched",
wantMirrorRules: make(map[string]map[string]float32),
},
{
name: "cannot mirror traffic after switch replica traffic",
name: "cannot mirror replica traffic after switch replica traffic",
req: &vtctldatapb.WorkflowMirrorTrafficRequest{
Keyspace: targetKs,
Workflow: workflow,
Expand All @@ -917,11 +917,11 @@ func TestMirrorTraffic(t *testing.T) {
fmt.Sprintf("%s.%s@replica", targetKs, table1): {fmt.Sprintf("%s.%s@replica", targetKs, table1)},
fmt.Sprintf("%s.%s@replica", targetKs, table2): {fmt.Sprintf("%s.%s@replica", targetKs, table2)},
},
wantErr: "cannot mirror traffic for workflow src2target at this time: traffic is switched",
wantErr: "cannot mirror [replica] traffic for workflow src2target at this time: traffic for those tablet types is switched",
wantMirrorRules: make(map[string]map[string]float32),
},
{
name: "cannot mirror traffic after switch traffic",
name: "cannot mirror write traffic after switch traffic",
req: &vtctldatapb.WorkflowMirrorTrafficRequest{
Keyspace: targetKs,
Workflow: workflow,
Expand All @@ -932,7 +932,7 @@ func TestMirrorTraffic(t *testing.T) {
table1: {fmt.Sprintf("%s.%s", targetKs, table1)},
table2: {fmt.Sprintf("%s.%s", targetKs, table2)},
},
wantErr: "cannot mirror traffic for workflow src2target at this time: traffic is switched",
wantErr: "cannot mirror [primary] traffic for workflow src2target at this time: traffic for those tablet types is switched",
wantMirrorRules: make(map[string]map[string]float32),
},
{
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vtctl/workflow/switcher_dry_run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func (dr *switcherDryRun) mirrorTableTraffic(ctx context.Context, types []topoda
for _, servedType := range types {
tabletTypes = append(tabletTypes, servedType.String())
}
dr.drLog.Logf("Mirroring %f percent of traffic from keyspace %s to keyspace %s for tablet types [%s]",
dr.drLog.Logf("Mirroring %.2f percent of traffic from keyspace %s to keyspace %s for tablet types [%s]",
percent, dr.ts.SourceKeyspaceName(), dr.ts.TargetKeyspaceName(), strings.Join(tabletTypes, ","))

return nil
Expand Down

0 comments on commit 71b9a00

Please sign in to comment.