Skip to content

Commit

Permalink
implement --max-report-sample-rows for VDiff (#14437)
Browse files Browse the repository at this point in the history 
Signed-off-by: Matthias Crauwels <[email protected]>
Signed-off-by: Rohit Nayak <[email protected]>
Co-authored-by: Rohit Nayak <[email protected]>
  • Loading branch information
@mcrauwel @rohit-nayak-ps
mcrauwel and rohit-nayak-ps authored Nov 24, 2023
1 parent 8b7ed8d commit e08d151
Show file tree
Hide file tree
Showing 15 changed files with 766 additions and 324 deletions.
3 changes: 3 additions & 0 deletions go/cmd/vtctldclient/command/vreplication/vdiff/vdiff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ var (
Limit uint32 // We only accept positive values but pass on an int64
FilteredReplicationWaitTime time.Duration
DebugQuery bool
MaxReportSampleRows uint32 // We only accept positive values but pass on an int64
OnlyPKs bool
UpdateTableStats bool
MaxExtraRowsToCompare uint32 // We only accept positive values but pass on an int64
Expand Down Expand Up @@ -279,6 +280,7 @@ func commandCreate(cmd *cobra.Command, args []string) error {
Wait: createOptions.Wait,
WaitUpdateInterval: protoutil.DurationToProto(createOptions.WaitUpdateInterval),
AutoRetry: createOptions.AutoRetry,
MaxReportSampleRows: int64(createOptions.MaxReportSampleRows),
})

if err != nil {
Expand Down Expand Up @@ -863,6 +865,7 @@ func registerCommands(root *cobra.Command) {
create.Flags().DurationVar(&createOptions.FilteredReplicationWaitTime, "filtered-replication-wait-time", 30*time.Second, "Specifies the maximum time to wait, in seconds, for replication to catch up when syncing tablet streams.")
create.Flags().Uint32Var(&createOptions.Limit, "limit", math.MaxUint32, "Max rows to stop comparing after.")
create.Flags().BoolVar(&createOptions.DebugQuery, "debug-query", false, "Adds a mysql query to the report that can be used for further debugging.")
create.Flags().Uint32Var(&createOptions.MaxReportSampleRows, "max-report-sample-rows", 10, "Maximum number of row differences to report (0 for all differences). NOTE: when increasing this value it is highly recommended to also specify --only-pks")
create.Flags().BoolVar(&createOptions.OnlyPKs, "only-pks", false, "When reporting missing rows, only show primary keys in the report.")
create.Flags().StringSliceVar(&createOptions.Tables, "tables", nil, "Only run vdiff for these tables in the workflow.")
create.Flags().Uint32Var(&createOptions.MaxExtraRowsToCompare, "max-extra-rows-to-compare", 1000, "If there are collation differences between the source and target, you can have rows that are identical but simply returned in a different order from MySQL. We will do a second pass to compare the rows for any actual differences in this case and this flag allows you to control the resources used for this operation.")
Expand Down
271 changes: 271 additions & 0 deletions go/cmd/vtctldclient/command/vreplication/vdiff/vdiff_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,277 @@ func TestVDiffUnsharded(t *testing.T) {
}
}
]`),
}, {
id: "9", // --max-vdiff-report-rows=20 --only-pks
result: sqltypes.MakeTestResult(fields,
"completed||t1|"+UUID+"|completed|30|"+starttime+"|30|"+comptime+"|1|"+
`{"TableName": "t1", "MatchingRows": 10, "ProcessedRows": 30, "MismatchedRows": 20, "ExtraRowsSource": 0, `+
`"ExtraRowsTarget": 0, "MismatchedRowsSample": [`+
`{"Source": {"Row": {"c1": "2"}}, "Target": {"Row": {"c1": "2"}}},`+
`{"Source": {"Row": {"c1": "3"}}, "Target": {"Row": {"c1": "3"}}},`+
`{"Source": {"Row": {"c1": "4"}}, "Target": {"Row": {"c1": "4"}}},`+
`{"Source": {"Row": {"c1": "5"}}, "Target": {"Row": {"c1": "5"}}},`+
`{"Source": {"Row": {"c1": "6"}}, "Target": {"Row": {"c1": "6"}}},`+
`{"Source": {"Row": {"c1": "7"}}, "Target": {"Row": {"c1": "7"}}},`+
`{"Source": {"Row": {"c1": "8"}}, "Target": {"Row": {"c1": "8"}}},`+
`{"Source": {"Row": {"c1": "9"}}, "Target": {"Row": {"c1": "9"}}},`+
`{"Source": {"Row": {"c1": "10"}}, "Target": {"Row": {"c1": "10"}}},`+
`{"Source": {"Row": {"c1": "11"}}, "Target": {"Row": {"c1": "11"}}},`+
`{"Source": {"Row": {"c1": "12"}}, "Target": {"Row": {"c1": "12"}}},`+
`{"Source": {"Row": {"c1": "13"}}, "Target": {"Row": {"c1": "13"}}},`+
`{"Source": {"Row": {"c1": "14"}}, "Target": {"Row": {"c1": "14"}}},`+
`{"Source": {"Row": {"c1": "15"}}, "Target": {"Row": {"c1": "15"}}},`+
`{"Source": {"Row": {"c1": "16"}}, "Target": {"Row": {"c1": "16"}}},`+
`{"Source": {"Row": {"c1": "17"}}, "Target": {"Row": {"c1": "17"}}},`+
`{"Source": {"Row": {"c1": "18"}}, "Target": {"Row": {"c1": "18"}}},`+
`{"Source": {"Row": {"c1": "19"}}, "Target": {"Row": {"c1": "19"}}},`+
`{"Source": {"Row": {"c1": "20"}}, "Target": {"Row": {"c1": "20"}}},`+
`{"Source": {"Row": {"c1": "21"}}, "Target": {"Row": {"c1": "21"}}}`+
`]}`),
report: fmt.Sprintf(badReportfmt,
env.targetKeyspace, UUID, 30, true, starttime, comptime, 30, 10, 20, 0, 0, 30, 10, 20, 0, 0,
`"MismatchedRowsSample": [
{
"Source": {
"Row": {
"c1": "2"
}
},
"Target": {
"Row": {
"c1": "2"
}
}
},
{
"Source": {
"Row": {
"c1": "3"
}
},
"Target": {
"Row": {
"c1": "3"
}
}
},
{
"Source": {
"Row": {
"c1": "4"
}
},
"Target": {
"Row": {
"c1": "4"
}
}
},
{
"Source": {
"Row": {
"c1": "5"
}
},
"Target": {
"Row": {
"c1": "5"
}
}
},
{
"Source": {
"Row": {
"c1": "6"
}
},
"Target": {
"Row": {
"c1": "6"
}
}
},
{
"Source": {
"Row": {
"c1": "7"
}
},
"Target": {
"Row": {
"c1": "7"
}
}
},
{
"Source": {
"Row": {
"c1": "8"
}
},
"Target": {
"Row": {
"c1": "8"
}
}
},
{
"Source": {
"Row": {
"c1": "9"
}
},
"Target": {
"Row": {
"c1": "9"
}
}
},
{
"Source": {
"Row": {
"c1": "10"
}
},
"Target": {
"Row": {
"c1": "10"
}
}
},
{
"Source": {
"Row": {
"c1": "11"
}
},
"Target": {
"Row": {
"c1": "11"
}
}
},
{
"Source": {
"Row": {
"c1": "12"
}
},
"Target": {
"Row": {
"c1": "12"
}
}
},
{
"Source": {
"Row": {
"c1": "13"
}
},
"Target": {
"Row": {
"c1": "13"
}
}
},
{
"Source": {
"Row": {
"c1": "14"
}
},
"Target": {
"Row": {
"c1": "14"
}
}
},
{
"Source": {
"Row": {
"c1": "15"
}
},
"Target": {
"Row": {
"c1": "15"
}
}
},
{
"Source": {
"Row": {
"c1": "16"
}
},
"Target": {
"Row": {
"c1": "16"
}
}
},
{
"Source": {
"Row": {
"c1": "17"
}
},
"Target": {
"Row": {
"c1": "17"
}
}
},
{
"Source": {
"Row": {
"c1": "18"
}
},
"Target": {
"Row": {
"c1": "18"
}
}
},
{
"Source": {
"Row": {
"c1": "19"
}
},
"Target": {
"Row": {
"c1": "19"
}
}
},
{
"Source": {
"Row": {
"c1": "20"
}
},
"Target": {
"Row": {
"c1": "20"
}
}
},
{
"Source": {
"Row": {
"c1": "21"
}
},
"Target": {
"Row": {
"c1": "21"
}
}
}
]`),
},
}

Expand Down
Loading

0 comments on commit e08d151

Please sign in to comment.