From 23024bd18c1537d69cea54d4b1fda715ba9988a5 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 6 Nov 2024 21:49:50 -0500 Subject: [PATCH 01/17] Enable VPlayerBatching by default Signed-off-by: Matt Lord --- go/vt/vttablet/common/flags.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/vt/vttablet/common/flags.go b/go/vt/vttablet/common/flags.go index f9775b8af3e..7eee441d5ec 100644 --- a/go/vt/vttablet/common/flags.go +++ b/go/vt/vttablet/common/flags.go @@ -32,8 +32,7 @@ const ( ) var ( - // Default flags: currently VReplicationExperimentalFlagVPlayerBatching is not enabled by default. - vreplicationExperimentalFlags = VReplicationExperimentalFlagOptimizeInserts | VReplicationExperimentalFlagAllowNoBlobBinlogRowImage + vreplicationExperimentalFlags = VReplicationExperimentalFlagOptimizeInserts | VReplicationExperimentalFlagAllowNoBlobBinlogRowImage | VReplicationExperimentalFlagVPlayerBatching vreplicationNetReadTimeout = 300 vreplicationNetWriteTimeout = 600 vreplicationCopyPhaseDuration = 1 * time.Hour From a5472f40b1fcdaa8e0fa7e8279e68b59cbac77c2 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 6 Nov 2024 21:50:43 -0500 Subject: [PATCH 02/17] Optimize row replication Signed-off-by: Matt Lord --- examples/common/scripts/vttablet-up.sh | 1 + .../vreplication/replicator_plan.go | 57 ++++++++++++------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/examples/common/scripts/vttablet-up.sh b/examples/common/scripts/vttablet-up.sh index daa40aee894..282cd0553ea 100755 --- a/examples/common/scripts/vttablet-up.sh +++ b/examples/common/scripts/vttablet-up.sh @@ -54,6 +54,7 @@ vttablet \ --service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \ --pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \ --heartbeat_on_demand_duration=5s \ + --pprof-http \ > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & # Block waiting for the tablet to be listening diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 6a416cb4414..eba3fe9eb46 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -64,6 +64,13 @@ type ReplicatorPlan struct { workflowConfig *vttablet.VReplicationConfig } +type colInfo struct { + typ querypb.Type + length int64 + offset int64 + field *querypb.Field +} + // buildExecution plan uses the field info as input and the partially built // TablePlan for that table to build a full plan. func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent) (*TablePlan, error) { @@ -224,6 +231,12 @@ type TablePlan struct { CollationEnv *collations.Environment WorkflowConfig *vttablet.VReplicationConfig + + // rowInfo is used as a lazily initiated cache of column information associated + // with querypb.Row values for bulk inserts. The base information is calculated + // once based on the table plan and only the row specific values are updated for + // each row as the row is processed. + rowInfo []*colInfo } // MarshalJSON performs a custom JSON Marshalling. @@ -622,34 +635,38 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { len(tp.Fields), len(bindLocations)) } - type colInfo struct { - typ querypb.Type - length int64 - offset int64 - field *querypb.Field - } - rowInfo := make([]*colInfo, 0) - offset := int64(0) - for i, field := range tp.Fields { // collect info required for fields to be bound - length := row.Lengths[i] - if !tp.FieldsToSkip[strings.ToLower(field.Name)] { - rowInfo = append(rowInfo, &colInfo{ - typ: field.Type, - length: length, - offset: offset, - field: field, - }) + if tp.rowInfo == nil { + tp.rowInfo = make([]*colInfo, 0, len(tp.Fields)) + for i, field := range tp.Fields { // collect info required for fields to be bound + length := row.Lengths[i] + if !tp.FieldsToSkip[strings.ToLower(field.Name)] { + tp.rowInfo = append(tp.rowInfo, &colInfo{ + typ: field.Type, + length: length, + offset: offset, + field: field, + }) + } + if length > 0 { + offset += row.Lengths[i] + } } - if length > 0 { - offset += row.Lengths[i] + } else { + for i, ri := range tp.rowInfo { + length := row.Lengths[i] + ri.length = length + ri.offset = offset + if length > 0 { + offset += row.Lengths[i] + } } } // bind field values to locations var offsetQuery int for i, loc := range bindLocations { - col := rowInfo[i] + col := tp.rowInfo[i] buf.WriteString(tp.BulkInsertValues.Query[offsetQuery:loc.Offset]) typ := col.typ From ab65f0c8041327ea65039804ea5b3396d54705f5 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 7 Nov 2024 11:09:22 -0500 Subject: [PATCH 03/17] Correct rowInfo cache updates with skipped cols Signed-off-by: Matt Lord --- .../tabletmanager/vreplication/replicator_plan.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index eba3fe9eb46..f881dd81de6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -653,10 +653,17 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { } } } else { - for i, ri := range tp.rowInfo { + // i is the index of the field in the plan and row while n is the index of the + // field in the column info list, which is a subset of the others when there + // are fields we need to skip such as generated columns. + n := 0 + for i := range tp.Fields { length := row.Lengths[i] - ri.length = length - ri.offset = offset + if !tp.FieldsToSkip[strings.ToLower(tp.Fields[i].Name)] { + tp.rowInfo[n].length = length + tp.rowInfo[n].offset = offset + n++ + } if length > 0 { offset += row.Lengths[i] } From 4371f8068543ee4f0e23247fdd06736fe9b86819 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 9 Nov 2024 11:57:01 -0500 Subject: [PATCH 04/17] Correct cache mgmt for schema changes and binlog_image=noblob Signed-off-by: Matt Lord --- .../vttablet/tabletmanager/vreplication/replicator_plan.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index f881dd81de6..71c14b70d4c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -100,6 +100,8 @@ func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent return nil, vterrors.Wrapf(err, "failed to build replication plan for %s table", fieldEvent.TableName) } tplan.Fields = fieldEvent.Fields + // Reset the rowInfo cache now that the plan's fields have changed. + tplan.rowInfo = nil return tplan, nil } @@ -232,10 +234,11 @@ type TablePlan struct { CollationEnv *collations.Environment WorkflowConfig *vttablet.VReplicationConfig - // rowInfo is used as a lazily initiated cache of column information associated + // rowInfo is used as a lazily initialized cache of column information associated // with querypb.Row values for bulk inserts. The base information is calculated - // once based on the table plan and only the row specific values are updated for + // using the TablePlan's Fields and only the row specific values are updated for // each row as the row is processed. + // NOTE: this needs to be set to nil anytime the TablePlan's Fields are changed. rowInfo []*colInfo } From ac27fd5ab95af2de0a5b16c9995cfc6b358be16b Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 9 Nov 2024 12:52:36 -0500 Subject: [PATCH 05/17] Adjust help text Signed-off-by: Matt Lord --- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 8c6eafe9c1c..c5f8db0a9a2 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -413,7 +413,7 @@ Flags: --vreplication_copy_phase_duration duration Duration for each copy phase loop (before running the next catchup: default 1h) (default 1h0m0s) --vreplication_copy_phase_max_innodb_history_list_length int The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 1000000) --vreplication_copy_phase_max_mysql_replication_lag int The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 43200) - --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 3) + --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 7) --vreplication_heartbeat_update_interval int Frequency (in seconds, default 1, max 60) at which the time_updated column of a vreplication stream when idling (default 1) --vreplication_max_time_to_retry_on_error duration stop automatically retrying when we've had consecutive failures with the same error for this long after the first occurrence --vreplication_net_read_timeout int Session value of net_read_timeout for vreplication, in seconds (default 300) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 35a07b265bc..84c6651d124 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -413,7 +413,7 @@ Flags: --vreplication_copy_phase_duration duration Duration for each copy phase loop (before running the next catchup: default 1h) (default 1h0m0s) --vreplication_copy_phase_max_innodb_history_list_length int The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 1000000) --vreplication_copy_phase_max_mysql_replication_lag int The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 43200) - --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 3) + --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 7) --vreplication_heartbeat_update_interval int Frequency (in seconds, default 1, max 60) at which the time_updated column of a vreplication stream when idling (default 1) --vreplication_max_time_to_retry_on_error duration stop automatically retrying when we've had consecutive failures with the same error for this long after the first occurrence --vreplication_net_read_timeout int Session value of net_read_timeout for vreplication, in seconds (default 300) From ad8233e1779bdce830ab50e42a833c7777746e5f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 9 Nov 2024 16:10:11 -0500 Subject: [PATCH 06/17] Enable pprof for testing Signed-off-by: Matt Lord --- go/vt/servenv/pprof.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/servenv/pprof.go b/go/vt/servenv/pprof.go index 957c0504c00..414276f48b0 100644 --- a/go/vt/servenv/pprof.go +++ b/go/vt/servenv/pprof.go @@ -299,7 +299,7 @@ func (prof *profile) init() (start func(), stop func()) { func init() { OnParse(func(fs *pflag.FlagSet) { - fs.BoolVar(&httpPprof, "pprof-http", httpPprof, "enable pprof http endpoints") + fs.BoolVar(&httpPprof, "pprof-http", true, "enable pprof http endpoints") fs.StringSliceVar(&pprofFlag, "pprof", pprofFlag, "enable profiling") }) OnInit(pprofInit) From 11b3493bae73d1089bf00d7de9cac63bf66dc763 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 9 Nov 2024 21:54:29 -0500 Subject: [PATCH 07/17] Get rid of the additional row/col info slice altogether Signed-off-by: Matt Lord --- .../vreplication/replicator_plan.go | 77 +++++++------------ .../tabletmanager/vreplication/vdbclient.go | 2 +- 2 files changed, 27 insertions(+), 52 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 71c14b70d4c..bc2c74a96b1 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -100,8 +100,6 @@ func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent return nil, vterrors.Wrapf(err, "failed to build replication plan for %s table", fieldEvent.TableName) } tplan.Fields = fieldEvent.Fields - // Reset the rowInfo cache now that the plan's fields have changed. - tplan.rowInfo = nil return tplan, nil } @@ -233,13 +231,6 @@ type TablePlan struct { CollationEnv *collations.Environment WorkflowConfig *vttablet.VReplicationConfig - - // rowInfo is used as a lazily initialized cache of column information associated - // with querypb.Row values for bulk inserts. The base information is calculated - // using the TablePlan's Fields and only the row specific values are updated for - // each row as the row is processed. - // NOTE: this needs to be set to nil anytime the TablePlan's Fields are changed. - rowInfo []*colInfo } // MarshalJSON performs a custom JSON Marshalling. @@ -638,56 +629,36 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { len(tp.Fields), len(bindLocations)) } - offset := int64(0) - if tp.rowInfo == nil { - tp.rowInfo = make([]*colInfo, 0, len(tp.Fields)) - for i, field := range tp.Fields { // collect info required for fields to be bound - length := row.Lengths[i] - if !tp.FieldsToSkip[strings.ToLower(field.Name)] { - tp.rowInfo = append(tp.rowInfo, &colInfo{ - typ: field.Type, - length: length, - offset: offset, - field: field, - }) - } - if length > 0 { - offset += row.Lengths[i] - } - } - } else { - // i is the index of the field in the plan and row while n is the index of the - // field in the column info list, which is a subset of the others when there - // are fields we need to skip such as generated columns. - n := 0 - for i := range tp.Fields { - length := row.Lengths[i] - if !tp.FieldsToSkip[strings.ToLower(tp.Fields[i].Name)] { - tp.rowInfo[n].length = length - tp.rowInfo[n].offset = offset - n++ - } + // Bind field values to locations. + var ( + offset int64 + offsetQuery int + fieldsIndex int + field *querypb.Field + ) + for i, loc := range bindLocations { + field = tp.Fields[fieldsIndex] + length := row.Lengths[fieldsIndex] + for tp.FieldsToSkip[strings.ToLower(field.Name)] { if length > 0 { - offset += row.Lengths[i] + offset += length } + fieldsIndex++ + field = tp.Fields[fieldsIndex] + length = row.Lengths[fieldsIndex] } - } - // bind field values to locations - var offsetQuery int - for i, loc := range bindLocations { - col := tp.rowInfo[i] buf.WriteString(tp.BulkInsertValues.Query[offsetQuery:loc.Offset]) - typ := col.typ + typ := field.Type switch typ { case querypb.Type_TUPLE: return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected Type_TUPLE for value %d", i) case querypb.Type_JSON: - if col.length < 0 { // An SQL NULL and not an actual JSON value + if length < 0 { // An SQL NULL and not an actual JSON value buf.WriteString(sqltypes.NullStr) } else { // A JSON value (which may be a JSON null literal value) - buf2 := row.Values[col.offset : col.offset+col.length] + buf2 := row.Values[offset : offset+length] vv, err := vjson.MarshalSQLValue(buf2) if err != nil { return err @@ -695,16 +666,16 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { buf.WriteString(vv.RawStr()) } default: - if col.length < 0 { + if length < 0 { // -1 means a null variable; serialize it directly buf.WriteString(sqltypes.NullStr) } else { - raw := row.Values[col.offset : col.offset+col.length] + raw := row.Values[offset : offset+length] var vv sqltypes.Value - if conversion, ok := tp.ConvertCharset[col.field.Name]; ok && col.length > 0 { + if conversion, ok := tp.ConvertCharset[field.Name]; ok && length > 0 { // Non-null string value, for which we have a charset conversion instruction - out, err := tp.convertStringCharset(raw, conversion, col.field.Name) + out, err := tp.convertStringCharset(raw, conversion, field.Name) if err != nil { return err } @@ -717,6 +688,10 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { } } offsetQuery = loc.Offset + loc.Length + if length > 0 { + offset += length + } + fieldsIndex++ } buf.WriteString(tp.BulkInsertValues.Query[offsetQuery:]) return nil diff --git a/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go b/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go index b8339cdf874..8a4409db06c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go @@ -171,7 +171,7 @@ func (vc *vdbClient) Execute(query string) (*sqltypes.Result, error) { func (vc *vdbClient) ExecuteWithRetry(ctx context.Context, query string) (*sqltypes.Result, error) { qr, err := vc.Execute(query) for err != nil { - if sqlErr, ok := err.(*sqlerror.SQLError); ok && sqlErr.Number() == sqlerror.ERLockDeadlock || sqlErr.Number() == sqlerror.ERLockWaitTimeout { + if sqlErr, ok := err.(*sqlerror.SQLError); ok && (sqlErr.Number() == sqlerror.ERLockDeadlock || sqlErr.Number() == sqlerror.ERLockWaitTimeout) { log.Infof("retryable error: %v, waiting for %v and retrying", sqlErr, dbLockRetryDelay) if err := vc.Rollback(); err != nil { return nil, err From ed81da1c63e7667248a8428a8aaf20d0a5b13485 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 9 Nov 2024 22:06:30 -0500 Subject: [PATCH 08/17] Restore pprof_http default Signed-off-by: Matt Lord --- go/vt/servenv/pprof.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/servenv/pprof.go b/go/vt/servenv/pprof.go index 414276f48b0..957c0504c00 100644 --- a/go/vt/servenv/pprof.go +++ b/go/vt/servenv/pprof.go @@ -299,7 +299,7 @@ func (prof *profile) init() (start func(), stop func()) { func init() { OnParse(func(fs *pflag.FlagSet) { - fs.BoolVar(&httpPprof, "pprof-http", true, "enable pprof http endpoints") + fs.BoolVar(&httpPprof, "pprof-http", httpPprof, "enable pprof http endpoints") fs.StringSliceVar(&pprofFlag, "pprof", pprofFlag, "enable profiling") }) OnInit(pprofInit) From fcfa37632c121c4ffde48440d5e57a48b0b2a2d7 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 10 Nov 2024 00:01:57 -0500 Subject: [PATCH 09/17] colInfo type is no longer needed Signed-off-by: Matt Lord --- .../vttablet/tabletmanager/vreplication/replicator_plan.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index bc2c74a96b1..3c2a1800e31 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -64,13 +64,6 @@ type ReplicatorPlan struct { workflowConfig *vttablet.VReplicationConfig } -type colInfo struct { - typ querypb.Type - length int64 - offset int64 - field *querypb.Field -} - // buildExecution plan uses the field info as input and the partially built // TablePlan for that table to build a full plan. func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent) (*TablePlan, error) { From 2e3334f1af7eb1d400104f179060d008f03059c4 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 10 Nov 2024 00:37:18 -0500 Subject: [PATCH 10/17] VPlayerBatching is now enabled by default in all tests Signed-off-by: Matt Lord --- .../endtoend/onlineddl/flow/onlineddl_flow_test.go | 4 ---- .../onlineddl_vrepl_mini_stress_test.go | 4 ---- .../onlineddl_vrepl_stress_suite_test.go | 4 ---- go/test/endtoend/vreplication/cluster_test.go | 13 ------------- go/test/endtoend/vreplication/fk_test.go | 4 ---- go/test/endtoend/vreplication/vdiff2_test.go | 4 ---- go/test/endtoend/vreplication/vreplication_test.go | 3 --- 7 files changed, 36 deletions(-) diff --git a/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go b/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go index c442c042f8a..035789e4b87 100644 --- a/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go +++ b/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go @@ -63,7 +63,6 @@ import ( "vitess.io/vitess/go/test/endtoend/throttler" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/schema" - vttablet "vitess.io/vitess/go/vt/vttablet/common" throttlebase "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" ) @@ -145,9 +144,6 @@ func TestMain(m *testing.M) { "--heartbeat_on_demand_duration", "5s", "--migration_check_interval", "2s", "--watch_replication_stream", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } clusterInstance.VtGateExtraArgs = []string{ "--ddl_strategy", "online", diff --git a/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go b/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go index e0dd9701cf8..88c145dc40c 100644 --- a/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go +++ b/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go @@ -38,7 +38,6 @@ import ( "vitess.io/vitess/go/test/endtoend/throttler" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/schema" - vttablet "vitess.io/vitess/go/vt/vttablet/common" ) type WriteMetrics struct { @@ -184,9 +183,6 @@ func TestMain(m *testing.M) { "--heartbeat_on_demand_duration", "5s", "--migration_check_interval", "5s", "--watch_replication_stream", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } clusterInstance.VtGateExtraArgs = []string{ "--ddl_strategy", "online", diff --git a/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go b/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go index 440b921f9ba..85b3585beb4 100644 --- a/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go +++ b/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go @@ -51,7 +51,6 @@ import ( "vitess.io/vitess/go/timer" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/schema" - vttablet "vitess.io/vitess/go/vt/vttablet/common" ) type testcase struct { @@ -436,9 +435,6 @@ func TestMain(m *testing.M) { "--migration_check_interval", "5s", "--vstream_packet_size", "4096", // Keep this value small and below 10k to ensure multilple vstream iterations "--watch_replication_stream", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } clusterInstance.VtGateExtraArgs = []string{ "--ddl_strategy", "online", diff --git a/go/test/endtoend/vreplication/cluster_test.go b/go/test/endtoend/vreplication/cluster_test.go index 119843651bc..dc5a72e5e88 100644 --- a/go/test/endtoend/vreplication/cluster_test.go +++ b/go/test/endtoend/vreplication/cluster_test.go @@ -39,7 +39,6 @@ import ( "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - vttablet "vitess.io/vitess/go/vt/vttablet/common" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) @@ -101,18 +100,6 @@ func (cc *ClusterConfig) enableGTIDCompression() func() { } } -// setAllVTTabletExperimentalFlags sets all the experimental flags for vttablet and returns a function -// that can be used to reset them in a defer. -func setAllVTTabletExperimentalFlags() func() { - experimentalArgs := fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching) - oldArgs := extraVTTabletArgs - extraVTTabletArgs = append(extraVTTabletArgs, experimentalArgs) - return func() { - extraVTTabletArgs = oldArgs - } -} - // VitessCluster represents all components within the test cluster type VitessCluster struct { t *testing.T diff --git a/go/test/endtoend/vreplication/fk_test.go b/go/test/endtoend/vreplication/fk_test.go index 34881cbcd1a..f977d5a74cd 100644 --- a/go/test/endtoend/vreplication/fk_test.go +++ b/go/test/endtoend/vreplication/fk_test.go @@ -29,7 +29,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/log" - vttablet "vitess.io/vitess/go/vt/vttablet/common" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) @@ -43,9 +42,6 @@ func TestFKWorkflow(t *testing.T) { extraVTTabletArgs = []string{ // Ensure that there are multiple copy phase cycles per table. "--vstream_packet_size=256", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } defer func() { extraVTTabletArgs = nil }() diff --git a/go/test/endtoend/vreplication/vdiff2_test.go b/go/test/endtoend/vreplication/vdiff2_test.go index aaf4cae5375..612ba00236b 100644 --- a/go/test/endtoend/vreplication/vdiff2_test.go +++ b/go/test/endtoend/vreplication/vdiff2_test.go @@ -36,7 +36,6 @@ import ( "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" - vttablet "vitess.io/vitess/go/vt/vttablet/common" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" @@ -140,9 +139,6 @@ func TestVDiff2(t *testing.T) { extraVTTabletArgs = []string{ // This forces us to use multiple vstream packets even with small test tables. "--vstream_packet_size=1", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } vc = NewVitessCluster(t, &clusterOptions{cells: strings.Split(cellNames, ",")}) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 04a5eabc33b..d3193298a0c 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -293,7 +293,6 @@ func TestVreplicationCopyThrottling(t *testing.T) { } func TestBasicVreplicationWorkflow(t *testing.T) { - defer setAllVTTabletExperimentalFlags() sourceKsOpts["DBTypeVersion"] = "mysql-8.0" targetKsOpts["DBTypeVersion"] = "mysql-8.0" testBasicVreplicationWorkflow(t, "noblob") @@ -595,8 +594,6 @@ func TestCellAliasVreplicationWorkflow(t *testing.T) { cells := []string{"zone1", "zone2"} resetCompression := mainClusterConfig.enableGTIDCompression() defer resetCompression() - resetExperimentalFlags := setAllVTTabletExperimentalFlags() - defer resetExperimentalFlags() vc = NewVitessCluster(t, &clusterOptions{cells: cells}) defer vc.TearDown() From aa359bcc900f1ad9fc32b05dc008ac5a6b571eba Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 10 Nov 2024 23:32:14 -0500 Subject: [PATCH 11/17] Add comment about field number gap in topodata.FieldEvent Signed-off-by: Matt Lord --- proto/binlogdata.proto | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 595760dcd52..e1df792776b 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -353,6 +353,10 @@ message FieldEvent { repeated query.Field fields = 2; string keyspace = 3; string shard = 4; + + // Field numbers in the gap between shard (4) and enum_set_string_values + // (25) are NOT reserved and can be used. + // Are ENUM and SET field values already mapped to strings in the ROW // events? This allows us to transition VTGate VStream consumers from // the pre v20 behavior of having to do this mapping themselves to the @@ -362,6 +366,9 @@ message FieldEvent { // vstreams managed by the vstreamManager. bool enum_set_string_values = 25; bool is_internal_table = 26; // set for sidecardb tables + + // Add new members in the field number gap between shard (4) and + // enum_set_string_values (25). } // ShardGtid contains the GTID position for one shard. From 5f5c76ab6dfa425fbe50ed9096ceb0a90edd2cc2 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 11 Nov 2024 01:55:44 -0500 Subject: [PATCH 12/17] Add unit test Signed-off-by: Matt Lord --- .../vreplication/replicator_plan.go | 7 +- .../vreplication/replicator_plan_test.go | 116 +++++++++++++++++- 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 3c2a1800e31..6cf88a48f75 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -617,9 +617,10 @@ func valsEqual(v1, v2 sqltypes.Value) bool { // on the source: sum/count for aggregation queries, for example. func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { bindLocations := tp.BulkInsertValues.BindLocations() - if len(tp.Fields) < len(bindLocations) { - return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations ", - len(tp.Fields), len(bindLocations)) + usedFieldCnt := len(tp.Fields) - len(tp.FieldsToSkip) + if usedFieldCnt != len(bindLocations) { + return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations", + usedFieldCnt, len(bindLocations)) } // Bind field values to locations. diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 644b4585914..3b46a08a5cd 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -21,17 +21,18 @@ import ( "strings" "testing" - vttablet "vitess.io/vitess/go/vt/vttablet/common" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/bytes2" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/sqlparser" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + vttablet "vitess.io/vitess/go/vt/vttablet/common" ) type TestReplicatorPlan struct { @@ -829,3 +830,114 @@ func TestBuildPlayerPlanExclude(t *testing.T) { wantPlan, _ := json.Marshal(want) assert.Equal(t, string(gotPlan), string(wantPlan)) } + +func TestAppendFromRow(t *testing.T) { + testCases := []struct { + name string + tp *TablePlan + row *querypb.Row + want string + wantErr string + }{ + { + name: "simple", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", + ":c1", ":c2", ":c3", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + }, + }, + row: sqltypes.RowToProto3( + []sqltypes.Value{ + sqltypes.NewInt64(1), + sqltypes.NewInt64(2), + sqltypes.NewInt64(3), + }, + ), + want: "values (1, 2, 3)", + }, + { + name: "too few fields", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", + ":c1", ":c2", ":c3", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + }, + }, + wantErr: "wrong number of fields: got 2 fields for 3 bind locations", + }, + { + name: "too few non-skipped fields", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", + ":c1", ":c2", ":c3", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + {Name: "c4", Type: querypb.Type_INT32}, + }, + FieldsToSkip: map[string]bool{ + "c3": true, + "c4": true, + }, + }, + wantErr: "wrong number of fields: got 2 fields for 3 bind locations", + }, + { + name: "lots o skippin", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", + ":c1", ":c2", ":c4", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + {Name: "c4", Type: querypb.Type_INT32}, + {Name: "c5", Type: querypb.Type_INT32}, + {Name: "c6", Type: querypb.Type_INT32}, + {Name: "c7", Type: querypb.Type_INT32}, + }, + FieldsToSkip: map[string]bool{ + "c3": true, + "c5": true, + "c6": true, + "c7": true, + }, + }, + row: sqltypes.RowToProto3( + []sqltypes.Value{ + sqltypes.NewInt64(1), + sqltypes.NewInt64(2), + sqltypes.NewInt64(3), + sqltypes.NewInt64(4), + sqltypes.NewInt64(5), + sqltypes.NewInt64(6), + sqltypes.NewInt64(7), + }, + ), + want: "values (1, 2, 4)", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bb := &bytes2.Buffer{} + err := tc.tp.appendFromRow(bb, tc.row) + if tc.wantErr != "" { + require.EqualError(t, err, tc.wantErr) + } else { + require.NoError(t, err) + require.Equal(t, tc.want, bb.String()) + } + }) + } +} From f0f61dbbbec3c3cb6490e66e236e0901064c0e12 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 11 Nov 2024 16:05:54 -0500 Subject: [PATCH 13/17] Deflake vdiffs in e2e tests using load generator This impacted e2e tests that were not previously using VPlayerBatching. The load generator constantly generates INSERTs, which are then effeciently batched in vplayer so we get ~ 7x more throughput than before and thus we need more time for filtered replication to catch up after we've stopped it for the vdiff. ESPECIALLY since we're using the --update-table-stats flag and the ANALYZE TABLE and its locking causes a pause in updates to the table the load generator is inserting into -- in particular for the test clusters that only have PRIMARY tablets as everything is interacting directly. Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/vdiff_helper_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/vreplication/vdiff_helper_test.go b/go/test/endtoend/vreplication/vdiff_helper_test.go index 561edfe8b7e..fd223d78082 100644 --- a/go/test/endtoend/vreplication/vdiff_helper_test.go +++ b/go/test/endtoend/vreplication/vdiff_helper_test.go @@ -35,7 +35,7 @@ import ( ) const ( - vdiffTimeout = 120 * time.Second // We can leverage auto retry on error with this longer-than-usual timeout + vdiffTimeout = 180 * time.Second // We can leverage auto retry on error with this longer-than-usual timeout vdiffRetryTimeout = 30 * time.Second vdiffStatusCheckInterval = 5 * time.Second vdiffRetryInterval = 5 * time.Second @@ -71,7 +71,8 @@ func doVtctlclientVDiff(t *testing.T, keyspace, workflow, cells string, want *ex ksWorkflow := fmt.Sprintf("%s.%s", keyspace, workflow) t.Run(fmt.Sprintf("vtctlclient vdiff %s", ksWorkflow), func(t *testing.T) { // update-table-stats is needed in order to test progress reports. - uuid, _ := performVDiff2Action(t, true, ksWorkflow, cells, "create", "", false, "--auto-retry", "--update-table-stats") + uuid, _ := performVDiff2Action(t, true, ksWorkflow, cells, "create", "", false, "--auto-retry", + "--update-table-stats", fmt.Sprintf("--filtered_replication_wait_time=%v", vdiffTimeout/2)) info := waitForVDiff2ToComplete(t, true, ksWorkflow, cells, uuid, time.Time{}) require.NotNil(t, info) require.Equal(t, workflow, info.Workflow) @@ -164,7 +165,7 @@ func doVtctldclientVDiff(t *testing.T, keyspace, workflow, cells string, want *e ksWorkflow := fmt.Sprintf("%s.%s", keyspace, workflow) t.Run(fmt.Sprintf("vtctldclient vdiff %s", ksWorkflow), func(t *testing.T) { // update-table-stats is needed in order to test progress reports. - flags := []string{"--auto-retry", "--update-table-stats"} + flags := []string{"--auto-retry", "--update-table-stats", fmt.Sprintf("--filtered-replication-wait-time=%v", vdiffTimeout/2)} if len(extraFlags) > 0 { flags = append(flags, extraFlags...) } From 21564fd6b496bcc0b06e37dbf1f6e53b5d8a4795 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 11 Nov 2024 16:55:26 -0500 Subject: [PATCH 14/17] Revert Fields and bindLocations comparison change It failed for some materializations Signed-off-by: Matt Lord --- go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 6cf88a48f75..62d6166b5ca 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -617,10 +617,9 @@ func valsEqual(v1, v2 sqltypes.Value) bool { // on the source: sum/count for aggregation queries, for example. func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { bindLocations := tp.BulkInsertValues.BindLocations() - usedFieldCnt := len(tp.Fields) - len(tp.FieldsToSkip) - if usedFieldCnt != len(bindLocations) { + if len(tp.Fields) < len(bindLocations) { return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations", - usedFieldCnt, len(bindLocations)) + len(tp.Fields), len(bindLocations)) } // Bind field values to locations. From b8a3271e6d939bc54abc10e1753949d5098073f1 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 11 Nov 2024 17:44:56 -0500 Subject: [PATCH 15/17] Adjust unit test Signed-off-by: Matt Lord --- .../vreplication/replicator_plan_test.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 3b46a08a5cd..8525541f0b0 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -873,25 +873,6 @@ func TestAppendFromRow(t *testing.T) { }, wantErr: "wrong number of fields: got 2 fields for 3 bind locations", }, - { - name: "too few non-skipped fields", - tp: &TablePlan{ - BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", - ":c1", ":c2", ":c3", - ), - Fields: []*querypb.Field{ - {Name: "c1", Type: querypb.Type_INT32}, - {Name: "c2", Type: querypb.Type_INT32}, - {Name: "c3", Type: querypb.Type_INT32}, - {Name: "c4", Type: querypb.Type_INT32}, - }, - FieldsToSkip: map[string]bool{ - "c3": true, - "c4": true, - }, - }, - wantErr: "wrong number of fields: got 2 fields for 3 bind locations", - }, { name: "lots o skippin", tp: &TablePlan{ From 5f15e3fd7ffeeb6e9c7815c2ebc59697a7fa2830 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 11 Nov 2024 21:57:56 -0500 Subject: [PATCH 16/17] Unit test tweaks Signed-off-by: Matt Lord --- .../vreplication/replicator_plan_test.go | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 8525541f0b0..09ace916f11 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -874,10 +874,47 @@ func TestAppendFromRow(t *testing.T) { wantErr: "wrong number of fields: got 2 fields for 3 bind locations", }, { - name: "lots o skippin", + name: "skip half", tp: &TablePlan{ - BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", - ":c1", ":c2", ":c4", + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a, %a)", + ":c1", ":c2", ":c4", ":c8", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + {Name: "c4", Type: querypb.Type_INT32}, + {Name: "c5", Type: querypb.Type_INT32}, + {Name: "c6", Type: querypb.Type_INT32}, + {Name: "c7", Type: querypb.Type_INT32}, + {Name: "c8", Type: querypb.Type_INT32}, + }, + FieldsToSkip: map[string]bool{ + "c3": true, + "c5": true, + "c6": true, + "c7": true, + }, + }, + row: sqltypes.RowToProto3( + []sqltypes.Value{ + sqltypes.NewInt64(1), + sqltypes.NewInt64(2), + sqltypes.NewInt64(3), + sqltypes.NewInt64(4), + sqltypes.NewInt64(5), + sqltypes.NewInt64(6), + sqltypes.NewInt64(7), + sqltypes.NewInt64(8), + }, + ), + want: "values (1, 2, 4, 8)", + }, + { + name: "skip all but one", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a)", + ":c4", ), Fields: []*querypb.Field{ {Name: "c1", Type: querypb.Type_INT32}, @@ -887,12 +924,16 @@ func TestAppendFromRow(t *testing.T) { {Name: "c5", Type: querypb.Type_INT32}, {Name: "c6", Type: querypb.Type_INT32}, {Name: "c7", Type: querypb.Type_INT32}, + {Name: "c8", Type: querypb.Type_INT32}, }, FieldsToSkip: map[string]bool{ + "c1": true, + "c2": true, "c3": true, "c5": true, "c6": true, "c7": true, + "c8": true, }, }, row: sqltypes.RowToProto3( @@ -904,9 +945,10 @@ func TestAppendFromRow(t *testing.T) { sqltypes.NewInt64(5), sqltypes.NewInt64(6), sqltypes.NewInt64(7), + sqltypes.NewInt64(8), }, ), - want: "values (1, 2, 4)", + want: "values (4)", }, } for _, tc := range testCases { From ae1928b98155857c213adbb7a5a2e11043f7293a Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 3 Dec 2024 08:34:23 -0500 Subject: [PATCH 17/17] Enable pprof-http for vtgate and vtctld as well Signed-off-by: Matt Lord --- examples/common/scripts/vtctld-up.sh | 1 + examples/common/scripts/vtgate-up.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/common/scripts/vtctld-up.sh b/examples/common/scripts/vtctld-up.sh index 6902a851997..4a4b2587c4f 100755 --- a/examples/common/scripts/vtctld-up.sh +++ b/examples/common/scripts/vtctld-up.sh @@ -33,6 +33,7 @@ vtctld \ --port $vtctld_web_port \ --grpc_port $grpc_port \ --pid_file $VTDATAROOT/tmp/vtctld.pid \ + --pprof-http \ > $VTDATAROOT/tmp/vtctld.out 2>&1 & for _ in {0..300}; do diff --git a/examples/common/scripts/vtgate-up.sh b/examples/common/scripts/vtgate-up.sh index dbaaad02367..fd7860cf6ba 100755 --- a/examples/common/scripts/vtgate-up.sh +++ b/examples/common/scripts/vtgate-up.sh @@ -41,6 +41,7 @@ vtgate \ --pid_file $VTDATAROOT/tmp/vtgate.pid \ --enable_buffer \ --mysql_auth_server_impl none \ + --pprof-http \ > $VTDATAROOT/tmp/vtgate.out 2>&1 & # Block waiting for vtgate to be listening