From 40061ccd9bb3e0c0d4df534f968a017e753e358a Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 24 Nov 2023 11:49:24 +0100 Subject: [PATCH 01/15] refactor: move the use of Input over to InsertRows for unsharded Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/insert.go | 87 +++++++++--------- go/vt/vtgate/engine/insert_rows.go | 73 +++++++++++++++ go/vt/vtgate/engine/insert_test.go | 137 ++++++++++++++++------------- go/vt/vtgate/planbuilder/insert.go | 9 +- 4 files changed, 199 insertions(+), 107 deletions(-) create mode 100644 go/vt/vtgate/engine/insert_rows.go diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 4fa8eeadce4..8455836d717 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -58,6 +58,8 @@ type ( // and Prefix, Mid and Suffix are used instead. Query string + InsertRows *InsertRows + // VindexValues specifies values for all the vindex columns. // This is a three-dimensional data structure: // Insert.Values[i] represents the values to be inserted for the i'th colvindex (i < len(Insert.Table.ColumnVindexes)) @@ -112,10 +114,7 @@ type ( ) func (ins *Insert) Inputs() ([]Primitive, []map[string]any) { - if ins.Input == nil { - return nil, nil - } - return []Primitive{ins.Input}, nil + return ins.InsertRows.Inputs() } // NewQueryInsert creates an Insert with a query string. @@ -224,7 +223,7 @@ func (ins *Insert) GetTableName() string { } // TryExecute performs a non-streaming exec. -func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool) (*sqltypes.Result, error) { +func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, _ bool) (*sqltypes.Result, error) { ctx, cancelFunc := addQueryTimeout(ctx, vcursor, ins.QueryTimeout) defer cancelFunc() @@ -232,7 +231,7 @@ func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map case InsertUnsharded: return ins.execInsertUnsharded(ctx, vcursor, bindVars) case InsertSharded: - return ins.execInsertSharded(ctx, vcursor, bindVars) + return ins.insertIntoShardedTableFromValues(ctx, vcursor, bindVars) case InsertSelect: return ins.execInsertFromSelect(ctx, vcursor, bindVars) default: @@ -277,7 +276,7 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa if unsharded { insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, result) } else { - insertID, qr, err = ins.insertIntoShardedTable(ctx, vcursor, bindVars, result) + insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result.Rows) } if err != nil { return err @@ -296,24 +295,6 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa return callback(output) } -func (ins *Insert) insertIntoShardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, result *sqltypes.Result) (int64, *sqltypes.Result, error) { - insertID, err := ins.processGenerateFromRows(ctx, vcursor, result.Rows) - if err != nil { - return 0, nil, err - } - - rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, result.Rows) - if err != nil { - return 0, nil, err - } - - qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, insertID) - if err != nil { - return 0, nil, err - } - return insertID, qr, nil -} - // GetFields fetches the field info. func (ins *Insert) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unreachable code for %q", ins.Query) @@ -329,14 +310,14 @@ func (ins *Insert) execInsertUnsharded(ctx context.Context, vcursor VCursor, bin if len(result.Rows) == 0 { return &sqltypes.Result{}, nil } - query = ins.getInsertQueryForUnsharded(result, bindVars) + query = ins.getInsertQueryFromSelectForUnsharded(result, bindVars) } _, qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) return qr, err } -func (ins *Insert) getInsertQueryForUnsharded(result *sqltypes.Result, bindVars map[string]*querypb.BindVariable) string { +func (ins *Insert) getInsertQueryFromSelectForUnsharded(result *sqltypes.Result, bindVars map[string]*querypb.BindVariable) string { var mids sqlparser.Values for r, inputRow := range result.Rows { row := sqlparser.ValTuple{} @@ -350,12 +331,16 @@ func (ins *Insert) getInsertQueryForUnsharded(result *sqltypes.Result, bindVars return ins.Prefix + sqlparser.String(mids) + ins.Suffix } -func (ins *Insert) execInsertSharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { +func (ins *Insert) insertIntoShardedTableFromValues( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, +) (*sqltypes.Result, error) { insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { return nil, err } - rss, queries, err := ins.getInsertShardedRoute(ctx, vcursor, bindVars) + rss, queries, err := ins.getInsertQueriesFromValues(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -363,6 +348,29 @@ func (ins *Insert) execInsertSharded(ctx context.Context, vcursor VCursor, bindV return ins.executeInsertQueries(ctx, vcursor, rss, queries, insertID) } +func (ins *Insert) insertIntoShardedTableFromSelect( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, + rows []sqltypes.Row, +) (int64, *sqltypes.Result, error) { + insertID, err := ins.processGenerateFromSelect(ctx, vcursor, rows) + if err != nil { + return 0, nil, err + } + + rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, rows) + if err != nil { + return 0, nil, err + } + + qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, insertID) + if err != nil { + return 0, nil, err + } + return insertID, qr, nil +} + func (ins *Insert) executeInsertQueries( ctx context.Context, vcursor VCursor, @@ -480,20 +488,15 @@ func (ins *Insert) getInsertSelectQueries( } func (ins *Insert) execInsertFromSelect(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - // run the SELECT query - if ins.Input == nil { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "something went wrong planning INSERT SELECT") - } - - result, err := vcursor.ExecutePrimitive(ctx, ins.Input, bindVars, false) + result, err := ins.InsertRows.execInsertFromSelect(ctx, vcursor, bindVars) if err != nil { return nil, err } - if len(result.Rows) == 0 { + if len(result.rows) == 0 { return &sqltypes.Result{}, nil } - _, qr, err := ins.insertIntoShardedTable(ctx, vcursor, bindVars, result) + _, qr, err := ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result.rows) return qr, err } @@ -575,10 +578,10 @@ func (ins *Insert) processGenerateFromValues( return insertID, nil } -// processGenerateFromRows generates new values using a sequence if necessary. +// processGenerateFromSelect generates new values using a sequence if necessary. // If no value was generated, it returns 0. Values are generated only // for cases where none are supplied. -func (ins *Insert) processGenerateFromRows( +func (ins *Insert) processGenerateFromSelect( ctx context.Context, vcursor VCursor, rows []sqltypes.Row, @@ -639,14 +642,14 @@ func (ins *Insert) processGenerateFromRows( return insertID, nil } -// getInsertShardedRoute performs all the vindex related work +// getInsertQueriesFromValues performs all the vindex related work // and returns a map of shard to queries. // Using the primary vindex, it computes the target keyspace ids. // For owned vindexes, it creates entries. // For unowned vindexes with no input values, it reverse maps. // For unowned vindexes with values, it validates. // If it's an IGNORE or ON DUPLICATE key insert, it drops unroutable rows. -func (ins *Insert) getInsertShardedRoute( +func (ins *Insert) getInsertQueriesFromValues( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, @@ -1023,7 +1026,7 @@ func (ins *Insert) description() PrimitiveDescription { } func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, result *sqltypes.Result) (int64, *sqltypes.Result, error) { - query := ins.getInsertQueryForUnsharded(result, bindVars) + query := ins.getInsertQueryFromSelectForUnsharded(result, bindVars) return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) } diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go new file mode 100644 index 00000000000..78fdb8f7a81 --- /dev/null +++ b/go/vt/vtgate/engine/insert_rows.go @@ -0,0 +1,73 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "context" + + "vitess.io/vitess/go/sqltypes" + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vterrors" +) + +type InsertRows struct { + // Generate is only set for inserts where a sequence must be generated. + Generate *Generate + + RowsFromValues sqlparser.Values + + // Input is a select query plan to retrieve results for inserting data. + RowsFromSelect Primitive +} + +func NewInsertRowsFromSelect(generate *Generate, rowsFromSelect Primitive) *InsertRows { + return &InsertRows{Generate: generate, RowsFromSelect: rowsFromSelect} +} + +type insertRowsResult struct { + rows []sqltypes.Row + insertID uint64 +} + +func (ir *InsertRows) Inputs() ([]Primitive, []map[string]any) { + if ir == nil || ir.RowsFromSelect == nil { + return nil, nil + } + return []Primitive{ir.RowsFromSelect}, nil +} + +func (ir *InsertRows) execInsertFromSelect( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, +) (insertRowsResult, error) { + // run the SELECT query + if ir.RowsFromSelect == nil { + return insertRowsResult{}, vterrors.VT13001("something went wrong planning INSERT SELECT") + } + + res, err := vcursor.ExecutePrimitive(ctx, ir.RowsFromSelect, bindVars, false) + if err != nil { + return insertRowsResult{}, err + } + + return insertRowsResult{ + rows: res.Rows, + insertID: 0, // TODO + }, nil +} diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index d08ef856275..5548308ef98 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1533,17 +1533,21 @@ func TestInsertSelectSimple(t *testing.T) { ks := vs.Keyspaces["sharded"] // A single row insert should be autocommitted + rb := &Route{ + Query: "dummy_select", + FieldQuery: "dummy_field_query", + RoutingParameters: &RoutingParameters{ + Opcode: Scatter, + Keyspace: ks.Keyspace}} ins := &Insert{ Opcode: InsertSelect, Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, - Input: &Route{ - Query: "dummy_select", - FieldQuery: "dummy_field_query", - RoutingParameters: &RoutingParameters{ - Opcode: Scatter, - Keyspace: ks.Keyspace}}} + InsertRows: &InsertRows{ + RowsFromSelect: rb, + }, + Input: rb} ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t1"].ColumnVindexes...) ins.Prefix = "prefix " @@ -1623,6 +1627,12 @@ func TestInsertSelectOwned(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + rb := &Route{ + Query: "dummy_select", + FieldQuery: "dummy_field_query", + RoutingParameters: &RoutingParameters{ + Opcode: Scatter, + Keyspace: ks.Keyspace}} ins := &Insert{ Opcode: InsertSelect, Keyspace: ks.Keyspace, @@ -1630,12 +1640,8 @@ func TestInsertSelectOwned(t *testing.T) { VindexValueOffset: [][]int{ {1}, // The primary vindex has a single column as sharding key {0}}, // the onecol vindex uses the 'name' column - Input: &Route{ - Query: "dummy_select", - FieldQuery: "dummy_field_query", - RoutingParameters: &RoutingParameters{ - Opcode: Scatter, - Keyspace: ks.Keyspace}}} + InsertRows: NewInsertRowsFromSelect(nil, rb), + Input: rb} ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t1"].ColumnVindexes...) ins.Prefix = "prefix " @@ -1734,14 +1740,15 @@ func TestInsertSelectGenerate(t *testing.T) { " suffix") ins.Query = "dummy_insert" ins.VindexValueOffset = [][]int{{1}} // The primary vindex has a single column as sharding key - ins.Input = &Route{ + rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} + ins.Input = rb - ins.Generate = &Generate{ + gen := &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -1749,6 +1756,8 @@ func TestInsertSelectGenerate(t *testing.T) { Query: "dummy_generate", Offset: 1, } + ins.Generate = gen + ins.InsertRows = NewInsertRowsFromSelect(gen, rb) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1817,18 +1826,20 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + rb := &Route{ + Query: "dummy_select", + FieldQuery: "dummy_field_query", + RoutingParameters: &RoutingParameters{ + Opcode: Scatter, + Keyspace: ks.Keyspace}} ins := &Insert{ Opcode: InsertSelect, Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{ {1}}, // The primary vindex has a single column as sharding key - Input: &Route{ - Query: "dummy_select", - FieldQuery: "dummy_field_query", - RoutingParameters: &RoutingParameters{ - Opcode: Scatter, - Keyspace: ks.Keyspace}}} + InsertRows: NewInsertRowsFromSelect(nil, rb), + Input: rb} ins.ColVindexes = ks.Tables["t1"].ColumnVindexes ins.Generate = &Generate{ @@ -1913,18 +1924,20 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + rb := &Route{ + Query: "dummy_select", + FieldQuery: "dummy_field_query", + RoutingParameters: &RoutingParameters{ + Opcode: Scatter, + Keyspace: ks.Keyspace}} ins := &Insert{ - Opcode: InsertSelect, - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{ - {1}}, // The primary vindex has a single column as sharding key - Input: &Route{ - Query: "dummy_select", - FieldQuery: "dummy_field_query", - RoutingParameters: &RoutingParameters{ - Opcode: Scatter, - Keyspace: ks.Keyspace}}} + Opcode: InsertSelect, + Keyspace: ks.Keyspace, + Query: "dummy_insert", + VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key + InsertRows: NewInsertRowsFromSelect(nil, rb), + Input: rb, + } ins.ColVindexes = ks.Tables["t1"].ColumnVindexes ins.Generate = &Generate{ @@ -2001,18 +2014,19 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + rb := &Route{ + Query: "dummy_select", + FieldQuery: "dummy_field_query", + RoutingParameters: &RoutingParameters{ + Opcode: Scatter, + Keyspace: ks.Keyspace}} ins := &Insert{ - Opcode: InsertSelect, - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{ - {1}}, // The primary vindex has a single column as sharding key - Input: &Route{ - Query: "dummy_select", - FieldQuery: "dummy_field_query", - RoutingParameters: &RoutingParameters{ - Opcode: Scatter, - Keyspace: ks.Keyspace}}} + Opcode: InsertSelect, + Keyspace: ks.Keyspace, + Query: "dummy_insert", + VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key + InsertRows: NewInsertRowsFromSelect(nil, rb), + Input: rb} ins.ColVindexes = ks.Tables["t1"].ColumnVindexes ins.Generate = &Generate{ @@ -2099,18 +2113,19 @@ func TestInsertSelectUnowned(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + rb := &Route{ + Query: "dummy_select", + FieldQuery: "dummy_field_query", + RoutingParameters: &RoutingParameters{ + Opcode: Scatter, + Keyspace: ks.Keyspace}} ins := &Insert{ - Opcode: InsertSelect, - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{ - {0}}, // the onecol vindex as unowned lookup sharding column - Input: &Route{ - Query: "dummy_select", - FieldQuery: "dummy_field_query", - RoutingParameters: &RoutingParameters{ - Opcode: Scatter, - Keyspace: ks.Keyspace}}} + Opcode: InsertSelect, + Keyspace: ks.Keyspace, + Query: "dummy_insert", + VindexValueOffset: [][]int{{0}}, // the onecol vindex as unowned lookup sharding column + InsertRows: NewInsertRowsFromSelect(nil, rb), + Input: rb} ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t2"].ColumnVindexes...) ins.Prefix = "prefix " @@ -2228,6 +2243,7 @@ func TestInsertSelectShardingCases(t *testing.T) { Suffix: " suffix", ColVindexes: sks1.Tables["s1"].ColumnVindexes, VindexValueOffset: [][]int{{0}}, + InsertRows: NewInsertRowsFromSelect(nil, sRoute), Input: sRoute, } @@ -2270,6 +2286,7 @@ func TestInsertSelectShardingCases(t *testing.T) { // sks1 and uks2 ins.Input = uRoute + ins.InsertRows = NewInsertRowsFromSelect(nil, uRoute) vc.Rewind() _, err = ins.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) @@ -2299,12 +2316,13 @@ func TestInsertSelectShardingCases(t *testing.T) { // uks1 and sks2 ins = &Insert{ - Opcode: InsertUnsharded, - Keyspace: uks1.Keyspace, - Query: "dummy_insert", - Prefix: "prefix ", - Suffix: " suffix", - Input: sRoute, + Opcode: InsertUnsharded, + Keyspace: uks1.Keyspace, + Query: "dummy_insert", + Prefix: "prefix ", + Suffix: " suffix", + Input: sRoute, + InsertRows: NewInsertRowsFromSelect(nil, sRoute), } vc.Rewind() @@ -2335,6 +2353,7 @@ func TestInsertSelectShardingCases(t *testing.T) { // uks1 and uks2 ins.Input = uRoute + ins.InsertRows = NewInsertRowsFromSelect(nil, uRoute) vc.Rewind() _, err = ins.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 39144fc858d..77689e651ed 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -22,7 +22,6 @@ import ( "vitess.io/vitess/go/vt/vtgate/engine" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - "vitess.io/vitess/go/vt/vtgate/semantics" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -108,11 +107,9 @@ var _ logicalPlan = (*insert)(nil) func (i *insert) Primitive() engine.Primitive { if i.source != nil { - i.eInsert.Input = i.source.Primitive() + input := i.source.Primitive() + i.eInsert.Input = input + i.eInsert.InsertRows = engine.NewInsertRowsFromSelect(nil, input) } return i.eInsert } - -func (i *insert) ContainsTables() semantics.TableSet { - panic("does not expect insert to get contains tables call") -} From ca02aff0a4daf94eae395f164fea02e9f4f268d8 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 24 Nov 2023 12:10:01 +0100 Subject: [PATCH 02/15] refactor: move all uses of the Input field to InsertRows Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/cached_size.go | 34 +++++++++++++++++++++++++++--- go/vt/vtgate/engine/insert.go | 24 +++++++++------------ go/vt/vtgate/engine/insert_rows.go | 15 ++++++++++++- go/vt/vtgate/engine/insert_test.go | 20 +++++------------- go/vt/vtgate/planbuilder/insert.go | 1 - 5 files changed, 60 insertions(+), 34 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 657e5323f05..39ddabece3d 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -386,12 +386,14 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(240) + size += int64(224) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) // field Query string size += hack.RuntimeAllocSize(int64(len(cached.Query))) + // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows + size += cached.InsertRows.CachedSize(true) // field VindexValues [][][]vitess.io/vitess/go/vt/vtgate/evalengine.Expr { size += hack.RuntimeAllocSize(int64(cap(cached.VindexValues)) * int64(24)) @@ -449,8 +451,34 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } } } - // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive - if cc, ok := cached.Input.(cachedObject); ok { + return size +} +func (cached *InsertRows) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(48) + } + // field Generate *vitess.io/vitess/go/vt/vtgate/engine.Generate + size += cached.Generate.CachedSize(true) + // field RowsFromValues vitess.io/vitess/go/vt/sqlparser.Values + { + size += hack.RuntimeAllocSize(int64(cap(cached.RowsFromValues)) * int64(24)) + for _, elem := range cached.RowsFromValues { + { + size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(16)) + for _, elem := range elem { + if cc, ok := elem.(cachedObject); ok { + size += cc.CachedSize(true) + } + } + } + } + } + // field RowsFromSelect vitess.io/vitess/go/vt/vtgate/engine.Primitive + if cc, ok := cached.RowsFromSelect.(cachedObject); ok { size += cc.CachedSize(true) } return size diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 8455836d717..1503273598a 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -97,9 +97,6 @@ type ( // that will appear in the result set of the select query. VindexValueOffset [][]int - // Input is a select query plan to retrieve results for inserting data. - Input Primitive `json:",omitempty"` - // ForceNonStreaming is true when the insert table and select table are same. // This will avoid locking by the select table. ForceNonStreaming bool @@ -242,7 +239,7 @@ func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map // TryStreamExecute performs a streaming exec. func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - if ins.Input == nil || ins.ForceNonStreaming { + if !ins.InsertRows.hasSelectInput() || ins.ForceNonStreaming { res, err := ins.TryExecute(ctx, vcursor, bindVars, wantfields) if err != nil { return err @@ -258,8 +255,7 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa unsharded := ins.Opcode == InsertUnsharded var mu sync.Mutex output := &sqltypes.Result{} - - err := vcursor.StreamExecutePrimitiveStandalone(ctx, ins.Input, bindVars, false, func(result *sqltypes.Result) error { + err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(result *sqltypes.Result) error { if len(result.Rows) == 0 { return nil } @@ -302,24 +298,24 @@ func (ins *Insert) GetFields(context.Context, VCursor, map[string]*querypb.BindV func (ins *Insert) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { query := ins.Query - if ins.Input != nil { - result, err := vcursor.ExecutePrimitive(ctx, ins.Input, bindVars, false) + if ins.InsertRows.hasSelectInput() { + result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) if err != nil { return nil, err } - if len(result.Rows) == 0 { + if len(result.rows) == 0 { return &sqltypes.Result{}, nil } - query = ins.getInsertQueryFromSelectForUnsharded(result, bindVars) + query = ins.getInsertQueryFromSelectForUnsharded(result.rows, bindVars) } _, qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) return qr, err } -func (ins *Insert) getInsertQueryFromSelectForUnsharded(result *sqltypes.Result, bindVars map[string]*querypb.BindVariable) string { +func (ins *Insert) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { var mids sqlparser.Values - for r, inputRow := range result.Rows { + for r, inputRow := range rows { row := sqlparser.ValTuple{} for c, value := range inputRow { bvName := insertVarOffset(r, c) @@ -488,7 +484,7 @@ func (ins *Insert) getInsertSelectQueries( } func (ins *Insert) execInsertFromSelect(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - result, err := ins.InsertRows.execInsertFromSelect(ctx, vcursor, bindVars) + result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -1026,7 +1022,7 @@ func (ins *Insert) description() PrimitiveDescription { } func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, result *sqltypes.Result) (int64, *sqltypes.Result, error) { - query := ins.getInsertQueryFromSelectForUnsharded(result, bindVars) + query := ins.getInsertQueryFromSelectForUnsharded(result.Rows, bindVars) return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) } diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go index 78fdb8f7a81..31329a331ac 100644 --- a/go/vt/vtgate/engine/insert_rows.go +++ b/go/vt/vtgate/engine/insert_rows.go @@ -51,7 +51,11 @@ func (ir *InsertRows) Inputs() ([]Primitive, []map[string]any) { return []Primitive{ir.RowsFromSelect}, nil } -func (ir *InsertRows) execInsertFromSelect( +func (ir *InsertRows) hasSelectInput() bool { + return ir != nil && ir.RowsFromSelect != nil +} + +func (ir *InsertRows) execSelect( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, @@ -71,3 +75,12 @@ func (ir *InsertRows) execInsertFromSelect( insertID: 0, // TODO }, nil } + +func (ir *InsertRows) execSelectStreaming( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, + callback func(result *sqltypes.Result) error, +) error { + return vcursor.StreamExecutePrimitiveStandalone(ctx, ir.RowsFromSelect, bindVars, false, callback) +} diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 5548308ef98..6b6d0e49124 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1546,8 +1546,7 @@ func TestInsertSelectSimple(t *testing.T) { VindexValueOffset: [][]int{{1}}, InsertRows: &InsertRows{ RowsFromSelect: rb, - }, - Input: rb} + }} ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t1"].ColumnVindexes...) ins.Prefix = "prefix " @@ -1641,7 +1640,7 @@ func TestInsertSelectOwned(t *testing.T) { {1}, // The primary vindex has a single column as sharding key {0}}, // the onecol vindex uses the 'name' column InsertRows: NewInsertRowsFromSelect(nil, rb), - Input: rb} + } ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t1"].ColumnVindexes...) ins.Prefix = "prefix " @@ -1746,7 +1745,6 @@ func TestInsertSelectGenerate(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins.Input = rb gen := &Generate{ Keyspace: &vindexes.Keyspace{ @@ -1838,8 +1836,7 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { Query: "dummy_insert", VindexValueOffset: [][]int{ {1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(nil, rb), - Input: rb} + InsertRows: NewInsertRowsFromSelect(nil, rb)} ins.ColVindexes = ks.Tables["t1"].ColumnVindexes ins.Generate = &Generate{ @@ -1936,7 +1933,6 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key InsertRows: NewInsertRowsFromSelect(nil, rb), - Input: rb, } ins.ColVindexes = ks.Tables["t1"].ColumnVindexes @@ -2025,8 +2021,7 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(nil, rb), - Input: rb} + InsertRows: NewInsertRowsFromSelect(nil, rb)} ins.ColVindexes = ks.Tables["t1"].ColumnVindexes ins.Generate = &Generate{ @@ -2124,8 +2119,7 @@ func TestInsertSelectUnowned(t *testing.T) { Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{0}}, // the onecol vindex as unowned lookup sharding column - InsertRows: NewInsertRowsFromSelect(nil, rb), - Input: rb} + InsertRows: NewInsertRowsFromSelect(nil, rb)} ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t2"].ColumnVindexes...) ins.Prefix = "prefix " @@ -2244,7 +2238,6 @@ func TestInsertSelectShardingCases(t *testing.T) { ColVindexes: sks1.Tables["s1"].ColumnVindexes, VindexValueOffset: [][]int{{0}}, InsertRows: NewInsertRowsFromSelect(nil, sRoute), - Input: sRoute, } vc := &loggingVCursor{ @@ -2285,7 +2278,6 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard sks1.-20: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // sks1 and uks2 - ins.Input = uRoute ins.InsertRows = NewInsertRowsFromSelect(nil, uRoute) vc.Rewind() @@ -2321,7 +2313,6 @@ func TestInsertSelectShardingCases(t *testing.T) { Query: "dummy_insert", Prefix: "prefix ", Suffix: " suffix", - Input: sRoute, InsertRows: NewInsertRowsFromSelect(nil, sRoute), } @@ -2352,7 +2343,6 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard uks1.0: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // uks1 and uks2 - ins.Input = uRoute ins.InsertRows = NewInsertRowsFromSelect(nil, uRoute) vc.Rewind() diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 77689e651ed..bd04f15b53a 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -108,7 +108,6 @@ var _ logicalPlan = (*insert)(nil) func (i *insert) Primitive() engine.Primitive { if i.source != nil { input := i.source.Primitive() - i.eInsert.Input = input i.eInsert.InsertRows = engine.NewInsertRowsFromSelect(nil, input) } return i.eInsert From 3bf6f59a4dabdd74403324602e24f43ad2cf1509 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 27 Nov 2023 12:12:52 +0530 Subject: [PATCH 03/15] refactor: moved sequence Generate to InsertRows Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert.go | 91 ++++-------------------------- go/vt/vtgate/engine/insert_rows.go | 89 +++++++++++++++++++++++++++-- go/vt/vtgate/engine/insert_test.go | 55 +++++++++--------- go/vt/vtgate/planbuilder/insert.go | 2 +- 4 files changed, 124 insertions(+), 113 deletions(-) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 1503273598a..18e46d74bf9 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -255,8 +255,8 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa unsharded := ins.Opcode == InsertUnsharded var mu sync.Mutex output := &sqltypes.Result{} - err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(result *sqltypes.Result) error { - if len(result.Rows) == 0 { + err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(irr insertRowsResult) error { + if len(irr.rows) == 0 { return nil } @@ -270,9 +270,9 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa var qr *sqltypes.Result var err error if unsharded { - insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, result) + insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr.rows) } else { - insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result.Rows) + insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, irr) } if err != nil { return err @@ -348,23 +348,18 @@ func (ins *Insert) insertIntoShardedTableFromSelect( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, - rows []sqltypes.Row, + irr insertRowsResult, ) (int64, *sqltypes.Result, error) { - insertID, err := ins.processGenerateFromSelect(ctx, vcursor, rows) - if err != nil { - return 0, nil, err - } - - rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, rows) + rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, irr.rows) if err != nil { return 0, nil, err } - qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, insertID) + qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, irr.insertID) if err != nil { return 0, nil, err } - return insertID, qr, nil + return irr.insertID, qr, nil } func (ins *Insert) executeInsertQueries( @@ -492,7 +487,7 @@ func (ins *Insert) execInsertFromSelect(ctx context.Context, vcursor VCursor, bi return &sqltypes.Result{}, nil } - _, qr, err := ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result.rows) + _, qr, err := ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result) return qr, err } @@ -574,70 +569,6 @@ func (ins *Insert) processGenerateFromValues( return insertID, nil } -// processGenerateFromSelect generates new values using a sequence if necessary. -// If no value was generated, it returns 0. Values are generated only -// for cases where none are supplied. -func (ins *Insert) processGenerateFromSelect( - ctx context.Context, - vcursor VCursor, - rows []sqltypes.Row, -) (insertID int64, err error) { - if ins.Generate == nil { - return 0, nil - } - var count int64 - offset := ins.Generate.Offset - genColPresent := offset < len(rows[0]) - if genColPresent { - for _, val := range rows { - if val[offset].IsNull() { - count++ - } - } - } else { - count = int64(len(rows)) - } - - if count == 0 { - return 0, nil - } - - // If generation is needed, generate the requested number of values (as one call). - rss, _, err := vcursor.ResolveDestinations(ctx, ins.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) - if err != nil { - return 0, err - } - if len(rss) != 1 { - return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) - } - bindVars := map[string]*querypb.BindVariable{"n": sqltypes.Int64BindVariable(count)} - qr, err := vcursor.ExecuteStandalone(ctx, ins, ins.Generate.Query, bindVars, rss[0]) - if err != nil { - return 0, err - } - // If no rows are returned, it's an internal error, and the code - // must panic, which will be caught and reported. - insertID, err = qr.Rows[0][0].ToCastInt64() - if err != nil { - return 0, err - } - - used := insertID - for idx, val := range rows { - if genColPresent { - if val[offset].IsNull() { - val[offset] = sqltypes.NewInt64(used) - used++ - } - } else { - rows[idx] = append(val, sqltypes.NewInt64(used)) - used++ - } - } - - return insertID, nil -} - // getInsertQueriesFromValues performs all the vindex related work // and returns a map of shard to queries. // Using the primary vindex, it computes the target keyspace ids. @@ -1021,8 +952,8 @@ func (ins *Insert) description() PrimitiveDescription { } } -func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, result *sqltypes.Result) (int64, *sqltypes.Result, error) { - query := ins.getInsertQueryFromSelectForUnsharded(result.Rows, bindVars) +func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, rows []sqltypes.Row) (int64, *sqltypes.Result, error) { + query := ins.getInsertQueryFromSelectForUnsharded(rows, bindVars) return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) } diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go index 31329a331ac..9c8ccd3a40a 100644 --- a/go/vt/vtgate/engine/insert_rows.go +++ b/go/vt/vtgate/engine/insert_rows.go @@ -20,7 +20,9 @@ import ( "context" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" ) @@ -41,7 +43,7 @@ func NewInsertRowsFromSelect(generate *Generate, rowsFromSelect Primitive) *Inse type insertRowsResult struct { rows []sqltypes.Row - insertID uint64 + insertID int64 } func (ir *InsertRows) Inputs() ([]Primitive, []map[string]any) { @@ -66,13 +68,18 @@ func (ir *InsertRows) execSelect( } res, err := vcursor.ExecutePrimitive(ctx, ir.RowsFromSelect, bindVars, false) + if err != nil || len(res.Rows) == 0 { + return insertRowsResult{}, err + } + + insertID, err := ir.processGenerateFromSelect(ctx, vcursor, res.Rows) if err != nil { return insertRowsResult{}, err } return insertRowsResult{ rows: res.Rows, - insertID: 0, // TODO + insertID: insertID, }, nil } @@ -80,7 +87,81 @@ func (ir *InsertRows) execSelectStreaming( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, - callback func(result *sqltypes.Result) error, + callback func(irr insertRowsResult) error, ) error { - return vcursor.StreamExecutePrimitiveStandalone(ctx, ir.RowsFromSelect, bindVars, false, callback) + return vcursor.StreamExecutePrimitiveStandalone(ctx, ir.RowsFromSelect, bindVars, false, func(result *sqltypes.Result) error { + insertID, err := ir.processGenerateFromSelect(ctx, vcursor, result.Rows) + if err != nil { + return err + } + + return callback(insertRowsResult{ + rows: result.Rows, + insertID: insertID, + }) + }) +} + +// processGenerateFromSelect generates new values using a sequence if necessary. +// If no value was generated, it returns 0. Values are generated only +// for cases where none are supplied. +func (ir *InsertRows) processGenerateFromSelect( + ctx context.Context, + vcursor VCursor, + rows []sqltypes.Row, +) (insertID int64, err error) { + if ir.Generate == nil { + return 0, nil + } + var count int64 + offset := ir.Generate.Offset + genColPresent := offset < len(rows[0]) + if genColPresent { + for _, val := range rows { + if val[offset].IsNull() { + count++ + } + } + } else { + count = int64(len(rows)) + } + + if count == 0 { + return 0, nil + } + + // If generation is needed, generate the requested number of values (as one call). + rss, _, err := vcursor.ResolveDestinations(ctx, ir.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) + if err != nil { + return 0, err + } + if len(rss) != 1 { + return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) + } + bindVars := map[string]*querypb.BindVariable{"n": sqltypes.Int64BindVariable(count)} + qr, err := vcursor.ExecuteStandalone(ctx, nil, ir.Generate.Query, bindVars, rss[0]) + if err != nil { + return 0, err + } + // If no rows are returned, it's an internal error, and the code + // must panic, which will be caught and reported. + insertID, err = qr.Rows[0][0].ToCastInt64() + if err != nil { + return 0, err + } + + used := insertID + for idx, val := range rows { + if genColPresent { + if val[offset].IsNull() { + val[offset] = sqltypes.NewInt64(used) + used++ + } + } else { + rows[idx] = append(val, sqltypes.NewInt64(used)) + used++ + } + } + + return insertID, nil } diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 6b6d0e49124..22c76cbe15e 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1824,6 +1824,14 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + generate := &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 1, + } rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -1836,17 +1844,8 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { Query: "dummy_insert", VindexValueOffset: [][]int{ {1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(nil, rb)} + InsertRows: NewInsertRowsFromSelect(generate, rb)} ins.ColVindexes = ks.Tables["t1"].ColumnVindexes - - ins.Generate = &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 1, - } ins.Prefix = "prefix " ins.Suffix = " suffix" @@ -1921,6 +1920,14 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + generate := &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 2, + } rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -1932,18 +1939,10 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(nil, rb), + InsertRows: NewInsertRowsFromSelect(generate, rb), } ins.ColVindexes = ks.Tables["t1"].ColumnVindexes - ins.Generate = &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 2, - } ins.Prefix = "prefix " ins.Suffix = " suffix" @@ -2010,6 +2009,14 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] + generate := &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 2, + } rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -2021,17 +2028,9 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(nil, rb)} + InsertRows: NewInsertRowsFromSelect(generate, rb)} ins.ColVindexes = ks.Tables["t1"].ColumnVindexes - ins.Generate = &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 2, - } ins.Prefix = "prefix " ins.Suffix = " suffix" diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index bd04f15b53a..c57b6259c1a 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -108,7 +108,7 @@ var _ logicalPlan = (*insert)(nil) func (i *insert) Primitive() engine.Primitive { if i.source != nil { input := i.source.Primitive() - i.eInsert.InsertRows = engine.NewInsertRowsFromSelect(nil, input) + i.eInsert.InsertRows = engine.NewInsertRowsFromSelect(i.eInsert.Generate, input) } return i.eInsert } From 9c2d2f3a894e98b44bf2e59a18f09b810a457c53 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 24 Nov 2023 13:32:53 +0100 Subject: [PATCH 04/15] refactor: reuse existing helper methods Signed-off-by: Andres Taylor Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert.go | 15 ++++++++++----- go/vt/vtgate/engine/insert_rows.go | 8 +++++--- go/vt/vtgate/engine/insert_test.go | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 18e46d74bf9..beeccc18ab0 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -86,7 +86,7 @@ type ( // // This is a clear violation of the SQL semantics since it means the statement // is not atomic in the presence of PK conflicts on one shard and not another. - // However some application use cases would prefer that the statement partially + // However, some application use cases would prefer that the statement partially // succeed in order to get the performance benefits of autocommit. MultiShardAutocommit bool @@ -499,12 +499,17 @@ func shouldGenerate(v sqltypes.Value) bool { // Unless the NO_AUTO_VALUE_ON_ZERO sql mode is active in mysql, it also // treats 0 as a value that should generate a new sequence. - n, err := v.ToCastUint64() - if err == nil && n == 0 { - return true + value, err := evalengine.CoerceTo(v, sqltypes.Uint64) + if err != nil { + return false + } + + id, err := value.ToCastUint64() + if err != nil { + return false } - return false + return id == 0 } // processGenerateFromValues generates new values using a sequence if necessary. diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go index 9c8ccd3a40a..7b0f75e08a6 100644 --- a/go/vt/vtgate/engine/insert_rows.go +++ b/go/vt/vtgate/engine/insert_rows.go @@ -27,6 +27,8 @@ import ( "vitess.io/vitess/go/vt/vterrors" ) +// insert into t values (select func(), + type InsertRows struct { // Generate is only set for inserts where a sequence must be generated. Generate *Generate @@ -117,8 +119,8 @@ func (ir *InsertRows) processGenerateFromSelect( offset := ir.Generate.Offset genColPresent := offset < len(rows[0]) if genColPresent { - for _, val := range rows { - if val[offset].IsNull() { + for _, row := range rows { + if shouldGenerate(row[offset]) { count++ } } @@ -153,7 +155,7 @@ func (ir *InsertRows) processGenerateFromSelect( used := insertID for idx, val := range rows { if genColPresent { - if val[offset].IsNull() { + if shouldGenerate(val[offset]) { val[offset] = sqltypes.NewInt64(used) used++ } diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 22c76cbe15e..53edaab5f24 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1767,7 +1767,7 @@ func TestInsertSelectGenerate(t *testing.T) { "varchar|int64"), "a|1", "a|null", - "b|null"), + "b|0"), // This is the result for the sequence query sqltypes.MakeTestResult( sqltypes.MakeTestFields( From fd2f7f81f07df7dc4107cee279ef4144029faa09 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 27 Nov 2023 16:47:12 +0530 Subject: [PATCH 05/15] refactor: move Generate to InsertRows for sqlparer.Values Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/cached_size.go | 2 - go/vt/vtgate/engine/insert.go | 106 ++--------------- go/vt/vtgate/engine/insert_rows.go | 112 +++++++++++++++--- go/vt/vtgate/engine/insert_test.go | 7 +- go/vt/vtgate/planbuilder/insert.go | 3 +- .../planbuilder/operator_transformers.go | 4 +- 6 files changed, 114 insertions(+), 120 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 39ddabece3d..d03b0f429b4 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -422,8 +422,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } // field TableName string size += hack.RuntimeAllocSize(int64(len(cached.TableName))) - // field Generate *vitess.io/vitess/go/vt/vtgate/engine.Generate - size += cached.Generate.CachedSize(true) // field Prefix string size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) // field Mid vitess.io/vitess/go/vt/sqlparser.Values diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index beeccc18ab0..af04513f04d 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -73,9 +73,6 @@ type ( // TableName is the name of the table on which row will be inserted. TableName string - // Generate is only set for inserts where a sequence must be generated. - Generate *Generate - // Prefix, Mid and Suffix are for sharded insert plans. Prefix string Mid sqlparser.Values @@ -117,9 +114,10 @@ func (ins *Insert) Inputs() ([]Primitive, []map[string]any) { // NewQueryInsert creates an Insert with a query string. func NewQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { return &Insert{ - Opcode: opcode, - Keyspace: keyspace, - Query: query, + Opcode: opcode, + Keyspace: keyspace, + Query: query, + InsertRows: NewInsertRows(nil), } } @@ -138,6 +136,7 @@ func NewInsert( Opcode: opcode, Ignore: ignore, Keyspace: keyspace, + InsertRows: NewInsertRows(nil), VindexValues: vindexValues, Prefix: prefix, Mid: mid, @@ -332,7 +331,7 @@ func (ins *Insert) insertIntoShardedTableFromValues( vcursor VCursor, bindVars map[string]*querypb.BindVariable, ) (*sqltypes.Result, error) { - insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) + insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -491,89 +490,6 @@ func (ins *Insert) execInsertFromSelect(ctx context.Context, vcursor VCursor, bi return qr, err } -// shouldGenerate determines if a sequence value should be generated for a given value -func shouldGenerate(v sqltypes.Value) bool { - if v.IsNull() { - return true - } - - // Unless the NO_AUTO_VALUE_ON_ZERO sql mode is active in mysql, it also - // treats 0 as a value that should generate a new sequence. - value, err := evalengine.CoerceTo(v, sqltypes.Uint64) - if err != nil { - return false - } - - id, err := value.ToCastUint64() - if err != nil { - return false - } - - return id == 0 -} - -// processGenerateFromValues generates new values using a sequence if necessary. -// If no value was generated, it returns 0. Values are generated only -// for cases where none are supplied. -func (ins *Insert) processGenerateFromValues( - ctx context.Context, - vcursor VCursor, - bindVars map[string]*querypb.BindVariable, -) (insertID int64, err error) { - if ins.Generate == nil { - return 0, nil - } - - // Scan input values to compute the number of values to generate, and - // keep track of where they should be filled. - env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) - resolved, err := env.Evaluate(ins.Generate.Values) - if err != nil { - return 0, err - } - count := int64(0) - values := resolved.TupleValues() - for _, val := range values { - if shouldGenerate(val) { - count++ - } - } - - // If generation is needed, generate the requested number of values (as one call). - if count != 0 { - rss, _, err := vcursor.ResolveDestinations(ctx, ins.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) - if err != nil { - return 0, err - } - if len(rss) != 1 { - return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) - } - bindVars := map[string]*querypb.BindVariable{"n": sqltypes.Int64BindVariable(count)} - qr, err := vcursor.ExecuteStandalone(ctx, ins, ins.Generate.Query, bindVars, rss[0]) - if err != nil { - return 0, err - } - // If no rows are returned, it's an internal error, and the code - // must panic, which will be caught and reported. - insertID, err = qr.Rows[0][0].ToCastInt64() - if err != nil { - return 0, err - } - } - - // Fill the holes where no value was supplied. - cur := insertID - for i, v := range values { - if shouldGenerate(v) { - bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.Int64BindVariable(cur) - cur++ - } else { - bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.ValueBindVariable(v) - } - } - return insertID, nil -} - // getInsertQueriesFromValues performs all the vindex related work // and returns a map of shard to queries. // Using the primary vindex, it computes the target keyspace ids. @@ -919,11 +835,11 @@ func (ins *Insert) description() PrimitiveDescription { other["VindexValues"] = valuesOffsets } - if ins.Generate != nil { - if ins.Generate.Values == nil { - other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ins.Generate.Query, ins.Generate.Offset) + if ins.InsertRows.Generate != nil { + if ins.InsertRows.Generate.Values == nil { + other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ins.InsertRows.Generate.Query, ins.InsertRows.Generate.Offset) } else { - other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.Generate.Query, sqlparser.String(ins.Generate.Values)) + other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.InsertRows.Generate.Query, sqlparser.String(ins.InsertRows.Generate.Values)) } } @@ -963,7 +879,7 @@ func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor } func (ins *Insert) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (int64, *sqltypes.Result, error) { - insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) + insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { return 0, nil, err } diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go index 7b0f75e08a6..1bd92167397 100644 --- a/go/vt/vtgate/engine/insert_rows.go +++ b/go/vt/vtgate/engine/insert_rows.go @@ -18,6 +18,9 @@ package engine import ( "context" + "strconv" + + "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" @@ -27,7 +30,7 @@ import ( "vitess.io/vitess/go/vt/vterrors" ) -// insert into t values (select func(), +const nextValBV = "n" type InsertRows struct { // Generate is only set for inserts where a sequence must be generated. @@ -43,6 +46,10 @@ func NewInsertRowsFromSelect(generate *Generate, rowsFromSelect Primitive) *Inse return &InsertRows{Generate: generate, RowsFromSelect: rowsFromSelect} } +func NewInsertRows(generate *Generate) *InsertRows { + return &InsertRows{Generate: generate} +} + type insertRowsResult struct { rows []sqltypes.Row insertID int64 @@ -132,6 +139,76 @@ func (ir *InsertRows) processGenerateFromSelect( return 0, nil } + insertID, err = ir.execGenerate(ctx, vcursor, count) + if err != nil { + return 0, err + } + + used := insertID + for idx, val := range rows { + if genColPresent { + if shouldGenerate(val[offset]) { + val[offset] = sqltypes.NewInt64(used) + used++ + } + } else { + rows[idx] = append(val, sqltypes.NewInt64(used)) + used++ + } + } + + return insertID, nil +} + +// processGenerateFromValues generates new values using a sequence if necessary. +// If no value was generated, it returns 0. Values are generated only +// for cases where none are supplied. +func (ir *InsertRows) processGenerateFromValues( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, +) (insertID int64, err error) { + if ir.Generate == nil { + return 0, nil + } + + // Scan input values to compute the number of values to generate, and + // keep track of where they should be filled. + env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) + resolved, err := env.Evaluate(ir.Generate.Values) + if err != nil { + return 0, err + } + count := int64(0) + values := resolved.TupleValues() + for _, val := range values { + if shouldGenerate(val) { + count++ + } + } + + // If generation is needed, generate the requested number of values (as one call). + if count != 0 { + insertID, err = ir.execGenerate(ctx, vcursor, count) + if err != nil { + return 0, err + } + } + + // Fill the holes where no value was supplied. + cur := insertID + for i, v := range values { + if shouldGenerate(v) { + bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.Int64BindVariable(cur) + cur++ + } else { + bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.ValueBindVariable(v) + } + } + return insertID, nil +} + +func (ir *InsertRows) execGenerate(ctx context.Context, vcursor VCursor, count int64) (int64, error) { // If generation is needed, generate the requested number of values (as one call). rss, _, err := vcursor.ResolveDestinations(ctx, ir.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) if err != nil { @@ -140,30 +217,33 @@ func (ir *InsertRows) processGenerateFromSelect( if len(rss) != 1 { return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) } - bindVars := map[string]*querypb.BindVariable{"n": sqltypes.Int64BindVariable(count)} + bindVars := map[string]*querypb.BindVariable{nextValBV: sqltypes.Int64BindVariable(count)} qr, err := vcursor.ExecuteStandalone(ctx, nil, ir.Generate.Query, bindVars, rss[0]) if err != nil { return 0, err } // If no rows are returned, it's an internal error, and the code // must panic, which will be caught and reported. - insertID, err = qr.Rows[0][0].ToCastInt64() + return qr.Rows[0][0].ToCastInt64() +} + +// shouldGenerate determines if a sequence value should be generated for a given value +func shouldGenerate(v sqltypes.Value) bool { + if v.IsNull() { + return true + } + + // Unless the NO_AUTO_VALUE_ON_ZERO sql mode is active in mysql, it also + // treats 0 as a value that should generate a new sequence. + value, err := evalengine.CoerceTo(v, sqltypes.Uint64) if err != nil { - return 0, err + return false } - used := insertID - for idx, val := range rows { - if genColPresent { - if shouldGenerate(val[offset]) { - val[offset] = sqltypes.NewInt64(used) - used++ - } - } else { - rows[idx] = append(val, sqltypes.NewInt64(used)) - used++ - } + id, err := value.ToCastUint64() + if err != nil { + return false } - return insertID, nil + return id == 0 } diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 53edaab5f24..fc34e1131c6 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -76,7 +76,7 @@ func TestInsertUnshardedGenerate(t *testing.T) { }, "dummy_insert", ) - ins.Generate = &Generate{ + ins.InsertRows.Generate = &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -129,7 +129,7 @@ func TestInsertUnshardedGenerate_Zeros(t *testing.T) { }, "dummy_insert", ) - ins.Generate = &Generate{ + ins.InsertRows.Generate = &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -417,7 +417,7 @@ func TestInsertShardedGenerate(t *testing.T) { " suffix", ) - ins.Generate = &Generate{ + ins.InsertRows.Generate = &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -1754,7 +1754,6 @@ func TestInsertSelectGenerate(t *testing.T) { Query: "dummy_generate", Offset: 1, } - ins.Generate = gen ins.InsertRows = NewInsertRowsFromSelect(gen, rb) vc := newDMLTestVCursor("-20", "20-") diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index c57b6259c1a..d6e30d79609 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -95,6 +95,7 @@ func insertUnshardedShortcut(stmt *sqlparser.Insert, ks *vindexes.Keyspace, tabl eIns.TableName = tables[0].Name.String() eIns.Opcode = engine.InsertUnsharded eIns.Query = generateQuery(stmt) + eIns.InsertRows = engine.NewInsertRows(nil) return &insert{eInsert: eIns} } @@ -108,7 +109,7 @@ var _ logicalPlan = (*insert)(nil) func (i *insert) Primitive() engine.Primitive { if i.source != nil { input := i.source.Primitive() - i.eInsert.InsertRows = engine.NewInsertRowsFromSelect(i.eInsert.Generate, input) + i.eInsert.InsertRows.RowsFromSelect = input } return i.eInsert } diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index d6732ada87d..9f81792b651 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -114,9 +114,9 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators Opcode: mapToInsertOpCode(rb.Routing.OpCode(), true), Keyspace: rb.Routing.Keyspace(), TableName: ins.VTable.Name.String(), + InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), Ignore: ins.Ignore, ForceNonStreaming: op.ForceNonStreaming, - Generate: autoIncGenerate(ins.AutoIncrement), ColVindexes: ins.ColVindexes, VindexValues: ins.VindexValues, VindexValueOffset: ins.VindexValueOffset, @@ -553,7 +553,7 @@ func buildInsertLogicalPlan( Keyspace: rb.Routing.Keyspace(), TableName: ins.VTable.Name.String(), Ignore: ins.Ignore, - Generate: autoIncGenerate(ins.AutoIncrement), + InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), ColVindexes: ins.ColVindexes, VindexValues: ins.VindexValues, VindexValueOffset: ins.VindexValueOffset, From db616374bd04c090f409e04cb331820af0caef96 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 27 Nov 2023 22:33:25 +0530 Subject: [PATCH 06/15] refactor: split insert engine to insert and insertSelect Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert.go | 24 +- go/vt/vtgate/engine/insert_select.go | 639 ++++++++++++++++++ go/vt/vtgate/engine/insert_test.go | 30 +- go/vt/vtgate/planbuilder/insert.go | 14 +- .../planbuilder/operator_transformers.go | 9 +- .../planbuilder/testdata/dml_cases.json | 4 +- 6 files changed, 672 insertions(+), 48 deletions(-) create mode 100644 go/vt/vtgate/engine/insert_select.go diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index af04513f04d..0de08602458 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -46,12 +46,15 @@ type ( // Opcode is the execution opcode. Opcode InsertOpcode + // Keyspace specifies the keyspace to send the query to. + Keyspace *vindexes.Keyspace + // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs // for sharded cases. Ignore bool - // Keyspace specifies the keyspace to send the query to. - Keyspace *vindexes.Keyspace + // TableName is the name of the table on which row will be inserted. + TableName string // Query specifies the query to be executed. // For InsertSharded plans, this value is unused, @@ -67,12 +70,13 @@ type ( // Insert.Values[i].Values[j].Values[k] represents the value pulled from row k for that column: (k < len(ins.rows)) VindexValues [][][]evalengine.Expr + // VindexValueOffset stores the offset for each column in the ColumnVindex + // that will appear in the result set of the select query. + VindexValueOffset [][]int + // ColVindexes are the vindexes that will use the VindexValues ColVindexes []*vindexes.ColumnVindex - // TableName is the name of the table on which row will be inserted. - TableName string - // Prefix, Mid and Suffix are for sharded insert plans. Prefix string Mid sqlparser.Values @@ -90,10 +94,6 @@ type ( // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query QueryTimeout int - // VindexValueOffset stores the offset for each column in the ColumnVindex - // that will appear in the result set of the select query. - VindexValueOffset [][]int - // ForceNonStreaming is true when the insert table and select table are same. // This will avoid locking by the select table. ForceNonStreaming bool @@ -181,15 +181,11 @@ const ( // for each ColVindex. If the table has an Autoinc column, // A Generate subplan must be created. InsertSharded - // InsertSelect is for routing an insert statement - // based on rows returned from the select statement. - InsertSelect ) var insName = map[InsertOpcode]string{ InsertUnsharded: "InsertUnsharded", InsertSharded: "InsertSharded", - InsertSelect: "InsertSelect", } // String returns the opcode @@ -228,8 +224,6 @@ func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map return ins.execInsertUnsharded(ctx, vcursor, bindVars) case InsertSharded: return ins.insertIntoShardedTableFromValues(ctx, vcursor, bindVars) - case InsertSelect: - return ins.execInsertFromSelect(ctx, vcursor, bindVars) default: // Unreachable. return nil, fmt.Errorf("unsupported query route: %v", ins) diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go new file mode 100644 index 00000000000..1370a93b218 --- /dev/null +++ b/go/vt/vtgate/engine/insert_select.go @@ -0,0 +1,639 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + + "vitess.io/vitess/go/slice" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/evalengine" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +var _ Primitive = (*InsertSelect)(nil) + +type ( + // InsertSelect represents the instructions to perform an insert operation. + InsertSelect struct { + // Opcode is the execution opcode. + // Opcode InsertOpcode + + // Keyspace specifies the keyspace to send the query to. + Keyspace *vindexes.Keyspace + + // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs + // for sharded cases. + Ignore bool + + // TableName is the name of the table on which row will be inserted. + TableName string + + // Query specifies the query to be executed. + // For InsertSharded plans, this value is unused, + // and Prefix, Mid and Suffix are used instead. + Query string + + InsertRows *InsertRows + + // VindexValueOffset stores the offset for each column in the ColumnVindex + // that will appear in the result set of the select query. + VindexValueOffset [][]int + + // ColVindexes are the vindexes that will use the VindexValues + ColVindexes []*vindexes.ColumnVindex + + // Prefix, Mid and Suffix are for sharded insert plans. + Prefix string + Mid sqlparser.Values + Suffix string + + // Option to override the standard behavior and allow a multi-shard insert + // to use single round trip autocommit. + // + // This is a clear violation of the SQL semantics since it means the statement + // is not atomic in the presence of PK conflicts on one shard and not another. + // However, some application use cases would prefer that the statement partially + // succeed in order to get the performance benefits of autocommit. + MultiShardAutocommit bool + + // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query + QueryTimeout int + + // ForceNonStreaming is true when the insert table and select table are same. + // This will avoid locking by the select table. + ForceNonStreaming bool + + PreventAutoCommit bool + + // Insert needs tx handling + txNeeded + } +) + +func (ins *InsertSelect) Inputs() ([]Primitive, []map[string]any) { + return ins.InsertRows.Inputs() +} + +// NewQueryInsertSelect creates an InsertSelect with a query string. +func NewQueryInsertSelect(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *InsertSelect { + return &InsertSelect{ + Keyspace: keyspace, + Query: query, + InsertRows: NewInsertRows(nil), + } +} + +// NewInsertSelect creates a new InsertSelect. +func NewInsertSelect( + opcode InsertOpcode, + ignore bool, + keyspace *vindexes.Keyspace, + vindexValues [][][]evalengine.Expr, + table *vindexes.Table, + prefix string, + mid sqlparser.Values, + suffix string, +) *InsertSelect { + ins := &InsertSelect{ + // Opcode: opcode, + Ignore: ignore, + Keyspace: keyspace, + InsertRows: NewInsertRows(nil), + Prefix: prefix, + Mid: mid, + Suffix: suffix, + } + if table != nil { + ins.TableName = table.Name.String() + for _, colVindex := range table.ColumnVindexes { + if colVindex.IsPartialVindex() { + continue + } + ins.ColVindexes = append(ins.ColVindexes, colVindex) + } + } + return ins +} + +// RouteType returns a description of the query routing type used by the primitive +func (ins *InsertSelect) RouteType() string { + return "InsertSelect" +} + +// GetKeyspaceName specifies the Keyspace that this primitive routes to. +func (ins *InsertSelect) GetKeyspaceName() string { + return ins.Keyspace.Name +} + +// GetTableName specifies the table that this primitive routes to. +func (ins *InsertSelect) GetTableName() string { + return ins.TableName +} + +// TryExecute performs a non-streaming exec. +func (ins *InsertSelect) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, _ bool) (*sqltypes.Result, error) { + ctx, cancelFunc := addQueryTimeout(ctx, vcursor, ins.QueryTimeout) + defer cancelFunc() + + if ins.Keyspace.Sharded { + return ins.execInsertFromSelect(ctx, vcursor, bindVars) + } + return ins.execInsertUnsharded(ctx, vcursor, bindVars) +} + +// TryStreamExecute performs a streaming exec. +func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { + if ins.ForceNonStreaming { + res, err := ins.TryExecute(ctx, vcursor, bindVars, wantfields) + if err != nil { + return err + } + return callback(res) + } + ctx, cancelFunc := addQueryTimeout(ctx, vcursor, ins.QueryTimeout) + defer cancelFunc() + + sharded := ins.Keyspace.Sharded + var mu sync.Mutex + output := &sqltypes.Result{} + err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(irr insertRowsResult) error { + if len(irr.rows) == 0 { + return nil + } + + // should process only one chunk at a time. + // as parallel chunk insert will try to use the same transaction in the vttablet + // this will cause transaction in use error. + mu.Lock() + defer mu.Unlock() + + var insertID int64 + var qr *sqltypes.Result + var err error + if sharded { + insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, irr) + } else { + insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr.rows) + } + if err != nil { + return err + } + + output.RowsAffected += qr.RowsAffected + // InsertID needs to be updated to the least insertID value in sqltypes.Result + if output.InsertID == 0 || output.InsertID > uint64(insertID) { + output.InsertID = uint64(insertID) + } + return nil + }) + if err != nil { + return err + } + return callback(output) +} + +// GetFields fetches the field info. +func (ins *InsertSelect) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unreachable code for %q", ins.Query) +} + +func (ins *InsertSelect) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { + query := ins.Query + if ins.InsertRows.hasSelectInput() { + result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) + if err != nil { + return nil, err + } + if len(result.rows) == 0 { + return &sqltypes.Result{}, nil + } + query = ins.getInsertQueryFromSelectForUnsharded(result.rows, bindVars) + } + + _, qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) + return qr, err +} + +func (ins *InsertSelect) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { + var mids sqlparser.Values + for r, inputRow := range rows { + row := sqlparser.ValTuple{} + for c, value := range inputRow { + bvName := insertVarOffset(r, c) + bindVars[bvName] = sqltypes.ValueBindVariable(value) + row = append(row, sqlparser.NewArgument(bvName)) + } + mids = append(mids, row) + } + return ins.Prefix + sqlparser.String(mids) + ins.Suffix +} + +func (ins *InsertSelect) insertIntoShardedTableFromSelect( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, + irr insertRowsResult, +) (int64, *sqltypes.Result, error) { + rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, irr.rows) + if err != nil { + return 0, nil, err + } + + qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, irr.insertID) + if err != nil { + return 0, nil, err + } + return irr.insertID, qr, nil +} + +func (ins *InsertSelect) executeInsertQueries( + ctx context.Context, + vcursor VCursor, + rss []*srvtopo.ResolvedShard, + queries []*querypb.BoundQuery, + insertID int64, +) (*sqltypes.Result, error) { + autocommit := (len(rss) == 1 || ins.MultiShardAutocommit) && vcursor.AutocommitApproval() + err := allowOnlyPrimary(rss...) + if err != nil { + return nil, err + } + result, errs := vcursor.ExecuteMultiShard(ctx, ins, rss, queries, true /* rollbackOnError */, autocommit) + if errs != nil { + return nil, vterrors.Aggregate(errs) + } + + if insertID != 0 { + result.InsertID = uint64(insertID) + } + return result, nil +} + +func (ins *InsertSelect) getInsertSelectQueries( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, + rows []sqltypes.Row, +) ([]*srvtopo.ResolvedShard, []*querypb.BoundQuery, error) { + colVindexes := ins.ColVindexes + if len(colVindexes) != len(ins.VindexValueOffset) { + return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "vindex value offsets and vindex info do not match") + } + + // Here we go over the incoming rows and extract values for the vindexes we need to update + shardingCols := make([][]sqltypes.Row, len(colVindexes)) + for _, inputRow := range rows { + for colIdx := range colVindexes { + offsets := ins.VindexValueOffset[colIdx] + row := make(sqltypes.Row, 0, len(offsets)) + for _, offset := range offsets { + if offset == -1 { // value not provided from select query + row = append(row, sqltypes.NULL) + continue + } + row = append(row, inputRow[offset]) + } + shardingCols[colIdx] = append(shardingCols[colIdx], row) + } + } + + keyspaceIDs, err := ins.processPrimary(ctx, vcursor, shardingCols[0], colVindexes[0]) + if err != nil { + return nil, nil, err + } + + for vIdx := 1; vIdx < len(colVindexes); vIdx++ { + colVindex := colVindexes[vIdx] + var err error + if colVindex.Owned { + err = ins.processOwned(ctx, vcursor, shardingCols[vIdx], colVindex, keyspaceIDs) + } else { + err = ins.processUnowned(ctx, vcursor, shardingCols[vIdx], colVindex, keyspaceIDs) + } + if err != nil { + return nil, nil, err + } + } + + var indexes []*querypb.Value + var destinations []key.Destination + for i, ksid := range keyspaceIDs { + if ksid != nil { + indexes = append(indexes, &querypb.Value{ + Value: strconv.AppendInt(nil, int64(i), 10), + }) + destinations = append(destinations, key.DestinationKeyspaceID(ksid)) + } + } + if len(destinations) == 0 { + // In this case, all we have is nil KeyspaceIds, we don't do + // anything at all. + return nil, nil, nil + } + + rss, indexesPerRss, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, indexes, destinations) + if err != nil { + return nil, nil, err + } + + queries := make([]*querypb.BoundQuery, len(rss)) + for i := range rss { + bvs := sqltypes.CopyBindVariables(bindVars) // we don't want to create one huge bindvars for all values + var mids sqlparser.Values + for _, indexValue := range indexesPerRss[i] { + index, _ := strconv.Atoi(string(indexValue.Value)) + if keyspaceIDs[index] != nil { + row := sqlparser.ValTuple{} + for colOffset, value := range rows[index] { + bvName := insertVarOffset(index, colOffset) + bvs[bvName] = sqltypes.ValueBindVariable(value) + row = append(row, sqlparser.NewArgument(bvName)) + } + mids = append(mids, row) + } + } + rewritten := ins.Prefix + sqlparser.String(mids) + ins.Suffix + queries[i] = &querypb.BoundQuery{ + Sql: rewritten, + BindVariables: bvs, + } + } + + return rss, queries, nil +} + +func (ins *InsertSelect) execInsertFromSelect(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { + result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) + if err != nil { + return nil, err + } + if len(result.rows) == 0 { + return &sqltypes.Result{}, nil + } + + _, qr, err := ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result) + return qr, err +} + +// processPrimary maps the primary vindex values to the keyspace ids. +func (ins *InsertSelect) processPrimary(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex) ([]ksID, error) { + destinations, err := vindexes.Map(ctx, colVindex.Vindex, vcursor, vindexColumnsKeys) + if err != nil { + return nil, err + } + + keyspaceIDs := make([]ksID, len(destinations)) + for i, destination := range destinations { + switch d := destination.(type) { + case key.DestinationKeyspaceID: + // This is a single keyspace id, we're good. + keyspaceIDs[i] = d + case key.DestinationNone: + // No valid keyspace id, we may return an error. + if !ins.Ignore { + return nil, fmt.Errorf("could not map %v to a keyspace id", vindexColumnsKeys[i]) + } + default: + return nil, fmt.Errorf("could not map %v to a unique keyspace id: %v", vindexColumnsKeys[i], destination) + } + } + + return keyspaceIDs, nil +} + +// processOwned creates vindex entries for the values of an owned column. +func (ins *InsertSelect) processOwned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { + if !ins.Ignore { + return colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, vindexColumnsKeys, ksids, false /* ignoreMode */) + } + + // InsertIgnore + var createIndexes []int + var createKeys []sqltypes.Row + var createKsids []ksID + + for rowNum, rowColumnKeys := range vindexColumnsKeys { + if ksids[rowNum] == nil { + continue + } + createIndexes = append(createIndexes, rowNum) + createKeys = append(createKeys, rowColumnKeys) + createKsids = append(createKsids, ksids[rowNum]) + } + if createKeys == nil { + return nil + } + + err := colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, createKeys, createKsids, true) + if err != nil { + return err + } + // After creation, verify that the keys map to the keyspace ids. If not, remove + // those that don't map. + verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, createKeys, createKsids) + if err != nil { + return err + } + for i, v := range verified { + if !v { + ksids[createIndexes[i]] = nil + } + } + return nil +} + +// processUnowned either reverse maps or validates the values for an unowned column. +func (ins *InsertSelect) processUnowned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { + var reverseIndexes []int + var reverseKsids []ksID + + var verifyIndexes []int + var verifyKeys []sqltypes.Row + var verifyKsids []ksID + + // Check if this VIndex is reversible or not. + reversibleVindex, isReversible := colVindex.Vindex.(vindexes.Reversible) + + for rowNum, rowColumnKeys := range vindexColumnsKeys { + // If we weren't able to determine a keyspace id from the primary VIndex, skip this row + if ksids[rowNum] == nil { + continue + } + + if rowColumnKeys[0].IsNull() { + // If the value of the column is `NULL`, but this is a reversible VIndex, + // we will try to generate the value from the keyspace id generated by the primary VIndex. + if isReversible { + reverseIndexes = append(reverseIndexes, rowNum) + reverseKsids = append(reverseKsids, ksids[rowNum]) + } + + // Otherwise, don't do anything. Whether `NULL` is a valid value for this column will be + // handled by MySQL. + } else { + // If a value for this column was specified, the keyspace id values from the + // secondary VIndex need to be verified against the keyspace id from the primary VIndex + verifyIndexes = append(verifyIndexes, rowNum) + verifyKeys = append(verifyKeys, rowColumnKeys) + verifyKsids = append(verifyKsids, ksids[rowNum]) + } + } + + // Reverse map values for secondary VIndex columns from the primary VIndex's keyspace id. + if reverseKsids != nil { + reverseKeys, err := reversibleVindex.ReverseMap(vcursor, reverseKsids) + if err != nil { + return err + } + + for i, reverseKey := range reverseKeys { + // Fill the first column with the reverse-mapped value. + vindexColumnsKeys[reverseIndexes[i]][0] = reverseKey + } + } + + // Verify that the keyspace ids generated by the primary and secondary VIndexes match + if verifyIndexes != nil { + // If values were supplied, we validate against keyspace id. + verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, verifyKeys, verifyKsids) + if err != nil { + return err + } + + var mismatchVindexKeys []sqltypes.Row + for i, v := range verified { + rowNum := verifyIndexes[i] + if !v { + if !ins.Ignore { + mismatchVindexKeys = append(mismatchVindexKeys, vindexColumnsKeys[rowNum]) + continue + } + + // Skip the whole row if this is a `INSERT IGNORE` or `INSERT ... ON DUPLICATE KEY ...` statement + // but the keyspace ids didn't match. + ksids[verifyIndexes[i]] = nil + } + } + + if mismatchVindexKeys != nil { + return fmt.Errorf("values %v for column %v does not map to keyspace ids", mismatchVindexKeys, colVindex.Columns) + } + } + + return nil +} + +func (ins *InsertSelect) description() PrimitiveDescription { + other := map[string]any{ + "Query": ins.Query, + "TableName": ins.GetTableName(), + "MultiShardAutocommit": ins.MultiShardAutocommit, + "QueryTimeout": ins.QueryTimeout, + "InsertIgnore": ins.Ignore, + "InputAsNonStreaming": ins.ForceNonStreaming, + "NoAutoCommit": ins.PreventAutoCommit, + } + + if ins.InsertRows.Generate != nil { + if ins.InsertRows.Generate.Values == nil { + other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ins.InsertRows.Generate.Query, ins.InsertRows.Generate.Offset) + } else { + other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.InsertRows.Generate.Query, sqlparser.String(ins.InsertRows.Generate.Values)) + } + } + + if len(ins.VindexValueOffset) > 0 { + valuesOffsets := map[string]string{} + for idx, ints := range ins.VindexValueOffset { + if len(ins.ColVindexes) < idx { + panic("ins.ColVindexes and ins.VindexValueOffset do not line up") + } + vindex := ins.ColVindexes[idx] + marshal, _ := json.Marshal(ints) + valuesOffsets[vindex.Name] = string(marshal) + } + other["VindexOffsetFromSelect"] = valuesOffsets + } + if len(ins.Mid) > 0 { + mids := slice.Map(ins.Mid, func(from sqlparser.ValTuple) string { + return sqlparser.String(from) + }) + shardQuery := fmt.Sprintf("%s%s%s", ins.Prefix, strings.Join(mids, ", "), ins.Suffix) + if shardQuery != ins.Query { + other["ShardedQuery"] = shardQuery + } + } + return PrimitiveDescription{ + OperatorType: "Insert", + Keyspace: ins.Keyspace, + Variant: "Select", + TargetTabletType: topodatapb.TabletType_PRIMARY, + Other: other, + } +} + +func (ins *InsertSelect) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, rows []sqltypes.Row) (int64, *sqltypes.Result, error) { + query := ins.getInsertQueryFromSelectForUnsharded(rows, bindVars) + return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) +} + +func (ins *InsertSelect) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (int64, *sqltypes.Result, error) { + insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) + if err != nil { + return 0, nil, err + } + + rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) + if err != nil { + return 0, nil, err + } + if len(rss) != 1 { + return 0, nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) + } + err = allowOnlyPrimary(rss...) + if err != nil { + return 0, nil, err + } + qr, err := execShard(ctx, ins, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) + if err != nil { + return 0, nil, err + } + + // If processGenerateFromValues generated new values, it supersedes + // any ids that MySQL might have generated. If both generated + // values, we don't return an error because this behavior + // is required to support migration. + if insertID != 0 { + qr.InsertID = uint64(insertID) + } else { + insertID = int64(qr.InsertID) + } + return insertID, qr, nil +} diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index fc34e1131c6..f70004176e6 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1539,8 +1539,8 @@ func TestInsertSelectSimple(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ + // Opcode: InsertSelect, Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, @@ -1632,8 +1632,8 @@ func TestInsertSelectOwned(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ + // Opcode: InsertSelect, Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{ @@ -1728,8 +1728,8 @@ func TestInsertSelectGenerate(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( - InsertSelect, + ins := NewInsertSelect( + 0, false, ks.Keyspace, nil, @@ -1837,8 +1837,7 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{ @@ -1933,8 +1932,7 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key @@ -2022,8 +2020,7 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key @@ -2112,8 +2109,7 @@ func TestInsertSelectUnowned(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ Keyspace: ks.Keyspace, Query: "dummy_insert", VindexValueOffset: [][]int{{0}}, // the onecol vindex as unowned lookup sharding column @@ -2227,8 +2223,7 @@ func TestInsertSelectShardingCases(t *testing.T) { RoutingParameters: &RoutingParameters{Opcode: Unsharded, Keyspace: uks2.Keyspace}} // sks1 and sks2 - ins := &Insert{ - Opcode: InsertSelect, + ins := &InsertSelect{ Keyspace: sks1.Keyspace, Query: "dummy_insert", Prefix: "prefix ", @@ -2305,8 +2300,7 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard sks1.-20: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // uks1 and sks2 - ins = &Insert{ - Opcode: InsertUnsharded, + ins = &InsertSelect{ Keyspace: uks1.Keyspace, Query: "dummy_insert", Prefix: "prefix ", diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index d6e30d79609..2f84046ccdf 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -100,16 +100,18 @@ func insertUnshardedShortcut(stmt *sqlparser.Insert, ks *vindexes.Keyspace, tabl } type insert struct { - eInsert *engine.Insert - source logicalPlan + eInsert *engine.Insert + eInsertSelect *engine.InsertSelect + source logicalPlan } var _ logicalPlan = (*insert)(nil) func (i *insert) Primitive() engine.Primitive { - if i.source != nil { - input := i.source.Primitive() - i.eInsert.InsertRows.RowsFromSelect = input + if i.source == nil { + return i.eInsert } - return i.eInsert + input := i.source.Primitive() + i.eInsertSelect.InsertRows.RowsFromSelect = input + return i.eInsertSelect } diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 9f81792b651..2724db64e56 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -110,18 +110,16 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators } ins := dmlOp.(*operators.Insert) - eins := &engine.Insert{ - Opcode: mapToInsertOpCode(rb.Routing.OpCode(), true), + eins := &engine.InsertSelect{ Keyspace: rb.Routing.Keyspace(), TableName: ins.VTable.Name.String(), InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), Ignore: ins.Ignore, ForceNonStreaming: op.ForceNonStreaming, ColVindexes: ins.ColVindexes, - VindexValues: ins.VindexValues, VindexValueOffset: ins.VindexValueOffset, } - lp := &insert{eInsert: eins} + lp := &insert{eInsertSelect: eins} eins.Prefix, eins.Mid, eins.Suffix = generateInsertShardedQuery(ins.AST) @@ -579,9 +577,6 @@ func mapToInsertOpCode(code engine.Opcode, insertSelect bool) engine.InsertOpcod if code == engine.Unsharded { return engine.InsertUnsharded } - if insertSelect { - return engine.InsertSelect - } return engine.InsertSharded } diff --git a/go/vt/vtgate/planbuilder/testdata/dml_cases.json b/go/vt/vtgate/planbuilder/testdata/dml_cases.json index 5ec8210b12d..ec39db0b158 100644 --- a/go/vt/vtgate/planbuilder/testdata/dml_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/dml_cases.json @@ -4084,7 +4084,7 @@ "Original": "insert into unsharded(col) select col from unsharded_tab", "Instructions": { "OperatorType": "Insert", - "Variant": "Unsharded", + "Variant": "Select", "Keyspace": { "Name": "main", "Sharded": false @@ -4119,7 +4119,7 @@ "Original": "insert into unsharded(col) select col from t1", "Instructions": { "OperatorType": "Insert", - "Variant": "Unsharded", + "Variant": "Select", "Keyspace": { "Name": "main", "Sharded": false From 71abc7ca4cd6600b4da074e6b11e74c67a216f43 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 27 Nov 2023 22:44:33 +0530 Subject: [PATCH 07/15] refactor: removed unused code from new engine primitives Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/cached_size.go | 67 ++++-- go/vt/vtgate/engine/insert.go | 196 +----------------- go/vt/vtgate/engine/insert_select.go | 15 -- go/vt/vtgate/engine/insert_test.go | 2 - .../planbuilder/operator_transformers.go | 15 +- 5 files changed, 64 insertions(+), 231 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index d03b0f429b4..0bcb1035234 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -386,10 +386,12 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(224) + size += int64(192) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) + // field TableName string + size += hack.RuntimeAllocSize(int64(len(cached.TableName))) // field Query string size += hack.RuntimeAllocSize(int64(len(cached.Query))) // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows @@ -420,8 +422,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { size += elem.CachedSize(true) } } - // field TableName string - size += hack.RuntimeAllocSize(int64(len(cached.TableName))) // field Prefix string size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) // field Mid vitess.io/vitess/go/vt/sqlparser.Values @@ -440,15 +440,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } // field Suffix string size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) - // field VindexValueOffset [][]int - { - size += hack.RuntimeAllocSize(int64(cap(cached.VindexValueOffset)) * int64(24)) - for _, elem := range cached.VindexValueOffset { - { - size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(8)) - } - } - } return size } func (cached *InsertRows) CachedSize(alloc bool) int64 { @@ -481,6 +472,58 @@ func (cached *InsertRows) CachedSize(alloc bool) int64 { } return size } +func (cached *InsertSelect) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(192) + } + // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace + size += cached.Keyspace.CachedSize(true) + // field TableName string + size += hack.RuntimeAllocSize(int64(len(cached.TableName))) + // field Query string + size += hack.RuntimeAllocSize(int64(len(cached.Query))) + // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows + size += cached.InsertRows.CachedSize(true) + // field VindexValueOffset [][]int + { + size += hack.RuntimeAllocSize(int64(cap(cached.VindexValueOffset)) * int64(24)) + for _, elem := range cached.VindexValueOffset { + { + size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(8)) + } + } + } + // field ColVindexes []*vitess.io/vitess/go/vt/vtgate/vindexes.ColumnVindex + { + size += hack.RuntimeAllocSize(int64(cap(cached.ColVindexes)) * int64(8)) + for _, elem := range cached.ColVindexes { + size += elem.CachedSize(true) + } + } + // field Prefix string + size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) + // field Mid vitess.io/vitess/go/vt/sqlparser.Values + { + size += hack.RuntimeAllocSize(int64(cap(cached.Mid)) * int64(24)) + for _, elem := range cached.Mid { + { + size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(16)) + for _, elem := range elem { + if cc, ok := elem.(cachedObject); ok { + size += cc.CachedSize(true) + } + } + } + } + } + // field Suffix string + size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) + return size +} //go:nocheckptr func (cached *Join) CachedSize(alloc bool) int64 { diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 0de08602458..b08551d3c5a 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -22,8 +22,6 @@ import ( "fmt" "strconv" "strings" - "sync" - "time" "vitess.io/vitess/go/slice" "vitess.io/vitess/go/sqltypes" @@ -70,10 +68,6 @@ type ( // Insert.Values[i].Values[j].Values[k] represents the value pulled from row k for that column: (k < len(ins.rows)) VindexValues [][][]evalengine.Expr - // VindexValueOffset stores the offset for each column in the ColumnVindex - // that will appear in the result set of the select query. - VindexValueOffset [][]int - // ColVindexes are the vindexes that will use the VindexValues ColVindexes []*vindexes.ColumnVindex @@ -232,56 +226,11 @@ func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map // TryStreamExecute performs a streaming exec. func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - if !ins.InsertRows.hasSelectInput() || ins.ForceNonStreaming { - res, err := ins.TryExecute(ctx, vcursor, bindVars, wantfields) - if err != nil { - return err - } - return callback(res) - } - if ins.QueryTimeout != 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Duration(ins.QueryTimeout)*time.Millisecond) - defer cancel() - } - - unsharded := ins.Opcode == InsertUnsharded - var mu sync.Mutex - output := &sqltypes.Result{} - err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(irr insertRowsResult) error { - if len(irr.rows) == 0 { - return nil - } - - // should process only one chunk at a time. - // as parallel chunk insert will try to use the same transaction in the vttablet - // this will cause transaction in use error. - mu.Lock() - defer mu.Unlock() - - var insertID int64 - var qr *sqltypes.Result - var err error - if unsharded { - insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr.rows) - } else { - insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, irr) - } - if err != nil { - return err - } - - output.RowsAffected += qr.RowsAffected - // InsertID needs to be updated to the least insertID value in sqltypes.Result - if output.InsertID == 0 || output.InsertID > uint64(insertID) { - output.InsertID = uint64(insertID) - } - return nil - }) + res, err := ins.TryExecute(ctx, vcursor, bindVars, wantfields) if err != nil { return err } - return callback(output) + return callback(res) } // GetFields fetches the field info. @@ -337,24 +286,6 @@ func (ins *Insert) insertIntoShardedTableFromValues( return ins.executeInsertQueries(ctx, vcursor, rss, queries, insertID) } -func (ins *Insert) insertIntoShardedTableFromSelect( - ctx context.Context, - vcursor VCursor, - bindVars map[string]*querypb.BindVariable, - irr insertRowsResult, -) (int64, *sqltypes.Result, error) { - rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, irr.rows) - if err != nil { - return 0, nil, err - } - - qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, irr.insertID) - if err != nil { - return 0, nil, err - } - return irr.insertID, qr, nil -} - func (ins *Insert) executeInsertQueries( ctx context.Context, vcursor VCursor, @@ -378,112 +309,6 @@ func (ins *Insert) executeInsertQueries( return result, nil } -func (ins *Insert) getInsertSelectQueries( - ctx context.Context, - vcursor VCursor, - bindVars map[string]*querypb.BindVariable, - rows []sqltypes.Row, -) ([]*srvtopo.ResolvedShard, []*querypb.BoundQuery, error) { - colVindexes := ins.ColVindexes - if len(colVindexes) != len(ins.VindexValueOffset) { - return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "vindex value offsets and vindex info do not match") - } - - // Here we go over the incoming rows and extract values for the vindexes we need to update - shardingCols := make([][]sqltypes.Row, len(colVindexes)) - for _, inputRow := range rows { - for colIdx := range colVindexes { - offsets := ins.VindexValueOffset[colIdx] - row := make(sqltypes.Row, 0, len(offsets)) - for _, offset := range offsets { - if offset == -1 { // value not provided from select query - row = append(row, sqltypes.NULL) - continue - } - row = append(row, inputRow[offset]) - } - shardingCols[colIdx] = append(shardingCols[colIdx], row) - } - } - - keyspaceIDs, err := ins.processPrimary(ctx, vcursor, shardingCols[0], colVindexes[0]) - if err != nil { - return nil, nil, err - } - - for vIdx := 1; vIdx < len(colVindexes); vIdx++ { - colVindex := colVindexes[vIdx] - var err error - if colVindex.Owned { - err = ins.processOwned(ctx, vcursor, shardingCols[vIdx], colVindex, keyspaceIDs) - } else { - err = ins.processUnowned(ctx, vcursor, shardingCols[vIdx], colVindex, keyspaceIDs) - } - if err != nil { - return nil, nil, err - } - } - - var indexes []*querypb.Value - var destinations []key.Destination - for i, ksid := range keyspaceIDs { - if ksid != nil { - indexes = append(indexes, &querypb.Value{ - Value: strconv.AppendInt(nil, int64(i), 10), - }) - destinations = append(destinations, key.DestinationKeyspaceID(ksid)) - } - } - if len(destinations) == 0 { - // In this case, all we have is nil KeyspaceIds, we don't do - // anything at all. - return nil, nil, nil - } - - rss, indexesPerRss, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, indexes, destinations) - if err != nil { - return nil, nil, err - } - - queries := make([]*querypb.BoundQuery, len(rss)) - for i := range rss { - bvs := sqltypes.CopyBindVariables(bindVars) // we don't want to create one huge bindvars for all values - var mids sqlparser.Values - for _, indexValue := range indexesPerRss[i] { - index, _ := strconv.Atoi(string(indexValue.Value)) - if keyspaceIDs[index] != nil { - row := sqlparser.ValTuple{} - for colOffset, value := range rows[index] { - bvName := insertVarOffset(index, colOffset) - bvs[bvName] = sqltypes.ValueBindVariable(value) - row = append(row, sqlparser.NewArgument(bvName)) - } - mids = append(mids, row) - } - } - rewritten := ins.Prefix + sqlparser.String(mids) + ins.Suffix - queries[i] = &querypb.BoundQuery{ - Sql: rewritten, - BindVariables: bvs, - } - } - - return rss, queries, nil -} - -func (ins *Insert) execInsertFromSelect(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) - if err != nil { - return nil, err - } - if len(result.rows) == 0 { - return &sqltypes.Result{}, nil - } - - _, qr, err := ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result) - return qr, err -} - // getInsertQueriesFromValues performs all the vindex related work // and returns a map of shard to queries. // Using the primary vindex, it computes the target keyspace ids. @@ -837,18 +662,6 @@ func (ins *Insert) description() PrimitiveDescription { } } - if len(ins.VindexValueOffset) > 0 { - valuesOffsets := map[string]string{} - for idx, ints := range ins.VindexValueOffset { - if len(ins.ColVindexes) < idx { - panic("ins.ColVindexes and ins.VindexValueOffset do not line up") - } - vindex := ins.ColVindexes[idx] - marshal, _ := json.Marshal(ints) - valuesOffsets[vindex.Name] = string(marshal) - } - other["VindexOffsetFromSelect"] = valuesOffsets - } if len(ins.Mid) > 0 { mids := slice.Map(ins.Mid, func(from sqlparser.ValTuple) string { return sqlparser.String(from) @@ -867,11 +680,6 @@ func (ins *Insert) description() PrimitiveDescription { } } -func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, rows []sqltypes.Row) (int64, *sqltypes.Result, error) { - query := ins.getInsertQueryFromSelectForUnsharded(rows, bindVars) - return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) -} - func (ins *Insert) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (int64, *sqltypes.Result, error) { insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index 1370a93b218..35dee544536 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -33,7 +33,6 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -42,8 +41,6 @@ var _ Primitive = (*InsertSelect)(nil) type ( // InsertSelect represents the instructions to perform an insert operation. InsertSelect struct { - // Opcode is the execution opcode. - // Opcode InsertOpcode // Keyspace specifies the keyspace to send the query to. Keyspace *vindexes.Keyspace @@ -101,28 +98,16 @@ func (ins *InsertSelect) Inputs() ([]Primitive, []map[string]any) { return ins.InsertRows.Inputs() } -// NewQueryInsertSelect creates an InsertSelect with a query string. -func NewQueryInsertSelect(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *InsertSelect { - return &InsertSelect{ - Keyspace: keyspace, - Query: query, - InsertRows: NewInsertRows(nil), - } -} - // NewInsertSelect creates a new InsertSelect. func NewInsertSelect( - opcode InsertOpcode, ignore bool, keyspace *vindexes.Keyspace, - vindexValues [][][]evalengine.Expr, table *vindexes.Table, prefix string, mid sqlparser.Values, suffix string, ) *InsertSelect { ins := &InsertSelect{ - // Opcode: opcode, Ignore: ignore, Keyspace: keyspace, InsertRows: NewInsertRows(nil), diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index f70004176e6..25e5ff24867 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1729,10 +1729,8 @@ func TestInsertSelectGenerate(t *testing.T) { ks := vs.Keyspaces["sharded"] ins := NewInsertSelect( - 0, false, ks.Keyspace, - nil, ks.Tables["t1"], "prefix ", nil, diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 2724db64e56..b5da547c89a 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -547,14 +547,13 @@ func buildInsertLogicalPlan( ) (logicalPlan, error) { ins := op.(*operators.Insert) eins := &engine.Insert{ - Opcode: mapToInsertOpCode(rb.Routing.OpCode(), false), - Keyspace: rb.Routing.Keyspace(), - TableName: ins.VTable.Name.String(), - Ignore: ins.Ignore, - InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), - ColVindexes: ins.ColVindexes, - VindexValues: ins.VindexValues, - VindexValueOffset: ins.VindexValueOffset, + Opcode: mapToInsertOpCode(rb.Routing.OpCode(), false), + Keyspace: rb.Routing.Keyspace(), + TableName: ins.VTable.Name.String(), + Ignore: ins.Ignore, + InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), + ColVindexes: ins.ColVindexes, + VindexValues: ins.VindexValues, } lp := &insert{eInsert: eins} From 82acc31b0cda39883488f72710927e4eee5e2d96 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Mon, 27 Nov 2023 23:50:04 +0530 Subject: [PATCH 08/15] refactor: moved some part to insert common Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/cached_size.go | 44 ++--- go/vt/vtgate/engine/insert.go | 36 ++-- go/vt/vtgate/engine/insert_rows.go | 24 ++- go/vt/vtgate/engine/insert_select.go | 51 ++--- go/vt/vtgate/engine/insert_test.go | 178 +++++++++--------- go/vt/vtgate/planbuilder/insert.go | 11 +- .../planbuilder/operator_transformers.go | 20 +- 7 files changed, 184 insertions(+), 180 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 0bcb1035234..3960ee8a21a 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -386,12 +386,10 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(192) + size += int64(160) } - // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace - size += cached.Keyspace.CachedSize(true) - // field TableName string - size += hack.RuntimeAllocSize(int64(len(cached.TableName))) + // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon + size += cached.InsertCommon.CachedSize(true) // field Query string size += hack.RuntimeAllocSize(int64(len(cached.Query))) // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows @@ -442,6 +440,20 @@ func (cached *Insert) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) return size } +func (cached *InsertCommon) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(48) + } + // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace + size += cached.Keyspace.CachedSize(true) + // field TableName string + size += hack.RuntimeAllocSize(int64(len(cached.TableName))) + return size +} func (cached *InsertRows) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) @@ -478,12 +490,10 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(192) + size += int64(144) } - // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace - size += cached.Keyspace.CachedSize(true) - // field TableName string - size += hack.RuntimeAllocSize(int64(len(cached.TableName))) + // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon + size += cached.InsertCommon.CachedSize(true) // field Query string size += hack.RuntimeAllocSize(int64(len(cached.Query))) // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows @@ -506,20 +516,6 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } // field Prefix string size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) - // field Mid vitess.io/vitess/go/vt/sqlparser.Values - { - size += hack.RuntimeAllocSize(int64(cap(cached.Mid)) * int64(24)) - for _, elem := range cached.Mid { - { - size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(16)) - for _, elem := range elem { - if cc, ok := elem.(cachedObject); ok { - size += cc.CachedSize(true) - } - } - } - } - } // field Suffix string size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) return size diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index b08551d3c5a..42a66afb971 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -39,8 +39,7 @@ import ( var _ Primitive = (*Insert)(nil) type ( - // Insert represents the instructions to perform an insert operation. - Insert struct { + InsertCommon struct { // Opcode is the execution opcode. Opcode InsertOpcode @@ -54,6 +53,14 @@ type ( // TableName is the name of the table on which row will be inserted. TableName string + // Insert needs tx handling + txNeeded + } + + // Insert represents the instructions to perform an insert operation. + Insert struct { + *InsertCommon + // Query specifies the query to be executed. // For InsertSharded plans, this value is unused, // and Prefix, Mid and Suffix are used instead. @@ -93,9 +100,6 @@ type ( ForceNonStreaming bool PreventAutoCommit bool - - // Insert needs tx handling - txNeeded } ksID = []byte @@ -108,8 +112,10 @@ func (ins *Insert) Inputs() ([]Primitive, []map[string]any) { // NewQueryInsert creates an Insert with a query string. func NewQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { return &Insert{ - Opcode: opcode, - Keyspace: keyspace, + InsertCommon: &InsertCommon{ + Opcode: opcode, + Keyspace: keyspace, + }, Query: query, InsertRows: NewInsertRows(nil), } @@ -127,9 +133,11 @@ func NewInsert( suffix string, ) *Insert { ins := &Insert{ - Opcode: opcode, - Ignore: ignore, - Keyspace: keyspace, + InsertCommon: &InsertCommon{ + Opcode: opcode, + Keyspace: keyspace, + Ignore: ignore, + }, InsertRows: NewInsertRows(nil), VindexValues: vindexValues, Prefix: prefix, @@ -654,13 +662,7 @@ func (ins *Insert) description() PrimitiveDescription { other["VindexValues"] = valuesOffsets } - if ins.InsertRows.Generate != nil { - if ins.InsertRows.Generate.Values == nil { - other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ins.InsertRows.Generate.Query, ins.InsertRows.Generate.Offset) - } else { - other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.InsertRows.Generate.Query, sqlparser.String(ins.InsertRows.Generate.Values)) - } - } + ins.InsertRows.describe(other) if len(ins.Mid) > 0 { mids := slice.Map(ins.Mid, func(from sqlparser.ValTuple) string { diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go index 1bd92167397..dc59e2aecab 100644 --- a/go/vt/vtgate/engine/insert_rows.go +++ b/go/vt/vtgate/engine/insert_rows.go @@ -18,6 +18,7 @@ package engine import ( "context" + "fmt" "strconv" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -127,7 +128,7 @@ func (ir *InsertRows) processGenerateFromSelect( genColPresent := offset < len(rows[0]) if genColPresent { for _, row := range rows { - if shouldGenerate(row[offset]) { + if shouldGenerate(row[offset], evalengine.ParseSQLMode(vcursor.SQLMode())) { count++ } } @@ -147,7 +148,7 @@ func (ir *InsertRows) processGenerateFromSelect( used := insertID for idx, val := range rows { if genColPresent { - if shouldGenerate(val[offset]) { + if shouldGenerate(val[offset], evalengine.ParseSQLMode(vcursor.SQLMode())) { val[offset] = sqltypes.NewInt64(used) used++ } @@ -182,7 +183,7 @@ func (ir *InsertRows) processGenerateFromValues( count := int64(0) values := resolved.TupleValues() for _, val := range values { - if shouldGenerate(val) { + if shouldGenerate(val, evalengine.ParseSQLMode(vcursor.SQLMode())) { count++ } } @@ -198,7 +199,7 @@ func (ir *InsertRows) processGenerateFromValues( // Fill the holes where no value was supplied. cur := insertID for i, v := range values { - if shouldGenerate(v) { + if shouldGenerate(v, evalengine.ParseSQLMode(vcursor.SQLMode())) { bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.Int64BindVariable(cur) cur++ } else { @@ -228,14 +229,14 @@ func (ir *InsertRows) execGenerate(ctx context.Context, vcursor VCursor, count i } // shouldGenerate determines if a sequence value should be generated for a given value -func shouldGenerate(v sqltypes.Value) bool { +func shouldGenerate(v sqltypes.Value, sqlmode evalengine.SQLMode) bool { if v.IsNull() { return true } // Unless the NO_AUTO_VALUE_ON_ZERO sql mode is active in mysql, it also // treats 0 as a value that should generate a new sequence. - value, err := evalengine.CoerceTo(v, sqltypes.Uint64) + value, err := evalengine.CoerceTo(v, sqltypes.Uint64, sqlmode) if err != nil { return false } @@ -247,3 +248,14 @@ func shouldGenerate(v sqltypes.Value) bool { return id == 0 } + +func (ir *InsertRows) describe(other map[string]any) { + if ir == nil || ir.Generate == nil { + return + } + if ir.Generate.Values == nil { + other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ir.Generate.Query, ir.Generate.Offset) + } else { + other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ir.Generate.Query, sqlparser.String(ir.Generate.Values)) + } +} diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index 35dee544536..a8016ff22c8 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -21,10 +21,8 @@ import ( "encoding/json" "fmt" "strconv" - "strings" "sync" - "vitess.io/vitess/go/slice" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" @@ -41,16 +39,7 @@ var _ Primitive = (*InsertSelect)(nil) type ( // InsertSelect represents the instructions to perform an insert operation. InsertSelect struct { - - // Keyspace specifies the keyspace to send the query to. - Keyspace *vindexes.Keyspace - - // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs - // for sharded cases. - Ignore bool - - // TableName is the name of the table on which row will be inserted. - TableName string + *InsertCommon // Query specifies the query to be executed. // For InsertSharded plans, this value is unused, @@ -68,7 +57,6 @@ type ( // Prefix, Mid and Suffix are for sharded insert plans. Prefix string - Mid sqlparser.Values Suffix string // Option to override the standard behavior and allow a multi-shard insert @@ -104,16 +92,21 @@ func NewInsertSelect( keyspace *vindexes.Keyspace, table *vindexes.Table, prefix string, - mid sqlparser.Values, suffix string, + vv [][]int, + query string, + ir *InsertRows, ) *InsertSelect { ins := &InsertSelect{ - Ignore: ignore, - Keyspace: keyspace, - InsertRows: NewInsertRows(nil), - Prefix: prefix, - Mid: mid, - Suffix: suffix, + InsertCommon: &InsertCommon{ + Ignore: ignore, + Keyspace: keyspace, + }, + InsertRows: ir, + Prefix: prefix, + Suffix: suffix, + VindexValueOffset: vv, + Query: query, } if table != nil { ins.TableName = table.Name.String() @@ -546,13 +539,7 @@ func (ins *InsertSelect) description() PrimitiveDescription { "NoAutoCommit": ins.PreventAutoCommit, } - if ins.InsertRows.Generate != nil { - if ins.InsertRows.Generate.Values == nil { - other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ins.InsertRows.Generate.Query, ins.InsertRows.Generate.Offset) - } else { - other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.InsertRows.Generate.Query, sqlparser.String(ins.InsertRows.Generate.Values)) - } - } + ins.InsertRows.describe(other) if len(ins.VindexValueOffset) > 0 { valuesOffsets := map[string]string{} @@ -566,15 +553,7 @@ func (ins *InsertSelect) description() PrimitiveDescription { } other["VindexOffsetFromSelect"] = valuesOffsets } - if len(ins.Mid) > 0 { - mids := slice.Map(ins.Mid, func(from sqlparser.ValTuple) string { - return sqlparser.String(from) - }) - shardQuery := fmt.Sprintf("%s%s%s", ins.Prefix, strings.Join(mids, ", "), ins.Suffix) - if shardQuery != ins.Query { - other["ShardedQuery"] = shardQuery - } - } + return PrimitiveDescription{ OperatorType: "Insert", Keyspace: ins.Keyspace, diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 25e5ff24867..5ca461b743e 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1539,18 +1539,16 @@ func TestInsertSelectSimple(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &InsertSelect{ - // Opcode: InsertSelect, - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{{1}}, - InsertRows: &InsertRows{ - RowsFromSelect: rb, - }} - - ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t1"].ColumnVindexes...) - ins.Prefix = "prefix " - ins.Suffix = " suffix" + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t1"], + "prefix ", + " suffix", + [][]int{{1}}, + "dummy_insert", + NewInsertRowsFromSelect(nil, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1632,19 +1630,19 @@ func TestInsertSelectOwned(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &InsertSelect{ - // Opcode: InsertSelect, - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{ + + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t1"], + "prefix ", + " suffix", + [][]int{ {1}, // The primary vindex has a single column as sharding key {0}}, // the onecol vindex uses the 'name' column - InsertRows: NewInsertRowsFromSelect(nil, rb), - } - - ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t1"].ColumnVindexes...) - ins.Prefix = "prefix " - ins.Suffix = " suffix" + "dummy_insert", + NewInsertRowsFromSelect(nil, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1728,15 +1726,6 @@ func TestInsertSelectGenerate(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsertSelect( - false, - ks.Keyspace, - ks.Tables["t1"], - "prefix ", - nil, - " suffix") - ins.Query = "dummy_insert" - ins.VindexValueOffset = [][]int{{1}} // The primary vindex has a single column as sharding key rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -1752,7 +1741,17 @@ func TestInsertSelectGenerate(t *testing.T) { Query: "dummy_generate", Offset: 1, } - ins.InsertRows = NewInsertRowsFromSelect(gen, rb) + + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t1"], + "prefix ", + " suffix", + [][]int{{1}}, // The primary vindex has a single column as sharding key + "dummy_insert", + NewInsertRowsFromSelect(gen, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1835,15 +1834,18 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &InsertSelect{ - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{ + + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t1"], + "prefix ", + " suffix", + [][]int{ {1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(generate, rb)} - ins.ColVindexes = ks.Tables["t1"].ColumnVindexes - ins.Prefix = "prefix " - ins.Suffix = " suffix" + "dummy_insert", + NewInsertRowsFromSelect(generate, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1930,16 +1932,16 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &InsertSelect{ - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(generate, rb), - } - - ins.ColVindexes = ks.Tables["t1"].ColumnVindexes - ins.Prefix = "prefix " - ins.Suffix = " suffix" + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t1"], + "prefix ", + " suffix", + [][]int{{1}}, // The primary vindex has a single column as sharding key, + "dummy_insert", + NewInsertRowsFromSelect(generate, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -2018,15 +2020,16 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &InsertSelect{ - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{{1}}, // The primary vindex has a single column as sharding key - InsertRows: NewInsertRowsFromSelect(generate, rb)} - - ins.ColVindexes = ks.Tables["t1"].ColumnVindexes - ins.Prefix = "prefix " - ins.Suffix = " suffix" + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t1"], + "prefix ", + " suffix", + [][]int{{1}}, // The primary vindex has a single column as sharding key, + "dummy_insert", + NewInsertRowsFromSelect(generate, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -2107,15 +2110,16 @@ func TestInsertSelectUnowned(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := &InsertSelect{ - Keyspace: ks.Keyspace, - Query: "dummy_insert", - VindexValueOffset: [][]int{{0}}, // the onecol vindex as unowned lookup sharding column - InsertRows: NewInsertRowsFromSelect(nil, rb)} - - ins.ColVindexes = append(ins.ColVindexes, ks.Tables["t2"].ColumnVindexes...) - ins.Prefix = "prefix " - ins.Suffix = " suffix" + ins := NewInsertSelect( + false, + ks.Keyspace, + ks.Tables["t2"], + "prefix ", + " suffix", + [][]int{{0}}, // // the onecol vindex as unowned lookup sharding column + "dummy_insert", + NewInsertRowsFromSelect(nil, rb), + ) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -2221,15 +2225,16 @@ func TestInsertSelectShardingCases(t *testing.T) { RoutingParameters: &RoutingParameters{Opcode: Unsharded, Keyspace: uks2.Keyspace}} // sks1 and sks2 - ins := &InsertSelect{ - Keyspace: sks1.Keyspace, - Query: "dummy_insert", - Prefix: "prefix ", - Suffix: " suffix", - ColVindexes: sks1.Tables["s1"].ColumnVindexes, - VindexValueOffset: [][]int{{0}}, - InsertRows: NewInsertRowsFromSelect(nil, sRoute), - } + ins := NewInsertSelect( + false, + sks1.Keyspace, + sks1.Tables["s1"], + "prefix ", + " suffix", + [][]int{{0}}, + "dummy_insert", + NewInsertRowsFromSelect(nil, sRoute), + ) vc := &loggingVCursor{ resolvedTargetTabletType: topodatapb.TabletType_PRIMARY, @@ -2298,13 +2303,16 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard sks1.-20: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // uks1 and sks2 - ins = &InsertSelect{ - Keyspace: uks1.Keyspace, - Query: "dummy_insert", - Prefix: "prefix ", - Suffix: " suffix", - InsertRows: NewInsertRowsFromSelect(nil, sRoute), - } + ins = NewInsertSelect( + false, + uks1.Keyspace, + nil, + "prefix ", + " suffix", + nil, + "dummy_insert", + NewInsertRowsFromSelect(nil, sRoute), + ) vc.Rewind() _, err = ins.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 2f84046ccdf..4bf2266ec3b 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -90,10 +90,13 @@ func errOutIfPlanCannotBeConstructed(ctx *plancontext.PlanningContext, vTbl *vin } func insertUnshardedShortcut(stmt *sqlparser.Insert, ks *vindexes.Keyspace, tables []*vindexes.Table) logicalPlan { - eIns := &engine.Insert{} - eIns.Keyspace = ks - eIns.TableName = tables[0].Name.String() - eIns.Opcode = engine.InsertUnsharded + eIns := &engine.Insert{ + InsertCommon: &engine.InsertCommon{ + Opcode: engine.InsertUnsharded, + Keyspace: ks, + TableName: tables[0].Name.String(), + }, + } eIns.Query = generateQuery(stmt) eIns.InsertRows = engine.NewInsertRows(nil) return &insert{eInsert: eIns} diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index b5da547c89a..c11082511e2 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -111,17 +111,19 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators ins := dmlOp.(*operators.Insert) eins := &engine.InsertSelect{ - Keyspace: rb.Routing.Keyspace(), - TableName: ins.VTable.Name.String(), + InsertCommon: &engine.InsertCommon{ + Keyspace: rb.Routing.Keyspace(), + TableName: ins.VTable.Name.String(), + Ignore: ins.Ignore, + }, InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), - Ignore: ins.Ignore, ForceNonStreaming: op.ForceNonStreaming, ColVindexes: ins.ColVindexes, VindexValueOffset: ins.VindexValueOffset, } lp := &insert{eInsertSelect: eins} - eins.Prefix, eins.Mid, eins.Suffix = generateInsertShardedQuery(ins.AST) + eins.Prefix, _, eins.Suffix = generateInsertShardedQuery(ins.AST) selectionPlan, err := transformToLogicalPlan(ctx, op.Select) if err != nil { @@ -547,10 +549,12 @@ func buildInsertLogicalPlan( ) (logicalPlan, error) { ins := op.(*operators.Insert) eins := &engine.Insert{ - Opcode: mapToInsertOpCode(rb.Routing.OpCode(), false), - Keyspace: rb.Routing.Keyspace(), - TableName: ins.VTable.Name.String(), - Ignore: ins.Ignore, + InsertCommon: &engine.InsertCommon{ + Opcode: mapToInsertOpCode(rb.Routing.OpCode(), false), + Keyspace: rb.Routing.Keyspace(), + TableName: ins.VTable.Name.String(), + Ignore: ins.Ignore, + }, InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), ColVindexes: ins.ColVindexes, VindexValues: ins.VindexValues, From 954f374d0f34dea70e34017529b397680e2d1e08 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 28 Nov 2023 13:39:07 +0530 Subject: [PATCH 09/15] refactor: unionize the code for insert select Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert_rows.go | 12 ++++ go/vt/vtgate/engine/insert_select.go | 103 +++++++++++---------------- 2 files changed, 55 insertions(+), 60 deletions(-) diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go index dc59e2aecab..f7302b96caf 100644 --- a/go/vt/vtgate/engine/insert_rows.go +++ b/go/vt/vtgate/engine/insert_rows.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strconv" + "sync" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -99,7 +100,18 @@ func (ir *InsertRows) execSelectStreaming( bindVars map[string]*querypb.BindVariable, callback func(irr insertRowsResult) error, ) error { + var mu sync.Mutex return vcursor.StreamExecutePrimitiveStandalone(ctx, ir.RowsFromSelect, bindVars, false, func(result *sqltypes.Result) error { + if len(result.Rows) == 0 { + return nil + } + + // should process only one chunk at a time. + // as parallel chunk insert will try to use the same transaction in the vttablet + // this will cause transaction in use error. + mu.Lock() + defer mu.Unlock() + insertID, err := ir.processGenerateFromSelect(ctx, vcursor, result.Rows) if err != nil { return err diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index a8016ff22c8..a6c1b546d2b 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "strconv" - "sync" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" @@ -159,26 +158,19 @@ func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, defer cancelFunc() sharded := ins.Keyspace.Sharded - var mu sync.Mutex output := &sqltypes.Result{} err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(irr insertRowsResult) error { if len(irr.rows) == 0 { return nil } - // should process only one chunk at a time. - // as parallel chunk insert will try to use the same transaction in the vttablet - // this will cause transaction in use error. - mu.Lock() - defer mu.Unlock() - var insertID int64 var qr *sqltypes.Result var err error if sharded { insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, irr) } else { - insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr.rows) + insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr) } if err != nil { return err @@ -203,22 +195,36 @@ func (ins *InsertSelect) GetFields(context.Context, VCursor, map[string]*querypb } func (ins *InsertSelect) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - query := ins.Query - if ins.InsertRows.hasSelectInput() { - result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) - if err != nil { - return nil, err - } - if len(result.rows) == 0 { - return &sqltypes.Result{}, nil - } - query = ins.getInsertQueryFromSelectForUnsharded(result.rows, bindVars) + irr, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) + if err != nil { + return nil, err } - - _, qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) + if len(irr.rows) == 0 { + return &sqltypes.Result{}, nil + } + _, qr, err := ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr) return qr, err } +func (ins *InsertSelect) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, irr insertRowsResult) (int64, *sqltypes.Result, error) { + query := ins.getInsertQueryFromSelectForUnsharded(irr.rows, bindVars) + qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) + if err != nil { + return 0, nil, err + } + + // If processGenerateFromValues generated new values, it supersedes + // any ids that MySQL might have generated. If both generated + // values, we don't return an error because this behavior + // is required to support migration. + if irr.insertID != 0 { + qr.InsertID = uint64(irr.insertID) + } else { + irr.insertID = int64(qr.InsertID) + } + return irr.insertID, qr, nil +} + func (ins *InsertSelect) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { var mids sqlparser.Values for r, inputRow := range rows { @@ -233,6 +239,21 @@ func (ins *InsertSelect) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Ro return ins.Prefix + sqlparser.String(mids) + ins.Suffix } +func (ins *InsertSelect) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (*sqltypes.Result, error) { + rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) + if err != nil { + return nil, err + } + if len(rss) != 1 { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) + } + err = allowOnlyPrimary(rss...) + if err != nil { + return nil, err + } + return execShard(ctx, ins, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) +} + func (ins *InsertSelect) insertIntoShardedTableFromSelect( ctx context.Context, vcursor VCursor, @@ -248,6 +269,7 @@ func (ins *InsertSelect) insertIntoShardedTableFromSelect( if err != nil { return 0, nil, err } + qr.InsertID = uint64(irr.insertID) return irr.insertID, qr, nil } @@ -562,42 +584,3 @@ func (ins *InsertSelect) description() PrimitiveDescription { Other: other, } } - -func (ins *InsertSelect) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, rows []sqltypes.Row) (int64, *sqltypes.Result, error) { - query := ins.getInsertQueryFromSelectForUnsharded(rows, bindVars) - return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) -} - -func (ins *InsertSelect) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (int64, *sqltypes.Result, error) { - insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) - if err != nil { - return 0, nil, err - } - - rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) - if err != nil { - return 0, nil, err - } - if len(rss) != 1 { - return 0, nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) - } - err = allowOnlyPrimary(rss...) - if err != nil { - return 0, nil, err - } - qr, err := execShard(ctx, ins, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) - if err != nil { - return 0, nil, err - } - - // If processGenerateFromValues generated new values, it supersedes - // any ids that MySQL might have generated. If both generated - // values, we don't return an error because this behavior - // is required to support migration. - if insertID != 0 { - qr.InsertID = uint64(insertID) - } else { - insertID = int64(qr.InsertID) - } - return insertID, qr, nil -} From 26ad8f660a3b58df176676a038fd664cc5da4869 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 28 Nov 2023 17:28:12 +0530 Subject: [PATCH 10/15] refactor: moved additional fields to insertcommon Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/cached_size.go | 29 +- go/vt/vtgate/engine/insert.go | 369 +++--------------- go/vt/vtgate/engine/insert_common.go | 232 +++++++++++ go/vt/vtgate/engine/insert_select.go | 220 +---------- go/vt/vtgate/engine/insert_test.go | 19 +- .../planbuilder/operator_transformers.go | 38 +- 6 files changed, 332 insertions(+), 575 deletions(-) create mode 100644 go/vt/vtgate/engine/insert_common.go diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 3960ee8a21a..252403249a1 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -386,7 +386,7 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(160) + size += int64(112) } // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon size += cached.InsertCommon.CachedSize(true) @@ -413,13 +413,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } } } - // field ColVindexes []*vitess.io/vitess/go/vt/vtgate/vindexes.ColumnVindex - { - size += hack.RuntimeAllocSize(int64(cap(cached.ColVindexes)) * int64(8)) - for _, elem := range cached.ColVindexes { - size += elem.CachedSize(true) - } - } // field Prefix string size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) // field Mid vitess.io/vitess/go/vt/sqlparser.Values @@ -446,12 +439,19 @@ func (cached *InsertCommon) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(48) + size += int64(96) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) // field TableName string size += hack.RuntimeAllocSize(int64(len(cached.TableName))) + // field ColVindexes []*vitess.io/vitess/go/vt/vtgate/vindexes.ColumnVindex + { + size += hack.RuntimeAllocSize(int64(cap(cached.ColVindexes)) * int64(8)) + for _, elem := range cached.ColVindexes { + size += elem.CachedSize(true) + } + } return size } func (cached *InsertRows) CachedSize(alloc bool) int64 { @@ -490,12 +490,10 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(144) + size += int64(80) } // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon size += cached.InsertCommon.CachedSize(true) - // field Query string - size += hack.RuntimeAllocSize(int64(len(cached.Query))) // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows size += cached.InsertRows.CachedSize(true) // field VindexValueOffset [][]int @@ -507,13 +505,6 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } } } - // field ColVindexes []*vitess.io/vitess/go/vt/vtgate/vindexes.ColumnVindex - { - size += hack.RuntimeAllocSize(int64(cap(cached.ColVindexes)) * int64(8)) - for _, elem := range cached.ColVindexes { - size += elem.CachedSize(true) - } - } // field Prefix string size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) // field Suffix string diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 42a66afb971..76c206980f5 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -23,7 +23,6 @@ import ( "strconv" "strings" - "vitess.io/vitess/go/slice" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" @@ -38,75 +37,30 @@ import ( var _ Primitive = (*Insert)(nil) -type ( - InsertCommon struct { - // Opcode is the execution opcode. - Opcode InsertOpcode +// Insert represents the instructions to perform an insert operation. +type Insert struct { + *InsertCommon - // Keyspace specifies the keyspace to send the query to. - Keyspace *vindexes.Keyspace + // Query specifies the query to be executed. + // For InsertSharded plans, this value is unused, + // and Prefix, Mid and Suffix are used instead. + Query string - // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs - // for sharded cases. - Ignore bool + InsertRows *InsertRows - // TableName is the name of the table on which row will be inserted. - TableName string + // VindexValues specifies values for all the vindex columns. + // This is a three-dimensional data structure: + // Insert.Values[i] represents the values to be inserted for the i'th colvindex (i < len(Insert.Table.ColumnVindexes)) + // Insert.Values[i].Values[j] represents values for the j'th column of the given colVindex (j < len(colVindex[i].Columns) + // Insert.Values[i].Values[j].Values[k] represents the value pulled from row k for that column: (k < len(ins.rows)) + VindexValues [][][]evalengine.Expr - // Insert needs tx handling - txNeeded - } - - // Insert represents the instructions to perform an insert operation. - Insert struct { - *InsertCommon - - // Query specifies the query to be executed. - // For InsertSharded plans, this value is unused, - // and Prefix, Mid and Suffix are used instead. - Query string - - InsertRows *InsertRows - - // VindexValues specifies values for all the vindex columns. - // This is a three-dimensional data structure: - // Insert.Values[i] represents the values to be inserted for the i'th colvindex (i < len(Insert.Table.ColumnVindexes)) - // Insert.Values[i].Values[j] represents values for the j'th column of the given colVindex (j < len(colVindex[i].Columns) - // Insert.Values[i].Values[j].Values[k] represents the value pulled from row k for that column: (k < len(ins.rows)) - VindexValues [][][]evalengine.Expr - - // ColVindexes are the vindexes that will use the VindexValues - ColVindexes []*vindexes.ColumnVindex - - // Prefix, Mid and Suffix are for sharded insert plans. - Prefix string - Mid sqlparser.Values - Suffix string - - // Option to override the standard behavior and allow a multi-shard insert - // to use single round trip autocommit. - // - // This is a clear violation of the SQL semantics since it means the statement - // is not atomic in the presence of PK conflicts on one shard and not another. - // However, some application use cases would prefer that the statement partially - // succeed in order to get the performance benefits of autocommit. - MultiShardAutocommit bool - - // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query - QueryTimeout int - - // ForceNonStreaming is true when the insert table and select table are same. - // This will avoid locking by the select table. - ForceNonStreaming bool - - PreventAutoCommit bool - } - - ksID = []byte -) + // Prefix, Mid and Suffix are for sharded insert plans. + Prefix string + Mid sqlparser.Values + Suffix string -func (ins *Insert) Inputs() ([]Primitive, []map[string]any) { - return ins.InsertRows.Inputs() + noInputs } // NewQueryInsert creates an Insert with a query string. @@ -206,16 +160,6 @@ func (ins *Insert) RouteType() string { return insName[ins.Opcode] } -// GetKeyspaceName specifies the Keyspace that this primitive routes to. -func (ins *Insert) GetKeyspaceName() string { - return ins.Keyspace.Name -} - -// GetTableName specifies the table that this primitive routes to. -func (ins *Insert) GetTableName() string { - return ins.TableName -} - // TryExecute performs a non-streaming exec. func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, _ bool) (*sqltypes.Result, error) { ctx, cancelFunc := addQueryTimeout(ctx, vcursor, ins.QueryTimeout) @@ -227,8 +171,7 @@ func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map case InsertSharded: return ins.insertIntoShardedTableFromValues(ctx, vcursor, bindVars) default: - // Unreachable. - return nil, fmt.Errorf("unsupported query route: %v", ins) + return nil, vterrors.VT13001("unexpected query route: %v", ins.Opcode) } } @@ -241,40 +184,36 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa return callback(res) } -// GetFields fetches the field info. -func (ins *Insert) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unreachable code for %q", ins.Query) -} - func (ins *Insert) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - query := ins.Query - if ins.InsertRows.hasSelectInput() { - result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) - if err != nil { - return nil, err - } - if len(result.rows) == 0 { - return &sqltypes.Result{}, nil - } - query = ins.getInsertQueryFromSelectForUnsharded(result.rows, bindVars) + insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) + if err != nil { + return nil, err } - _, qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) - return qr, err -} + rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) + if err != nil { + return nil, err + } + if len(rss) != 1 { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) + } + if err = allowOnlyPrimary(rss...); err != nil { + return nil, err + } + qr, err := execShard(ctx, ins, vcursor, ins.Query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) + if err != nil { + return nil, err + } -func (ins *Insert) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { - var mids sqlparser.Values - for r, inputRow := range rows { - row := sqlparser.ValTuple{} - for c, value := range inputRow { - bvName := insertVarOffset(r, c) - bindVars[bvName] = sqltypes.ValueBindVariable(value) - row = append(row, sqlparser.NewArgument(bvName)) - } - mids = append(mids, row) + // If processGenerateFromValues generated new values, it supersedes + // any ids that MySQL might have generated. If both generated + // values, we don't return an error because this behavior + // is required to support migration. + if insertID != 0 { + qr.InsertID = uint64(insertID) } - return ins.Prefix + sqlparser.String(mids) + ins.Suffix + return qr, nil + } func (ins *Insert) insertIntoShardedTableFromValues( @@ -472,174 +411,11 @@ func (ins *Insert) getInsertQueriesFromValues( return rss, queries, nil } -// processPrimary maps the primary vindex values to the keyspace ids. -func (ins *Insert) processPrimary(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex) ([]ksID, error) { - destinations, err := vindexes.Map(ctx, colVindex.Vindex, vcursor, vindexColumnsKeys) - if err != nil { - return nil, err - } - - keyspaceIDs := make([]ksID, len(destinations)) - for i, destination := range destinations { - switch d := destination.(type) { - case key.DestinationKeyspaceID: - // This is a single keyspace id, we're good. - keyspaceIDs[i] = d - case key.DestinationNone: - // No valid keyspace id, we may return an error. - if !ins.Ignore { - return nil, fmt.Errorf("could not map %v to a keyspace id", vindexColumnsKeys[i]) - } - default: - return nil, fmt.Errorf("could not map %v to a unique keyspace id: %v", vindexColumnsKeys[i], destination) - } - } - - return keyspaceIDs, nil -} - -// processOwned creates vindex entries for the values of an owned column. -func (ins *Insert) processOwned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { - if !ins.Ignore { - return colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, vindexColumnsKeys, ksids, false /* ignoreMode */) - } - - // InsertIgnore - var createIndexes []int - var createKeys []sqltypes.Row - var createKsids []ksID - - for rowNum, rowColumnKeys := range vindexColumnsKeys { - if ksids[rowNum] == nil { - continue - } - createIndexes = append(createIndexes, rowNum) - createKeys = append(createKeys, rowColumnKeys) - createKsids = append(createKsids, ksids[rowNum]) - } - if createKeys == nil { - return nil - } - - err := colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, createKeys, createKsids, true) - if err != nil { - return err - } - // After creation, verify that the keys map to the keyspace ids. If not, remove - // those that don't map. - verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, createKeys, createKsids) - if err != nil { - return err - } - for i, v := range verified { - if !v { - ksids[createIndexes[i]] = nil - } - } - return nil -} - -// processUnowned either reverse maps or validates the values for an unowned column. -func (ins *Insert) processUnowned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { - var reverseIndexes []int - var reverseKsids []ksID - - var verifyIndexes []int - var verifyKeys []sqltypes.Row - var verifyKsids []ksID - - // Check if this VIndex is reversible or not. - reversibleVindex, isReversible := colVindex.Vindex.(vindexes.Reversible) - - for rowNum, rowColumnKeys := range vindexColumnsKeys { - // If we weren't able to determine a keyspace id from the primary VIndex, skip this row - if ksids[rowNum] == nil { - continue - } - - if rowColumnKeys[0].IsNull() { - // If the value of the column is `NULL`, but this is a reversible VIndex, - // we will try to generate the value from the keyspace id generated by the primary VIndex. - if isReversible { - reverseIndexes = append(reverseIndexes, rowNum) - reverseKsids = append(reverseKsids, ksids[rowNum]) - } - - // Otherwise, don't do anything. Whether `NULL` is a valid value for this column will be - // handled by MySQL. - } else { - // If a value for this column was specified, the keyspace id values from the - // secondary VIndex need to be verified against the keyspace id from the primary VIndex - verifyIndexes = append(verifyIndexes, rowNum) - verifyKeys = append(verifyKeys, rowColumnKeys) - verifyKsids = append(verifyKsids, ksids[rowNum]) - } - } - - // Reverse map values for secondary VIndex columns from the primary VIndex's keyspace id. - if reverseKsids != nil { - reverseKeys, err := reversibleVindex.ReverseMap(vcursor, reverseKsids) - if err != nil { - return err - } - - for i, reverseKey := range reverseKeys { - // Fill the first column with the reverse-mapped value. - vindexColumnsKeys[reverseIndexes[i]][0] = reverseKey - } - } - - // Verify that the keyspace ids generated by the primary and secondary VIndexes match - if verifyIndexes != nil { - // If values were supplied, we validate against keyspace id. - verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, verifyKeys, verifyKsids) - if err != nil { - return err - } - - var mismatchVindexKeys []sqltypes.Row - for i, v := range verified { - rowNum := verifyIndexes[i] - if !v { - if !ins.Ignore { - mismatchVindexKeys = append(mismatchVindexKeys, vindexColumnsKeys[rowNum]) - continue - } - - // Skip the whole row if this is a `INSERT IGNORE` or `INSERT ... ON DUPLICATE KEY ...` statement - // but the keyspace ids didn't match. - ksids[verifyIndexes[i]] = nil - } - } - - if mismatchVindexKeys != nil { - return fmt.Errorf("values %v for column %v does not map to keyspace ids", mismatchVindexKeys, colVindex.Columns) - } - } - - return nil -} - -// InsertVarName returns a name for the bind var for this column. This method is used by the planner and engine, -// to make sure they both produce the same names -func InsertVarName(col sqlparser.IdentifierCI, rowNum int) string { - return fmt.Sprintf("_%s_%d", col.CompliantName(), rowNum) -} - -func insertVarOffset(rowNum, colOffset int) string { - return fmt.Sprintf("_c%d_%d", rowNum, colOffset) -} - func (ins *Insert) description() PrimitiveDescription { - other := map[string]any{ - "Query": ins.Query, - "TableName": ins.GetTableName(), - "MultiShardAutocommit": ins.MultiShardAutocommit, - "QueryTimeout": ins.QueryTimeout, - "InsertIgnore": ins.Ignore, - "InputAsNonStreaming": ins.ForceNonStreaming, - "NoAutoCommit": ins.PreventAutoCommit, - } + other := ins.commonDesc() + other["Query"] = ins.Query + other["TableName"] = ins.GetTableName() + ins.InsertRows.describe(other) if len(ins.VindexValues) > 0 { valuesOffsets := map[string]string{} @@ -662,17 +438,6 @@ func (ins *Insert) description() PrimitiveDescription { other["VindexValues"] = valuesOffsets } - ins.InsertRows.describe(other) - - if len(ins.Mid) > 0 { - mids := slice.Map(ins.Mid, func(from sqlparser.ValTuple) string { - return sqlparser.String(from) - }) - shardQuery := fmt.Sprintf("%s%s%s", ins.Prefix, strings.Join(mids, ", "), ins.Suffix) - if shardQuery != ins.Query { - other["ShardedQuery"] = shardQuery - } - } return PrimitiveDescription{ OperatorType: "Insert", Keyspace: ins.Keyspace, @@ -682,36 +447,8 @@ func (ins *Insert) description() PrimitiveDescription { } } -func (ins *Insert) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (int64, *sqltypes.Result, error) { - insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) - if err != nil { - return 0, nil, err - } - - rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) - if err != nil { - return 0, nil, err - } - if len(rss) != 1 { - return 0, nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) - } - err = allowOnlyPrimary(rss...) - if err != nil { - return 0, nil, err - } - qr, err := execShard(ctx, ins, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) - if err != nil { - return 0, nil, err - } - - // If processGenerateFromValues generated new values, it supersedes - // any ids that MySQL might have generated. If both generated - // values, we don't return an error because this behavior - // is required to support migration. - if insertID != 0 { - qr.InsertID = uint64(insertID) - } else { - insertID = int64(qr.InsertID) - } - return insertID, qr, nil +// InsertVarName returns a name for the bind var for this column. This method is used by the planner and engine, +// to make sure they both produce the same names +func InsertVarName(col sqlparser.IdentifierCI, rowNum int) string { + return fmt.Sprintf("_%s_%d", col.CompliantName(), rowNum) } diff --git a/go/vt/vtgate/engine/insert_common.go b/go/vt/vtgate/engine/insert_common.go new file mode 100644 index 00000000000..fb793a78ffc --- /dev/null +++ b/go/vt/vtgate/engine/insert_common.go @@ -0,0 +1,232 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "context" + "fmt" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +type InsertCommon struct { + // Opcode is the execution opcode. + Opcode InsertOpcode + + // Keyspace specifies the keyspace to send the query to. + Keyspace *vindexes.Keyspace + + // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs + // for sharded cases. + Ignore bool + + // TableName is the name of the table on which row will be inserted. + TableName string + + // Option to override the standard behavior and allow a multi-shard insert + // to use single round trip autocommit. + // + // This is a clear violation of the SQL semantics since it means the statement + // is not atomic in the presence of PK conflicts on one shard and not another. + // However, some application use cases would prefer that the statement partially + // succeed in order to get the performance benefits of autocommit. + MultiShardAutocommit bool + + // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query + QueryTimeout int + + // ForceNonStreaming is true when the insert table and select table are same. + // This will avoid locking by the select table. + ForceNonStreaming bool + + PreventAutoCommit bool + + // ColVindexes are the vindexes that will use the VindexValues + ColVindexes []*vindexes.ColumnVindex + + // Insert needs tx handling + txNeeded +} + +type ksID = []byte + +// GetKeyspaceName specifies the Keyspace that this primitive routes to. +func (ic *InsertCommon) GetKeyspaceName() string { + return ic.Keyspace.Name +} + +// GetTableName specifies the table that this primitive routes to. +func (ic *InsertCommon) GetTableName() string { + return ic.TableName +} + +// GetFields fetches the field info. +func (ic *InsertCommon) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { + return nil, vterrors.VT13001("unexpected fields call for insert query") +} + +// processPrimary maps the primary vindex values to the keyspace ids. +func (ic *InsertCommon) processPrimary(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex) ([]ksID, error) { + destinations, err := vindexes.Map(ctx, colVindex.Vindex, vcursor, vindexColumnsKeys) + if err != nil { + return nil, err + } + + keyspaceIDs := make([]ksID, len(destinations)) + for i, destination := range destinations { + switch d := destination.(type) { + case key.DestinationKeyspaceID: + // This is a single keyspace id, we're good. + keyspaceIDs[i] = d + case key.DestinationNone: + // No valid keyspace id, we may return an error. + if !ic.Ignore { + return nil, fmt.Errorf("could not map %v to a keyspace id", vindexColumnsKeys[i]) + } + default: + return nil, fmt.Errorf("could not map %v to a unique keyspace id: %v", vindexColumnsKeys[i], destination) + } + } + + return keyspaceIDs, nil +} + +// processOwned creates vindex entries for the values of an owned column. +func (ic *InsertCommon) processOwned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { + if !ic.Ignore { + return colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, vindexColumnsKeys, ksids, false /* ignoreMode */) + } + + // InsertIgnore + var createIndexes []int + var createKeys []sqltypes.Row + var createKsids []ksID + + for rowNum, rowColumnKeys := range vindexColumnsKeys { + if ksids[rowNum] == nil { + continue + } + createIndexes = append(createIndexes, rowNum) + createKeys = append(createKeys, rowColumnKeys) + createKsids = append(createKsids, ksids[rowNum]) + } + if createKeys == nil { + return nil + } + + err := colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, createKeys, createKsids, true) + if err != nil { + return err + } + // After creation, verify that the keys map to the keyspace ids. If not, remove + // those that don't map. + verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, createKeys, createKsids) + if err != nil { + return err + } + for i, v := range verified { + if !v { + ksids[createIndexes[i]] = nil + } + } + return nil +} + +// processUnowned either reverse maps or validates the values for an unowned column. +func (ic *InsertCommon) processUnowned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { + var reverseIndexes []int + var reverseKsids []ksID + + var verifyIndexes []int + var verifyKeys []sqltypes.Row + var verifyKsids []ksID + + // Check if this VIndex is reversible or not. + reversibleVindex, isReversible := colVindex.Vindex.(vindexes.Reversible) + + for rowNum, rowColumnKeys := range vindexColumnsKeys { + // If we weren't able to determine a keyspace id from the primary VIndex, skip this row + if ksids[rowNum] == nil { + continue + } + + if rowColumnKeys[0].IsNull() { + // If the value of the column is `NULL`, but this is a reversible VIndex, + // we will try to generate the value from the keyspace id generated by the primary VIndex. + if isReversible { + reverseIndexes = append(reverseIndexes, rowNum) + reverseKsids = append(reverseKsids, ksids[rowNum]) + } + + // Otherwise, don't do anything. Whether `NULL` is a valid value for this column will be + // handled by MySQL. + } else { + // If a value for this column was specified, the keyspace id values from the + // secondary VIndex need to be verified against the keyspace id from the primary VIndex + verifyIndexes = append(verifyIndexes, rowNum) + verifyKeys = append(verifyKeys, rowColumnKeys) + verifyKsids = append(verifyKsids, ksids[rowNum]) + } + } + + // Reverse map values for secondary VIndex columns from the primary VIndex's keyspace id. + if reverseKsids != nil { + reverseKeys, err := reversibleVindex.ReverseMap(vcursor, reverseKsids) + if err != nil { + return err + } + + for i, reverseKey := range reverseKeys { + // Fill the first column with the reverse-mapped value. + vindexColumnsKeys[reverseIndexes[i]][0] = reverseKey + } + } + + // Verify that the keyspace ids generated by the primary and secondary VIndexes match + if verifyIndexes != nil { + // If values were supplied, we validate against keyspace id. + verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, verifyKeys, verifyKsids) + if err != nil { + return err + } + + var mismatchVindexKeys []sqltypes.Row + for i, v := range verified { + rowNum := verifyIndexes[i] + if !v { + if !ic.Ignore { + mismatchVindexKeys = append(mismatchVindexKeys, vindexColumnsKeys[rowNum]) + continue + } + + // Skip the whole row if this is a `INSERT IGNORE` or `INSERT ... ON DUPLICATE KEY ...` statement + // but the keyspace ids didn't match. + ksids[verifyIndexes[i]] = nil + } + } + + if mismatchVindexKeys != nil { + return fmt.Errorf("values %v for column %v does not map to keyspace ids", mismatchVindexKeys, colVindex.Columns) + } + } + + return nil +} diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index a6c1b546d2b..c393b66c4ab 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -40,44 +40,15 @@ type ( InsertSelect struct { *InsertCommon - // Query specifies the query to be executed. - // For InsertSharded plans, this value is unused, - // and Prefix, Mid and Suffix are used instead. - Query string - InsertRows *InsertRows // VindexValueOffset stores the offset for each column in the ColumnVindex // that will appear in the result set of the select query. VindexValueOffset [][]int - // ColVindexes are the vindexes that will use the VindexValues - ColVindexes []*vindexes.ColumnVindex - // Prefix, Mid and Suffix are for sharded insert plans. Prefix string Suffix string - - // Option to override the standard behavior and allow a multi-shard insert - // to use single round trip autocommit. - // - // This is a clear violation of the SQL semantics since it means the statement - // is not atomic in the presence of PK conflicts on one shard and not another. - // However, some application use cases would prefer that the statement partially - // succeed in order to get the performance benefits of autocommit. - MultiShardAutocommit bool - - // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query - QueryTimeout int - - // ForceNonStreaming is true when the insert table and select table are same. - // This will avoid locking by the select table. - ForceNonStreaming bool - - PreventAutoCommit bool - - // Insert needs tx handling - txNeeded } ) @@ -93,7 +64,6 @@ func NewInsertSelect( prefix string, suffix string, vv [][]int, - query string, ir *InsertRows, ) *InsertSelect { ins := &InsertSelect{ @@ -105,7 +75,6 @@ func NewInsertSelect( Prefix: prefix, Suffix: suffix, VindexValueOffset: vv, - Query: query, } if table != nil { ins.TableName = table.Name.String() @@ -124,16 +93,6 @@ func (ins *InsertSelect) RouteType() string { return "InsertSelect" } -// GetKeyspaceName specifies the Keyspace that this primitive routes to. -func (ins *InsertSelect) GetKeyspaceName() string { - return ins.Keyspace.Name -} - -// GetTableName specifies the table that this primitive routes to. -func (ins *InsertSelect) GetTableName() string { - return ins.TableName -} - // TryExecute performs a non-streaming exec. func (ins *InsertSelect) TryExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, _ bool) (*sqltypes.Result, error) { ctx, cancelFunc := addQueryTimeout(ctx, vcursor, ins.QueryTimeout) @@ -189,11 +148,6 @@ func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, return callback(output) } -// GetFields fetches the field info. -func (ins *InsertSelect) GetFields(context.Context, VCursor, map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unreachable code for %q", ins.Query) -} - func (ins *InsertSelect) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { irr, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) if err != nil { @@ -402,165 +356,9 @@ func (ins *InsertSelect) execInsertFromSelect(ctx context.Context, vcursor VCurs return qr, err } -// processPrimary maps the primary vindex values to the keyspace ids. -func (ins *InsertSelect) processPrimary(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex) ([]ksID, error) { - destinations, err := vindexes.Map(ctx, colVindex.Vindex, vcursor, vindexColumnsKeys) - if err != nil { - return nil, err - } - - keyspaceIDs := make([]ksID, len(destinations)) - for i, destination := range destinations { - switch d := destination.(type) { - case key.DestinationKeyspaceID: - // This is a single keyspace id, we're good. - keyspaceIDs[i] = d - case key.DestinationNone: - // No valid keyspace id, we may return an error. - if !ins.Ignore { - return nil, fmt.Errorf("could not map %v to a keyspace id", vindexColumnsKeys[i]) - } - default: - return nil, fmt.Errorf("could not map %v to a unique keyspace id: %v", vindexColumnsKeys[i], destination) - } - } - - return keyspaceIDs, nil -} - -// processOwned creates vindex entries for the values of an owned column. -func (ins *InsertSelect) processOwned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { - if !ins.Ignore { - return colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, vindexColumnsKeys, ksids, false /* ignoreMode */) - } - - // InsertIgnore - var createIndexes []int - var createKeys []sqltypes.Row - var createKsids []ksID - - for rowNum, rowColumnKeys := range vindexColumnsKeys { - if ksids[rowNum] == nil { - continue - } - createIndexes = append(createIndexes, rowNum) - createKeys = append(createKeys, rowColumnKeys) - createKsids = append(createKsids, ksids[rowNum]) - } - if createKeys == nil { - return nil - } - - err := colVindex.Vindex.(vindexes.Lookup).Create(ctx, vcursor, createKeys, createKsids, true) - if err != nil { - return err - } - // After creation, verify that the keys map to the keyspace ids. If not, remove - // those that don't map. - verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, createKeys, createKsids) - if err != nil { - return err - } - for i, v := range verified { - if !v { - ksids[createIndexes[i]] = nil - } - } - return nil -} - -// processUnowned either reverse maps or validates the values for an unowned column. -func (ins *InsertSelect) processUnowned(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex, ksids []ksID) error { - var reverseIndexes []int - var reverseKsids []ksID - - var verifyIndexes []int - var verifyKeys []sqltypes.Row - var verifyKsids []ksID - - // Check if this VIndex is reversible or not. - reversibleVindex, isReversible := colVindex.Vindex.(vindexes.Reversible) - - for rowNum, rowColumnKeys := range vindexColumnsKeys { - // If we weren't able to determine a keyspace id from the primary VIndex, skip this row - if ksids[rowNum] == nil { - continue - } - - if rowColumnKeys[0].IsNull() { - // If the value of the column is `NULL`, but this is a reversible VIndex, - // we will try to generate the value from the keyspace id generated by the primary VIndex. - if isReversible { - reverseIndexes = append(reverseIndexes, rowNum) - reverseKsids = append(reverseKsids, ksids[rowNum]) - } - - // Otherwise, don't do anything. Whether `NULL` is a valid value for this column will be - // handled by MySQL. - } else { - // If a value for this column was specified, the keyspace id values from the - // secondary VIndex need to be verified against the keyspace id from the primary VIndex - verifyIndexes = append(verifyIndexes, rowNum) - verifyKeys = append(verifyKeys, rowColumnKeys) - verifyKsids = append(verifyKsids, ksids[rowNum]) - } - } - - // Reverse map values for secondary VIndex columns from the primary VIndex's keyspace id. - if reverseKsids != nil { - reverseKeys, err := reversibleVindex.ReverseMap(vcursor, reverseKsids) - if err != nil { - return err - } - - for i, reverseKey := range reverseKeys { - // Fill the first column with the reverse-mapped value. - vindexColumnsKeys[reverseIndexes[i]][0] = reverseKey - } - } - - // Verify that the keyspace ids generated by the primary and secondary VIndexes match - if verifyIndexes != nil { - // If values were supplied, we validate against keyspace id. - verified, err := vindexes.Verify(ctx, colVindex.Vindex, vcursor, verifyKeys, verifyKsids) - if err != nil { - return err - } - - var mismatchVindexKeys []sqltypes.Row - for i, v := range verified { - rowNum := verifyIndexes[i] - if !v { - if !ins.Ignore { - mismatchVindexKeys = append(mismatchVindexKeys, vindexColumnsKeys[rowNum]) - continue - } - - // Skip the whole row if this is a `INSERT IGNORE` or `INSERT ... ON DUPLICATE KEY ...` statement - // but the keyspace ids didn't match. - ksids[verifyIndexes[i]] = nil - } - } - - if mismatchVindexKeys != nil { - return fmt.Errorf("values %v for column %v does not map to keyspace ids", mismatchVindexKeys, colVindex.Columns) - } - } - - return nil -} - func (ins *InsertSelect) description() PrimitiveDescription { - other := map[string]any{ - "Query": ins.Query, - "TableName": ins.GetTableName(), - "MultiShardAutocommit": ins.MultiShardAutocommit, - "QueryTimeout": ins.QueryTimeout, - "InsertIgnore": ins.Ignore, - "InputAsNonStreaming": ins.ForceNonStreaming, - "NoAutoCommit": ins.PreventAutoCommit, - } - + other := ins.commonDesc() + other["TableName"] = ins.GetTableName() ins.InsertRows.describe(other) if len(ins.VindexValueOffset) > 0 { @@ -584,3 +382,17 @@ func (ins *InsertSelect) description() PrimitiveDescription { Other: other, } } + +func (ic *InsertCommon) commonDesc() map[string]any { + return map[string]any{ + "MultiShardAutocommit": ic.MultiShardAutocommit, + "QueryTimeout": ic.QueryTimeout, + "InsertIgnore": ic.Ignore, + "InputAsNonStreaming": ic.ForceNonStreaming, + "NoAutoCommit": ic.PreventAutoCommit, + } +} + +func insertVarOffset(rowNum, colOffset int) string { + return fmt.Sprintf("_c%d_%d", rowNum, colOffset) +} diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 5ca461b743e..6b574ccf070 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -1539,16 +1539,7 @@ func TestInsertSelectSimple(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( - false, - ks.Keyspace, - ks.Tables["t1"], - "prefix ", - " suffix", - [][]int{{1}}, - "dummy_insert", - NewInsertRowsFromSelect(nil, rb), - ) + ins := NewInsertSelect(false, ks.Keyspace, ks.Tables["t1"], "prefix ", " suffix", [][]int{{1}}, NewInsertRowsFromSelect(nil, rb)) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1640,7 +1631,6 @@ func TestInsertSelectOwned(t *testing.T) { [][]int{ {1}, // The primary vindex has a single column as sharding key {0}}, // the onecol vindex uses the 'name' column - "dummy_insert", NewInsertRowsFromSelect(nil, rb), ) @@ -1749,7 +1739,6 @@ func TestInsertSelectGenerate(t *testing.T) { "prefix ", " suffix", [][]int{{1}}, // The primary vindex has a single column as sharding key - "dummy_insert", NewInsertRowsFromSelect(gen, rb), ) @@ -1843,7 +1832,6 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { " suffix", [][]int{ {1}}, // The primary vindex has a single column as sharding key - "dummy_insert", NewInsertRowsFromSelect(generate, rb), ) @@ -1939,7 +1927,6 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { "prefix ", " suffix", [][]int{{1}}, // The primary vindex has a single column as sharding key, - "dummy_insert", NewInsertRowsFromSelect(generate, rb), ) @@ -2027,7 +2014,6 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { "prefix ", " suffix", [][]int{{1}}, // The primary vindex has a single column as sharding key, - "dummy_insert", NewInsertRowsFromSelect(generate, rb), ) @@ -2117,7 +2103,6 @@ func TestInsertSelectUnowned(t *testing.T) { "prefix ", " suffix", [][]int{{0}}, // // the onecol vindex as unowned lookup sharding column - "dummy_insert", NewInsertRowsFromSelect(nil, rb), ) @@ -2232,7 +2217,6 @@ func TestInsertSelectShardingCases(t *testing.T) { "prefix ", " suffix", [][]int{{0}}, - "dummy_insert", NewInsertRowsFromSelect(nil, sRoute), ) @@ -2310,7 +2294,6 @@ func TestInsertSelectShardingCases(t *testing.T) { "prefix ", " suffix", nil, - "dummy_insert", NewInsertRowsFromSelect(nil, sRoute), ) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index c11082511e2..7329a77c486 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -112,13 +112,13 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators ins := dmlOp.(*operators.Insert) eins := &engine.InsertSelect{ InsertCommon: &engine.InsertCommon{ - Keyspace: rb.Routing.Keyspace(), - TableName: ins.VTable.Name.String(), - Ignore: ins.Ignore, + Keyspace: rb.Routing.Keyspace(), + TableName: ins.VTable.Name.String(), + Ignore: ins.Ignore, + ForceNonStreaming: op.ForceNonStreaming, + ColVindexes: ins.ColVindexes, }, InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), - ForceNonStreaming: op.ForceNonStreaming, - ColVindexes: ins.ColVindexes, VindexValueOffset: ins.VindexValueOffset, } lp := &insert{eInsertSelect: eins} @@ -548,15 +548,22 @@ func buildInsertLogicalPlan( hints *queryHints, ) (logicalPlan, error) { ins := op.(*operators.Insert) + + ic := &engine.InsertCommon{ + Opcode: mapToInsertOpCode(rb.Routing.OpCode()), + Keyspace: rb.Routing.Keyspace(), + TableName: ins.VTable.Name.String(), + Ignore: ins.Ignore, + ColVindexes: ins.ColVindexes, + } + if hints != nil { + ic.MultiShardAutocommit = hints.multiShardAutocommit + ic.QueryTimeout = hints.queryTimeout + } + eins := &engine.Insert{ - InsertCommon: &engine.InsertCommon{ - Opcode: mapToInsertOpCode(rb.Routing.OpCode(), false), - Keyspace: rb.Routing.Keyspace(), - TableName: ins.VTable.Name.String(), - Ignore: ins.Ignore, - }, + InsertCommon: ic, InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), - ColVindexes: ins.ColVindexes, VindexValues: ins.VindexValues, } lp := &insert{eInsert: eins} @@ -567,16 +574,11 @@ func buildInsertLogicalPlan( eins.Prefix, eins.Mid, eins.Suffix = generateInsertShardedQuery(ins.AST) } - if hints != nil { - eins.MultiShardAutocommit = hints.multiShardAutocommit - eins.QueryTimeout = hints.queryTimeout - } - eins.Query = generateQuery(stmt) return lp, nil } -func mapToInsertOpCode(code engine.Opcode, insertSelect bool) engine.InsertOpcode { +func mapToInsertOpCode(code engine.Opcode) engine.InsertOpcode { if code == engine.Unsharded { return engine.InsertUnsharded } From fe026278b1357a0112d934310f39f71180e7b721 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 28 Nov 2023 18:38:44 +0530 Subject: [PATCH 11/15] refactor: removed InsertRows as it is redundant after engine split Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/cached_size.go | 40 +-- go/vt/vtgate/engine/insert.go | 11 +- go/vt/vtgate/engine/insert_common.go | 145 ++++++++++ go/vt/vtgate/engine/insert_rows.go | 273 ------------------ go/vt/vtgate/engine/insert_select.go | 88 +++++- go/vt/vtgate/engine/insert_test.go | 93 +++--- go/vt/vtgate/planbuilder/insert.go | 3 +- .../planbuilder/operator_transformers.go | 4 +- 8 files changed, 280 insertions(+), 377 deletions(-) delete mode 100644 go/vt/vtgate/engine/insert_rows.go diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 252403249a1..eb2dc2211d8 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -392,8 +392,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { size += cached.InsertCommon.CachedSize(true) // field Query string size += hack.RuntimeAllocSize(int64(len(cached.Query))) - // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows - size += cached.InsertRows.CachedSize(true) // field VindexValues [][][]vitess.io/vitess/go/vt/vtgate/evalengine.Expr { size += hack.RuntimeAllocSize(int64(cap(cached.VindexValues)) * int64(24)) @@ -445,6 +443,8 @@ func (cached *InsertCommon) CachedSize(alloc bool) int64 { size += cached.Keyspace.CachedSize(true) // field TableName string size += hack.RuntimeAllocSize(int64(len(cached.TableName))) + // field Generate *vitess.io/vitess/go/vt/vtgate/engine.Generate + size += cached.Generate.CachedSize(true) // field ColVindexes []*vitess.io/vitess/go/vt/vtgate/vindexes.ColumnVindex { size += hack.RuntimeAllocSize(int64(cap(cached.ColVindexes)) * int64(8)) @@ -454,36 +454,6 @@ func (cached *InsertCommon) CachedSize(alloc bool) int64 { } return size } -func (cached *InsertRows) CachedSize(alloc bool) int64 { - if cached == nil { - return int64(0) - } - size := int64(0) - if alloc { - size += int64(48) - } - // field Generate *vitess.io/vitess/go/vt/vtgate/engine.Generate - size += cached.Generate.CachedSize(true) - // field RowsFromValues vitess.io/vitess/go/vt/sqlparser.Values - { - size += hack.RuntimeAllocSize(int64(cap(cached.RowsFromValues)) * int64(24)) - for _, elem := range cached.RowsFromValues { - { - size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(16)) - for _, elem := range elem { - if cc, ok := elem.(cachedObject); ok { - size += cc.CachedSize(true) - } - } - } - } - } - // field RowsFromSelect vitess.io/vitess/go/vt/vtgate/engine.Primitive - if cc, ok := cached.RowsFromSelect.(cachedObject); ok { - size += cc.CachedSize(true) - } - return size -} func (cached *InsertSelect) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) @@ -494,8 +464,10 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon size += cached.InsertCommon.CachedSize(true) - // field InsertRows *vitess.io/vitess/go/vt/vtgate/engine.InsertRows - size += cached.InsertRows.CachedSize(true) + // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive + if cc, ok := cached.Input.(cachedObject); ok { + size += cc.CachedSize(true) + } // field VindexValueOffset [][]int { size += hack.RuntimeAllocSize(int64(cap(cached.VindexValueOffset)) * int64(24)) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 76c206980f5..e4bca087b08 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -46,8 +46,6 @@ type Insert struct { // and Prefix, Mid and Suffix are used instead. Query string - InsertRows *InsertRows - // VindexValues specifies values for all the vindex columns. // This is a three-dimensional data structure: // Insert.Values[i] represents the values to be inserted for the i'th colvindex (i < len(Insert.Table.ColumnVindexes)) @@ -70,8 +68,7 @@ func NewQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query stri Opcode: opcode, Keyspace: keyspace, }, - Query: query, - InsertRows: NewInsertRows(nil), + Query: query, } } @@ -92,7 +89,6 @@ func NewInsert( Keyspace: keyspace, Ignore: ignore, }, - InsertRows: NewInsertRows(nil), VindexValues: vindexValues, Prefix: prefix, Mid: mid, @@ -185,7 +181,7 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa } func (ins *Insert) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) + insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -221,7 +217,7 @@ func (ins *Insert) insertIntoShardedTableFromValues( vcursor VCursor, bindVars map[string]*querypb.BindVariable, ) (*sqltypes.Result, error) { - insertID, err := ins.InsertRows.processGenerateFromValues(ctx, vcursor, bindVars) + insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -415,7 +411,6 @@ func (ins *Insert) description() PrimitiveDescription { other := ins.commonDesc() other["Query"] = ins.Query other["TableName"] = ins.GetTableName() - ins.InsertRows.describe(other) if len(ins.VindexValues) > 0 { valuesOffsets := map[string]string{} diff --git a/go/vt/vtgate/engine/insert_common.go b/go/vt/vtgate/engine/insert_common.go index fb793a78ffc..f92432ed341 100644 --- a/go/vt/vtgate/engine/insert_common.go +++ b/go/vt/vtgate/engine/insert_common.go @@ -19,11 +19,14 @@ package engine import ( "context" "fmt" + "strconv" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -59,6 +62,9 @@ type InsertCommon struct { PreventAutoCommit bool + // Generate is only set for inserts where a sequence must be generated. + Generate *Generate + // ColVindexes are the vindexes that will use the VindexValues ColVindexes []*vindexes.ColumnVindex @@ -230,3 +236,142 @@ func (ic *InsertCommon) processUnowned(ctx context.Context, vcursor VCursor, vin return nil } + +// processGenerateFromSelect generates new values using a sequence if necessary. +// If no value was generated, it returns 0. Values are generated only +// for cases where none are supplied. +func (ic *InsertCommon) processGenerateFromSelect( + ctx context.Context, + vcursor VCursor, + rows []sqltypes.Row, +) (insertID int64, err error) { + if ic.Generate == nil { + return 0, nil + } + var count int64 + offset := ic.Generate.Offset + genColPresent := offset < len(rows[0]) + if genColPresent { + for _, row := range rows { + if shouldGenerate(row[offset], evalengine.ParseSQLMode(vcursor.SQLMode())) { + count++ + } + } + } else { + count = int64(len(rows)) + } + + if count == 0 { + return 0, nil + } + + insertID, err = ic.execGenerate(ctx, vcursor, count) + if err != nil { + return 0, err + } + + used := insertID + for idx, val := range rows { + if genColPresent { + if shouldGenerate(val[offset], evalengine.ParseSQLMode(vcursor.SQLMode())) { + val[offset] = sqltypes.NewInt64(used) + used++ + } + } else { + rows[idx] = append(val, sqltypes.NewInt64(used)) + used++ + } + } + + return insertID, nil +} + +// processGenerateFromValues generates new values using a sequence if necessary. +// If no value was generated, it returns 0. Values are generated only +// for cases where none are supplied. +func (ic *InsertCommon) processGenerateFromValues( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, +) (insertID int64, err error) { + if ic.Generate == nil { + return 0, nil + } + + // Scan input values to compute the number of values to generate, and + // keep track of where they should be filled. + env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) + resolved, err := env.Evaluate(ic.Generate.Values) + if err != nil { + return 0, err + } + count := int64(0) + values := resolved.TupleValues() + for _, val := range values { + if shouldGenerate(val, evalengine.ParseSQLMode(vcursor.SQLMode())) { + count++ + } + } + + // If generation is needed, generate the requested number of values (as one call). + if count != 0 { + insertID, err = ic.execGenerate(ctx, vcursor, count) + if err != nil { + return 0, err + } + } + + // Fill the holes where no value was supplied. + cur := insertID + for i, v := range values { + if shouldGenerate(v, evalengine.ParseSQLMode(vcursor.SQLMode())) { + bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.Int64BindVariable(cur) + cur++ + } else { + bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.ValueBindVariable(v) + } + } + return insertID, nil +} + +const nextValBV = "n" + +func (ic *InsertCommon) execGenerate(ctx context.Context, vcursor VCursor, count int64) (int64, error) { + // If generation is needed, generate the requested number of values (as one call). + rss, _, err := vcursor.ResolveDestinations(ctx, ic.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) + if err != nil { + return 0, err + } + if len(rss) != 1 { + return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) + } + bindVars := map[string]*querypb.BindVariable{nextValBV: sqltypes.Int64BindVariable(count)} + qr, err := vcursor.ExecuteStandalone(ctx, nil, ic.Generate.Query, bindVars, rss[0]) + if err != nil { + return 0, err + } + // If no rows are returned, it's an internal error, and the code + // must panic, which will be caught and reported. + return qr.Rows[0][0].ToCastInt64() +} + +// shouldGenerate determines if a sequence value should be generated for a given value +func shouldGenerate(v sqltypes.Value, sqlmode evalengine.SQLMode) bool { + if v.IsNull() { + return true + } + + // Unless the NO_AUTO_VALUE_ON_ZERO sql mode is active in mysql, it also + // treats 0 as a value that should generate a new sequence. + value, err := evalengine.CoerceTo(v, sqltypes.Uint64, sqlmode) + if err != nil { + return false + } + + id, err := value.ToCastUint64() + if err != nil { + return false + } + + return id == 0 +} diff --git a/go/vt/vtgate/engine/insert_rows.go b/go/vt/vtgate/engine/insert_rows.go deleted file mode 100644 index f7302b96caf..00000000000 --- a/go/vt/vtgate/engine/insert_rows.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright 2023 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package engine - -import ( - "context" - "fmt" - "strconv" - "sync" - - "vitess.io/vitess/go/vt/vtgate/evalengine" - - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/key" - querypb "vitess.io/vitess/go/vt/proto/query" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vterrors" -) - -const nextValBV = "n" - -type InsertRows struct { - // Generate is only set for inserts where a sequence must be generated. - Generate *Generate - - RowsFromValues sqlparser.Values - - // Input is a select query plan to retrieve results for inserting data. - RowsFromSelect Primitive -} - -func NewInsertRowsFromSelect(generate *Generate, rowsFromSelect Primitive) *InsertRows { - return &InsertRows{Generate: generate, RowsFromSelect: rowsFromSelect} -} - -func NewInsertRows(generate *Generate) *InsertRows { - return &InsertRows{Generate: generate} -} - -type insertRowsResult struct { - rows []sqltypes.Row - insertID int64 -} - -func (ir *InsertRows) Inputs() ([]Primitive, []map[string]any) { - if ir == nil || ir.RowsFromSelect == nil { - return nil, nil - } - return []Primitive{ir.RowsFromSelect}, nil -} - -func (ir *InsertRows) hasSelectInput() bool { - return ir != nil && ir.RowsFromSelect != nil -} - -func (ir *InsertRows) execSelect( - ctx context.Context, - vcursor VCursor, - bindVars map[string]*querypb.BindVariable, -) (insertRowsResult, error) { - // run the SELECT query - if ir.RowsFromSelect == nil { - return insertRowsResult{}, vterrors.VT13001("something went wrong planning INSERT SELECT") - } - - res, err := vcursor.ExecutePrimitive(ctx, ir.RowsFromSelect, bindVars, false) - if err != nil || len(res.Rows) == 0 { - return insertRowsResult{}, err - } - - insertID, err := ir.processGenerateFromSelect(ctx, vcursor, res.Rows) - if err != nil { - return insertRowsResult{}, err - } - - return insertRowsResult{ - rows: res.Rows, - insertID: insertID, - }, nil -} - -func (ir *InsertRows) execSelectStreaming( - ctx context.Context, - vcursor VCursor, - bindVars map[string]*querypb.BindVariable, - callback func(irr insertRowsResult) error, -) error { - var mu sync.Mutex - return vcursor.StreamExecutePrimitiveStandalone(ctx, ir.RowsFromSelect, bindVars, false, func(result *sqltypes.Result) error { - if len(result.Rows) == 0 { - return nil - } - - // should process only one chunk at a time. - // as parallel chunk insert will try to use the same transaction in the vttablet - // this will cause transaction in use error. - mu.Lock() - defer mu.Unlock() - - insertID, err := ir.processGenerateFromSelect(ctx, vcursor, result.Rows) - if err != nil { - return err - } - - return callback(insertRowsResult{ - rows: result.Rows, - insertID: insertID, - }) - }) -} - -// processGenerateFromSelect generates new values using a sequence if necessary. -// If no value was generated, it returns 0. Values are generated only -// for cases where none are supplied. -func (ir *InsertRows) processGenerateFromSelect( - ctx context.Context, - vcursor VCursor, - rows []sqltypes.Row, -) (insertID int64, err error) { - if ir.Generate == nil { - return 0, nil - } - var count int64 - offset := ir.Generate.Offset - genColPresent := offset < len(rows[0]) - if genColPresent { - for _, row := range rows { - if shouldGenerate(row[offset], evalengine.ParseSQLMode(vcursor.SQLMode())) { - count++ - } - } - } else { - count = int64(len(rows)) - } - - if count == 0 { - return 0, nil - } - - insertID, err = ir.execGenerate(ctx, vcursor, count) - if err != nil { - return 0, err - } - - used := insertID - for idx, val := range rows { - if genColPresent { - if shouldGenerate(val[offset], evalengine.ParseSQLMode(vcursor.SQLMode())) { - val[offset] = sqltypes.NewInt64(used) - used++ - } - } else { - rows[idx] = append(val, sqltypes.NewInt64(used)) - used++ - } - } - - return insertID, nil -} - -// processGenerateFromValues generates new values using a sequence if necessary. -// If no value was generated, it returns 0. Values are generated only -// for cases where none are supplied. -func (ir *InsertRows) processGenerateFromValues( - ctx context.Context, - vcursor VCursor, - bindVars map[string]*querypb.BindVariable, -) (insertID int64, err error) { - if ir.Generate == nil { - return 0, nil - } - - // Scan input values to compute the number of values to generate, and - // keep track of where they should be filled. - env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) - resolved, err := env.Evaluate(ir.Generate.Values) - if err != nil { - return 0, err - } - count := int64(0) - values := resolved.TupleValues() - for _, val := range values { - if shouldGenerate(val, evalengine.ParseSQLMode(vcursor.SQLMode())) { - count++ - } - } - - // If generation is needed, generate the requested number of values (as one call). - if count != 0 { - insertID, err = ir.execGenerate(ctx, vcursor, count) - if err != nil { - return 0, err - } - } - - // Fill the holes where no value was supplied. - cur := insertID - for i, v := range values { - if shouldGenerate(v, evalengine.ParseSQLMode(vcursor.SQLMode())) { - bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.Int64BindVariable(cur) - cur++ - } else { - bindVars[SeqVarName+strconv.Itoa(i)] = sqltypes.ValueBindVariable(v) - } - } - return insertID, nil -} - -func (ir *InsertRows) execGenerate(ctx context.Context, vcursor VCursor, count int64) (int64, error) { - // If generation is needed, generate the requested number of values (as one call). - rss, _, err := vcursor.ResolveDestinations(ctx, ir.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) - if err != nil { - return 0, err - } - if len(rss) != 1 { - return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) - } - bindVars := map[string]*querypb.BindVariable{nextValBV: sqltypes.Int64BindVariable(count)} - qr, err := vcursor.ExecuteStandalone(ctx, nil, ir.Generate.Query, bindVars, rss[0]) - if err != nil { - return 0, err - } - // If no rows are returned, it's an internal error, and the code - // must panic, which will be caught and reported. - return qr.Rows[0][0].ToCastInt64() -} - -// shouldGenerate determines if a sequence value should be generated for a given value -func shouldGenerate(v sqltypes.Value, sqlmode evalengine.SQLMode) bool { - if v.IsNull() { - return true - } - - // Unless the NO_AUTO_VALUE_ON_ZERO sql mode is active in mysql, it also - // treats 0 as a value that should generate a new sequence. - value, err := evalengine.CoerceTo(v, sqltypes.Uint64, sqlmode) - if err != nil { - return false - } - - id, err := value.ToCastUint64() - if err != nil { - return false - } - - return id == 0 -} - -func (ir *InsertRows) describe(other map[string]any) { - if ir == nil || ir.Generate == nil { - return - } - if ir.Generate.Values == nil { - other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ir.Generate.Query, ir.Generate.Offset) - } else { - other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ir.Generate.Query, sqlparser.String(ir.Generate.Values)) - } -} diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index c393b66c4ab..02d0ca10632 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "strconv" + "sync" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" @@ -36,24 +37,25 @@ import ( var _ Primitive = (*InsertSelect)(nil) type ( - // InsertSelect represents the instructions to perform an insert operation. + // InsertSelect represents the instructions to perform an insert operation with input rows from a select. InsertSelect struct { *InsertCommon - InsertRows *InsertRows + // Input is a select query plan to retrieve results for inserting data. + Input Primitive // VindexValueOffset stores the offset for each column in the ColumnVindex // that will appear in the result set of the select query. VindexValueOffset [][]int - // Prefix, Mid and Suffix are for sharded insert plans. + // Prefix, Suffix are for sharded insert plans. Prefix string Suffix string } ) func (ins *InsertSelect) Inputs() ([]Primitive, []map[string]any) { - return ins.InsertRows.Inputs() + return []Primitive{ins.Input}, nil } // NewInsertSelect creates a new InsertSelect. @@ -64,14 +66,14 @@ func NewInsertSelect( prefix string, suffix string, vv [][]int, - ir *InsertRows, + input Primitive, ) *InsertSelect { ins := &InsertSelect{ InsertCommon: &InsertCommon{ Ignore: ignore, Keyspace: keyspace, }, - InsertRows: ir, + Input: input, Prefix: prefix, Suffix: suffix, VindexValueOffset: vv, @@ -118,7 +120,7 @@ func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, sharded := ins.Keyspace.Sharded output := &sqltypes.Result{} - err := ins.InsertRows.execSelectStreaming(ctx, vcursor, bindVars, func(irr insertRowsResult) error { + err := ins.execSelectStreaming(ctx, vcursor, bindVars, func(irr insertRowsResult) error { if len(irr.rows) == 0 { return nil } @@ -149,7 +151,7 @@ func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, } func (ins *InsertSelect) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - irr, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) + irr, err := ins.execSelect(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -344,7 +346,7 @@ func (ins *InsertSelect) getInsertSelectQueries( } func (ins *InsertSelect) execInsertFromSelect(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - result, err := ins.InsertRows.execSelect(ctx, vcursor, bindVars) + result, err := ins.execSelect(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -359,7 +361,6 @@ func (ins *InsertSelect) execInsertFromSelect(ctx context.Context, vcursor VCurs func (ins *InsertSelect) description() PrimitiveDescription { other := ins.commonDesc() other["TableName"] = ins.GetTableName() - ins.InsertRows.describe(other) if len(ins.VindexValueOffset) > 0 { valuesOffsets := map[string]string{} @@ -384,15 +385,80 @@ func (ins *InsertSelect) description() PrimitiveDescription { } func (ic *InsertCommon) commonDesc() map[string]any { - return map[string]any{ + other := map[string]any{ "MultiShardAutocommit": ic.MultiShardAutocommit, "QueryTimeout": ic.QueryTimeout, "InsertIgnore": ic.Ignore, "InputAsNonStreaming": ic.ForceNonStreaming, "NoAutoCommit": ic.PreventAutoCommit, } + + if ic.Generate != nil { + if ic.Generate.Values == nil { + other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ic.Generate.Query, ic.Generate.Offset) + } else { + other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ic.Generate.Query, sqlparser.String(ic.Generate.Values)) + } + } + return other } func insertVarOffset(rowNum, colOffset int) string { return fmt.Sprintf("_c%d_%d", rowNum, colOffset) } + +type insertRowsResult struct { + rows []sqltypes.Row + insertID int64 +} + +func (ins *InsertSelect) execSelect( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, +) (insertRowsResult, error) { + res, err := vcursor.ExecutePrimitive(ctx, ins.Input, bindVars, false) + if err != nil || len(res.Rows) == 0 { + return insertRowsResult{}, err + } + + insertID, err := ins.processGenerateFromSelect(ctx, vcursor, res.Rows) + if err != nil { + return insertRowsResult{}, err + } + + return insertRowsResult{ + rows: res.Rows, + insertID: insertID, + }, nil +} + +func (ins *InsertSelect) execSelectStreaming( + ctx context.Context, + vcursor VCursor, + bindVars map[string]*querypb.BindVariable, + callback func(irr insertRowsResult) error, +) error { + var mu sync.Mutex + return vcursor.StreamExecutePrimitiveStandalone(ctx, ins.Input, bindVars, false, func(result *sqltypes.Result) error { + if len(result.Rows) == 0 { + return nil + } + + // should process only one chunk at a time. + // as parallel chunk insert will try to use the same transaction in the vttablet + // this will cause transaction in use error. + mu.Lock() + defer mu.Unlock() + + insertID, err := ins.processGenerateFromSelect(ctx, vcursor, result.Rows) + if err != nil { + return err + } + + return callback(insertRowsResult{ + rows: result.Rows, + insertID: insertID, + }) + }) +} diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 6b574ccf070..103408b0c20 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -76,7 +76,7 @@ func TestInsertUnshardedGenerate(t *testing.T) { }, "dummy_insert", ) - ins.InsertRows.Generate = &Generate{ + ins.Generate = &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -129,7 +129,7 @@ func TestInsertUnshardedGenerate_Zeros(t *testing.T) { }, "dummy_insert", ) - ins.InsertRows.Generate = &Generate{ + ins.Generate = &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -417,7 +417,7 @@ func TestInsertShardedGenerate(t *testing.T) { " suffix", ) - ins.InsertRows.Generate = &Generate{ + ins.Generate = &Generate{ Keyspace: &vindexes.Keyspace{ Name: "ks2", Sharded: false, @@ -1539,7 +1539,7 @@ func TestInsertSelectSimple(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect(false, ks.Keyspace, ks.Tables["t1"], "prefix ", " suffix", [][]int{{1}}, NewInsertRowsFromSelect(nil, rb)) + ins := NewInsertSelect(false, ks.Keyspace, ks.Tables["t1"], "prefix ", " suffix", [][]int{{1}}, rb) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1631,7 +1631,7 @@ func TestInsertSelectOwned(t *testing.T) { [][]int{ {1}, // The primary vindex has a single column as sharding key {0}}, // the onecol vindex uses the 'name' column - NewInsertRowsFromSelect(nil, rb), + rb, ) vc := newDMLTestVCursor("-20", "20-") @@ -1723,15 +1723,6 @@ func TestInsertSelectGenerate(t *testing.T) { Opcode: Scatter, Keyspace: ks.Keyspace}} - gen := &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 1, - } - ins := NewInsertSelect( false, ks.Keyspace, @@ -1739,8 +1730,16 @@ func TestInsertSelectGenerate(t *testing.T) { "prefix ", " suffix", [][]int{{1}}, // The primary vindex has a single column as sharding key - NewInsertRowsFromSelect(gen, rb), + rb, ) + ins.Generate = &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 1, + } vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1809,14 +1808,6 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - generate := &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 1, - } rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -1832,8 +1823,16 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { " suffix", [][]int{ {1}}, // The primary vindex has a single column as sharding key - NewInsertRowsFromSelect(generate, rb), + rb, ) + ins.Generate = &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 1, + } vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1906,14 +1905,6 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - generate := &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 2, - } rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -1927,8 +1918,16 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { "prefix ", " suffix", [][]int{{1}}, // The primary vindex has a single column as sharding key, - NewInsertRowsFromSelect(generate, rb), + rb, ) + ins.Generate = &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 2, + } vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1993,14 +1992,6 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - generate := &Generate{ - Keyspace: &vindexes.Keyspace{ - Name: "ks2", - Sharded: false, - }, - Query: "dummy_generate", - Offset: 2, - } rb := &Route{ Query: "dummy_select", FieldQuery: "dummy_field_query", @@ -2014,8 +2005,16 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { "prefix ", " suffix", [][]int{{1}}, // The primary vindex has a single column as sharding key, - NewInsertRowsFromSelect(generate, rb), + rb, ) + ins.Generate = &Generate{ + Keyspace: &vindexes.Keyspace{ + Name: "ks2", + Sharded: false, + }, + Query: "dummy_generate", + Offset: 2, + } vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -2103,7 +2102,7 @@ func TestInsertSelectUnowned(t *testing.T) { "prefix ", " suffix", [][]int{{0}}, // // the onecol vindex as unowned lookup sharding column - NewInsertRowsFromSelect(nil, rb), + rb, ) vc := newDMLTestVCursor("-20", "20-") @@ -2217,7 +2216,7 @@ func TestInsertSelectShardingCases(t *testing.T) { "prefix ", " suffix", [][]int{{0}}, - NewInsertRowsFromSelect(nil, sRoute), + sRoute, ) vc := &loggingVCursor{ @@ -2258,7 +2257,7 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard sks1.-20: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // sks1 and uks2 - ins.InsertRows = NewInsertRowsFromSelect(nil, uRoute) + ins.Input = uRoute vc.Rewind() _, err = ins.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) @@ -2294,7 +2293,7 @@ func TestInsertSelectShardingCases(t *testing.T) { "prefix ", " suffix", nil, - NewInsertRowsFromSelect(nil, sRoute), + sRoute, ) vc.Rewind() @@ -2324,7 +2323,7 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard uks1.0: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // uks1 and uks2 - ins.InsertRows = NewInsertRowsFromSelect(nil, uRoute) + ins.Input = uRoute vc.Rewind() _, err = ins.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 4bf2266ec3b..52f3131e2a9 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -98,7 +98,6 @@ func insertUnshardedShortcut(stmt *sqlparser.Insert, ks *vindexes.Keyspace, tabl }, } eIns.Query = generateQuery(stmt) - eIns.InsertRows = engine.NewInsertRows(nil) return &insert{eInsert: eIns} } @@ -115,6 +114,6 @@ func (i *insert) Primitive() engine.Primitive { return i.eInsert } input := i.source.Primitive() - i.eInsertSelect.InsertRows.RowsFromSelect = input + i.eInsertSelect.Input = input return i.eInsertSelect } diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 7329a77c486..6afb5721988 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -116,9 +116,9 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators TableName: ins.VTable.Name.String(), Ignore: ins.Ignore, ForceNonStreaming: op.ForceNonStreaming, + Generate: autoIncGenerate(ins.AutoIncrement), ColVindexes: ins.ColVindexes, }, - InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), VindexValueOffset: ins.VindexValueOffset, } lp := &insert{eInsertSelect: eins} @@ -554,6 +554,7 @@ func buildInsertLogicalPlan( Keyspace: rb.Routing.Keyspace(), TableName: ins.VTable.Name.String(), Ignore: ins.Ignore, + Generate: autoIncGenerate(ins.AutoIncrement), ColVindexes: ins.ColVindexes, } if hints != nil { @@ -563,7 +564,6 @@ func buildInsertLogicalPlan( eins := &engine.Insert{ InsertCommon: ic, - InsertRows: engine.NewInsertRows(autoIncGenerate(ins.AutoIncrement)), VindexValues: ins.VindexValues, } lp := &insert{eInsert: eins} From c6dabf40c063c42c9b5ad2359a57127cecb8108d Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Wed, 29 Nov 2023 12:47:55 +0530 Subject: [PATCH 12/15] refactor: extract more common methods Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert.go | 191 ++++++++------------------- go/vt/vtgate/engine/insert_common.go | 162 ++++++++++++++++++----- go/vt/vtgate/engine/insert_select.go | 155 ++++++++-------------- 3 files changed, 239 insertions(+), 269 deletions(-) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index e4bca087b08..31ca635d8ec 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -18,7 +18,6 @@ package engine import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -61,96 +60,6 @@ type Insert struct { noInputs } -// NewQueryInsert creates an Insert with a query string. -func NewQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { - return &Insert{ - InsertCommon: &InsertCommon{ - Opcode: opcode, - Keyspace: keyspace, - }, - Query: query, - } -} - -// NewInsert creates a new Insert. -func NewInsert( - opcode InsertOpcode, - ignore bool, - keyspace *vindexes.Keyspace, - vindexValues [][][]evalengine.Expr, - table *vindexes.Table, - prefix string, - mid sqlparser.Values, - suffix string, -) *Insert { - ins := &Insert{ - InsertCommon: &InsertCommon{ - Opcode: opcode, - Keyspace: keyspace, - Ignore: ignore, - }, - VindexValues: vindexValues, - Prefix: prefix, - Mid: mid, - Suffix: suffix, - } - if table != nil { - ins.TableName = table.Name.String() - for _, colVindex := range table.ColumnVindexes { - if colVindex.IsPartialVindex() { - continue - } - ins.ColVindexes = append(ins.ColVindexes, colVindex) - } - } - return ins -} - -// Generate represents the instruction to generate -// a value from a sequence. -type Generate struct { - Keyspace *vindexes.Keyspace - Query string - // Values are the supplied values for the column, which - // will be stored as a list within the expression. New - // values will be generated based on how many were not - // supplied (NULL). - Values evalengine.Expr - // Insert using Select, offset for auto increment column - Offset int -} - -// InsertOpcode is a number representing the opcode -// for the Insert primitive. -type InsertOpcode int - -const ( - // InsertUnsharded is for routing an insert statement - // to an unsharded keyspace. - InsertUnsharded = InsertOpcode(iota) - // InsertSharded is for routing an insert statement - // to individual shards. Requires: A list of Values, one - // for each ColVindex. If the table has an Autoinc column, - // A Generate subplan must be created. - InsertSharded -) - -var insName = map[InsertOpcode]string{ - InsertUnsharded: "InsertUnsharded", - InsertSharded: "InsertSharded", -} - -// String returns the opcode -func (code InsertOpcode) String() string { - return strings.ReplaceAll(insName[code], "Insert", "") -} - -// MarshalJSON serializes the InsertOpcode as a JSON string. -// It's used for testing and diagnostics. -func (code InsertOpcode) MarshalJSON() ([]byte, error) { - return json.Marshal(insName[code]) -} - // RouteType returns a description of the query routing type used by the primitive func (ins *Insert) RouteType() string { return insName[ins.Opcode] @@ -163,9 +72,9 @@ func (ins *Insert) TryExecute(ctx context.Context, vcursor VCursor, bindVars map switch ins.Opcode { case InsertUnsharded: - return ins.execInsertUnsharded(ctx, vcursor, bindVars) + return ins.insertIntoUnshardedTable(ctx, vcursor, bindVars) case InsertSharded: - return ins.insertIntoShardedTableFromValues(ctx, vcursor, bindVars) + return ins.insertIntoShardedTable(ctx, vcursor, bindVars) default: return nil, vterrors.VT13001("unexpected query route: %v", ins.Opcode) } @@ -180,39 +89,16 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa return callback(res) } -func (ins *Insert) execInsertUnsharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { +func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) if err != nil { return nil, err } - rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) - if err != nil { - return nil, err - } - if len(rss) != 1 { - return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) - } - if err = allowOnlyPrimary(rss...); err != nil { - return nil, err - } - qr, err := execShard(ctx, ins, vcursor, ins.Query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) - if err != nil { - return nil, err - } - - // If processGenerateFromValues generated new values, it supersedes - // any ids that MySQL might have generated. If both generated - // values, we don't return an error because this behavior - // is required to support migration. - if insertID != 0 { - qr.InsertID = uint64(insertID) - } - return qr, nil - + return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, ins.Query, insertID) } -func (ins *Insert) insertIntoShardedTableFromValues( +func (ins *Insert) insertIntoShardedTable( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, @@ -221,7 +107,7 @@ func (ins *Insert) insertIntoShardedTableFromValues( if err != nil { return nil, err } - rss, queries, err := ins.getInsertQueriesFromValues(ctx, vcursor, bindVars) + rss, queries, err := ins.getInsertQueries(ctx, vcursor, bindVars) if err != nil { return nil, err } @@ -252,14 +138,14 @@ func (ins *Insert) executeInsertQueries( return result, nil } -// getInsertQueriesFromValues performs all the vindex related work +// getInsertQueries performs all the vindex related work // and returns a map of shard to queries. // Using the primary vindex, it computes the target keyspace ids. // For owned vindexes, it creates entries. // For unowned vindexes with no input values, it reverse maps. // For unowned vindexes with values, it validates. // If it's an IGNORE or ON DUPLICATE key insert, it drops unroutable rows. -func (ins *Insert) getInsertQueriesFromValues( +func (ins *Insert) getInsertQueries( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, @@ -314,24 +200,12 @@ func (ins *Insert) getInsertQueriesFromValues( if len(vindexRowsValues) == 0 || len(colVindexes) == 0 { return nil, nil, vterrors.NewErrorf(vtrpcpb.Code_FAILED_PRECONDITION, vterrors.RequiresPrimaryKey, vterrors.PrimaryVindexNotSet, ins.TableName) } - keyspaceIDs, err := ins.processPrimary(ctx, vcursor, vindexRowsValues[0], colVindexes[0]) + + keyspaceIDs, err := ins.processVindexes(ctx, vcursor, vindexRowsValues, colVindexes) if err != nil { return nil, nil, err } - for vIdx := 1; vIdx < len(colVindexes); vIdx++ { - colVindex := colVindexes[vIdx] - var err error - if colVindex.Owned { - err = ins.processOwned(ctx, vcursor, vindexRowsValues[vIdx], colVindex, keyspaceIDs) - } else { - err = ins.processUnowned(ctx, vcursor, vindexRowsValues[vIdx], colVindex, keyspaceIDs) - } - if err != nil { - return nil, nil, err - } - } - // Build 3-d bindvars. Skip rows with nil keyspace ids in case // we're executing an insert ignore. for vIdx, colVindex := range colVindexes { @@ -447,3 +321,48 @@ func (ins *Insert) description() PrimitiveDescription { func InsertVarName(col sqlparser.IdentifierCI, rowNum int) string { return fmt.Sprintf("_%s_%d", col.CompliantName(), rowNum) } + +// NewQueryInsert creates an Insert with a query string. +func NewQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { + return &Insert{ + InsertCommon: &InsertCommon{ + Opcode: opcode, + Keyspace: keyspace, + }, + Query: query, + } +} + +// NewInsert creates a new Insert. +func NewInsert( + opcode InsertOpcode, + ignore bool, + keyspace *vindexes.Keyspace, + vindexValues [][][]evalengine.Expr, + table *vindexes.Table, + prefix string, + mid sqlparser.Values, + suffix string, +) *Insert { + ins := &Insert{ + InsertCommon: &InsertCommon{ + Opcode: opcode, + Keyspace: keyspace, + Ignore: ignore, + }, + VindexValues: vindexValues, + Prefix: prefix, + Mid: mid, + Suffix: suffix, + } + if table != nil { + ins.TableName = table.Name.String() + for _, colVindex := range table.ColumnVindexes { + if colVindex.IsPartialVindex() { + continue + } + ins.ColVindexes = append(ins.ColVindexes, colVindex) + } + } + return ins +} diff --git a/go/vt/vtgate/engine/insert_common.go b/go/vt/vtgate/engine/insert_common.go index f92432ed341..1de57135c7d 100644 --- a/go/vt/vtgate/engine/insert_common.go +++ b/go/vt/vtgate/engine/insert_common.go @@ -18,8 +18,10 @@ package engine import ( "context" + "encoding/json" "fmt" "strconv" + "strings" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" @@ -30,49 +32,98 @@ import ( "vitess.io/vitess/go/vt/vtgate/vindexes" ) -type InsertCommon struct { - // Opcode is the execution opcode. - Opcode InsertOpcode +type ( + InsertCommon struct { + // Opcode is the execution opcode. + Opcode InsertOpcode - // Keyspace specifies the keyspace to send the query to. - Keyspace *vindexes.Keyspace + // Keyspace specifies the keyspace to send the query to. + Keyspace *vindexes.Keyspace - // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs - // for sharded cases. - Ignore bool + // Ignore is for INSERT IGNORE and INSERT...ON DUPLICATE KEY constructs + // for sharded cases. + Ignore bool - // TableName is the name of the table on which row will be inserted. - TableName string + // TableName is the name of the table on which row will be inserted. + TableName string - // Option to override the standard behavior and allow a multi-shard insert - // to use single round trip autocommit. - // - // This is a clear violation of the SQL semantics since it means the statement - // is not atomic in the presence of PK conflicts on one shard and not another. - // However, some application use cases would prefer that the statement partially - // succeed in order to get the performance benefits of autocommit. - MultiShardAutocommit bool + // Option to override the standard behavior and allow a multi-shard insert + // to use single round trip autocommit. + // + // This is a clear violation of the SQL semantics since it means the statement + // is not atomic in the presence of PK conflicts on one shard and not another. + // However, some application use cases would prefer that the statement partially + // succeed in order to get the performance benefits of autocommit. + MultiShardAutocommit bool - // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query - QueryTimeout int + // QueryTimeout contains the optional timeout (in milliseconds) to apply to this query + QueryTimeout int - // ForceNonStreaming is true when the insert table and select table are same. - // This will avoid locking by the select table. - ForceNonStreaming bool + // ForceNonStreaming is true when the insert table and select table are same. + // This will avoid locking by the select table. + ForceNonStreaming bool - PreventAutoCommit bool + PreventAutoCommit bool - // Generate is only set for inserts where a sequence must be generated. - Generate *Generate + // Generate is only set for inserts where a sequence must be generated. + Generate *Generate - // ColVindexes are the vindexes that will use the VindexValues - ColVindexes []*vindexes.ColumnVindex + // ColVindexes are the vindexes that will use the VindexValues + ColVindexes []*vindexes.ColumnVindex - // Insert needs tx handling - txNeeded + // Insert needs tx handling + txNeeded + } + + ksID = []byte + + // Generate represents the instruction to generate + // a value from a sequence. + Generate struct { + Keyspace *vindexes.Keyspace + Query string + // Values are the supplied values for the column, which + // will be stored as a list within the expression. New + // values will be generated based on how many were not + // supplied (NULL). + Values evalengine.Expr + // Insert using Select, offset for auto increment column + Offset int + } + + // InsertOpcode is a number representing the opcode + // for the Insert primitive. + InsertOpcode int +) + +const nextValBV = "n" + +const ( + // InsertUnsharded is for routing an insert statement + // to an unsharded keyspace. + InsertUnsharded = InsertOpcode(iota) + // InsertSharded is for routing an insert statement + // to individual shards. Requires: A list of Values, one + // for each ColVindex. If the table has an Autoinc column, + // A Generate subplan must be created. + InsertSharded +) + +var insName = map[InsertOpcode]string{ + InsertUnsharded: "InsertUnsharded", + InsertSharded: "InsertSharded", } -type ksID = []byte +// String returns the opcode +func (code InsertOpcode) String() string { + return strings.ReplaceAll(insName[code], "Insert", "") +} + +// MarshalJSON serializes the InsertOpcode as a JSON string. +// It's used for testing and diagnostics. +func (code InsertOpcode) MarshalJSON() ([]byte, error) { + return json.Marshal(insName[code]) +} // GetKeyspaceName specifies the Keyspace that this primitive routes to. func (ic *InsertCommon) GetKeyspaceName() string { @@ -89,6 +140,53 @@ func (ic *InsertCommon) GetFields(context.Context, VCursor, map[string]*querypb. return nil, vterrors.VT13001("unexpected fields call for insert query") } +func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string, insertID int64) (*sqltypes.Result, error) { + rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) + if err != nil { + return nil, err + } + if len(rss) != 1 { + return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) + } + err = allowOnlyPrimary(rss...) + if err != nil { + return nil, err + } + qr, err := execShard(ctx, nil, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) + if err != nil { + return nil, err + } + + // If processGenerateFromValues generated new values, it supersedes + // any ids that MySQL might have generated. If both generated + // values, we don't return an error because this behavior + // is required to support migration. + if insertID != 0 { + qr.InsertID = uint64(insertID) + } + return qr, nil +} + +func (ins *InsertCommon) processVindexes(ctx context.Context, vcursor VCursor, vindexRowsValues [][]sqltypes.Row, colVindexes []*vindexes.ColumnVindex) ([]ksID, error) { + keyspaceIDs, err := ins.processPrimary(ctx, vcursor, vindexRowsValues[0], colVindexes[0]) + if err != nil { + return nil, err + } + + for vIdx := 1; vIdx < len(colVindexes); vIdx++ { + colVindex := colVindexes[vIdx] + if colVindex.Owned { + err = ins.processOwned(ctx, vcursor, vindexRowsValues[vIdx], colVindex, keyspaceIDs) + } else { + err = ins.processUnowned(ctx, vcursor, vindexRowsValues[vIdx], colVindex, keyspaceIDs) + } + if err != nil { + return nil, err + } + } + return keyspaceIDs, nil +} + // processPrimary maps the primary vindex values to the keyspace ids. func (ic *InsertCommon) processPrimary(ctx context.Context, vcursor VCursor, vindexColumnsKeys []sqltypes.Row, colVindex *vindexes.ColumnVindex) ([]ksID, error) { destinations, err := vindexes.Map(ctx, colVindex.Vindex, vcursor, vindexColumnsKeys) @@ -334,8 +432,6 @@ func (ic *InsertCommon) processGenerateFromValues( return insertID, nil } -const nextValBV = "n" - func (ic *InsertCommon) execGenerate(ctx context.Context, vcursor VCursor, count int64) (int64, error) { // If generation is needed, generate the requested number of values (as one call). rss, _, err := vcursor.ResolveDestinations(ctx, ic.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index 02d0ca10632..198ecce67d6 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -58,38 +58,6 @@ func (ins *InsertSelect) Inputs() ([]Primitive, []map[string]any) { return []Primitive{ins.Input}, nil } -// NewInsertSelect creates a new InsertSelect. -func NewInsertSelect( - ignore bool, - keyspace *vindexes.Keyspace, - table *vindexes.Table, - prefix string, - suffix string, - vv [][]int, - input Primitive, -) *InsertSelect { - ins := &InsertSelect{ - InsertCommon: &InsertCommon{ - Ignore: ignore, - Keyspace: keyspace, - }, - Input: input, - Prefix: prefix, - Suffix: suffix, - VindexValueOffset: vv, - } - if table != nil { - ins.TableName = table.Name.String() - for _, colVindex := range table.ColumnVindexes { - if colVindex.IsPartialVindex() { - continue - } - ins.ColVindexes = append(ins.ColVindexes, colVindex) - } - } - return ins -} - // RouteType returns a description of the query routing type used by the primitive func (ins *InsertSelect) RouteType() string { return "InsertSelect" @@ -101,7 +69,7 @@ func (ins *InsertSelect) TryExecute(ctx context.Context, vcursor VCursor, bindVa defer cancelFunc() if ins.Keyspace.Sharded { - return ins.execInsertFromSelect(ctx, vcursor, bindVars) + return ins.execInsertSharded(ctx, vcursor, bindVars) } return ins.execInsertUnsharded(ctx, vcursor, bindVars) } @@ -125,13 +93,12 @@ func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, return nil } - var insertID int64 var qr *sqltypes.Result var err error if sharded { - insertID, qr, err = ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, irr) + qr, err = ins.insertIntoShardedTable(ctx, vcursor, bindVars, irr) } else { - insertID, qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr) + qr, err = ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr) } if err != nil { return err @@ -139,8 +106,8 @@ func (ins *InsertSelect) TryStreamExecute(ctx context.Context, vcursor VCursor, output.RowsAffected += qr.RowsAffected // InsertID needs to be updated to the least insertID value in sqltypes.Result - if output.InsertID == 0 || output.InsertID > uint64(insertID) { - output.InsertID = uint64(insertID) + if output.InsertID == 0 || output.InsertID > qr.InsertID { + output.InsertID = qr.InsertID } return nil }) @@ -158,30 +125,15 @@ func (ins *InsertSelect) execInsertUnsharded(ctx context.Context, vcursor VCurso if len(irr.rows) == 0 { return &sqltypes.Result{}, nil } - _, qr, err := ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr) - return qr, err + return ins.insertIntoUnshardedTable(ctx, vcursor, bindVars, irr) } -func (ins *InsertSelect) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, irr insertRowsResult) (int64, *sqltypes.Result, error) { - query := ins.getInsertQueryFromSelectForUnsharded(irr.rows, bindVars) - qr, err := ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query) - if err != nil { - return 0, nil, err - } - - // If processGenerateFromValues generated new values, it supersedes - // any ids that MySQL might have generated. If both generated - // values, we don't return an error because this behavior - // is required to support migration. - if irr.insertID != 0 { - qr.InsertID = uint64(irr.insertID) - } else { - irr.insertID = int64(qr.InsertID) - } - return irr.insertID, qr, nil +func (ins *InsertSelect) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, irr insertRowsResult) (*sqltypes.Result, error) { + query := ins.getInsertUnshardedQuery(irr.rows, bindVars) + return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query, irr.insertID) } -func (ins *InsertSelect) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { +func (ins *InsertSelect) getInsertUnshardedQuery(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { var mids sqlparser.Values for r, inputRow := range rows { row := sqlparser.ValTuple{} @@ -195,38 +147,23 @@ func (ins *InsertSelect) getInsertQueryFromSelectForUnsharded(rows []sqltypes.Ro return ins.Prefix + sqlparser.String(mids) + ins.Suffix } -func (ins *InsertSelect) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string) (*sqltypes.Result, error) { - rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) - if err != nil { - return nil, err - } - if len(rss) != 1 { - return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "Keyspace does not have exactly one shard: %v", rss) - } - err = allowOnlyPrimary(rss...) - if err != nil { - return nil, err - } - return execShard(ctx, ins, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) -} - -func (ins *InsertSelect) insertIntoShardedTableFromSelect( +func (ins *InsertSelect) insertIntoShardedTable( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, irr insertRowsResult, -) (int64, *sqltypes.Result, error) { - rss, queries, err := ins.getInsertSelectQueries(ctx, vcursor, bindVars, irr.rows) +) (*sqltypes.Result, error) { + rss, queries, err := ins.getInsertQueries(ctx, vcursor, bindVars, irr.rows) if err != nil { - return 0, nil, err + return nil, err } qr, err := ins.executeInsertQueries(ctx, vcursor, rss, queries, irr.insertID) if err != nil { - return 0, nil, err + return nil, err } qr.InsertID = uint64(irr.insertID) - return irr.insertID, qr, nil + return qr, nil } func (ins *InsertSelect) executeInsertQueries( @@ -252,7 +189,7 @@ func (ins *InsertSelect) executeInsertQueries( return result, nil } -func (ins *InsertSelect) getInsertSelectQueries( +func (ins *InsertSelect) getInsertQueries( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, @@ -264,7 +201,7 @@ func (ins *InsertSelect) getInsertSelectQueries( } // Here we go over the incoming rows and extract values for the vindexes we need to update - shardingCols := make([][]sqltypes.Row, len(colVindexes)) + vindexRowsValues := make([][]sqltypes.Row, len(colVindexes)) for _, inputRow := range rows { for colIdx := range colVindexes { offsets := ins.VindexValueOffset[colIdx] @@ -276,28 +213,15 @@ func (ins *InsertSelect) getInsertSelectQueries( } row = append(row, inputRow[offset]) } - shardingCols[colIdx] = append(shardingCols[colIdx], row) + vindexRowsValues[colIdx] = append(vindexRowsValues[colIdx], row) } } - keyspaceIDs, err := ins.processPrimary(ctx, vcursor, shardingCols[0], colVindexes[0]) + keyspaceIDs, err := ins.processVindexes(ctx, vcursor, vindexRowsValues, colVindexes) if err != nil { return nil, nil, err } - for vIdx := 1; vIdx < len(colVindexes); vIdx++ { - colVindex := colVindexes[vIdx] - var err error - if colVindex.Owned { - err = ins.processOwned(ctx, vcursor, shardingCols[vIdx], colVindex, keyspaceIDs) - } else { - err = ins.processUnowned(ctx, vcursor, shardingCols[vIdx], colVindex, keyspaceIDs) - } - if err != nil { - return nil, nil, err - } - } - var indexes []*querypb.Value var destinations []key.Destination for i, ksid := range keyspaceIDs { @@ -345,7 +269,7 @@ func (ins *InsertSelect) getInsertSelectQueries( return rss, queries, nil } -func (ins *InsertSelect) execInsertFromSelect(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { +func (ins *InsertSelect) execInsertSharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { result, err := ins.execSelect(ctx, vcursor, bindVars) if err != nil { return nil, err @@ -354,8 +278,7 @@ func (ins *InsertSelect) execInsertFromSelect(ctx context.Context, vcursor VCurs return &sqltypes.Result{}, nil } - _, qr, err := ins.insertIntoShardedTableFromSelect(ctx, vcursor, bindVars, result) - return qr, err + return ins.insertIntoShardedTable(ctx, vcursor, bindVars, result) } func (ins *InsertSelect) description() PrimitiveDescription { @@ -447,7 +370,7 @@ func (ins *InsertSelect) execSelectStreaming( // should process only one chunk at a time. // as parallel chunk insert will try to use the same transaction in the vttablet - // this will cause transaction in use error. + // this will cause transaction in use error out with "transaction in use" error. mu.Lock() defer mu.Unlock() @@ -462,3 +385,35 @@ func (ins *InsertSelect) execSelectStreaming( }) }) } + +// NewInsertSelect creates a new InsertSelect. +func NewInsertSelect( + ignore bool, + keyspace *vindexes.Keyspace, + table *vindexes.Table, + prefix string, + suffix string, + vv [][]int, + input Primitive, +) *InsertSelect { + ins := &InsertSelect{ + InsertCommon: &InsertCommon{ + Ignore: ignore, + Keyspace: keyspace, + }, + Input: input, + Prefix: prefix, + Suffix: suffix, + VindexValueOffset: vv, + } + if table != nil { + ins.TableName = table.Name.String() + for _, colVindex := range table.ColumnVindexes { + if colVindex.IsPartialVindex() { + continue + } + ins.ColVindexes = append(ins.ColVindexes, colVindex) + } + } + return ins +} From 762453dfd424633c983f2c43533a584c3bfb98c1 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Wed, 29 Nov 2023 13:07:29 +0530 Subject: [PATCH 13/15] added calling primitive for vexplain log Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert.go | 6 +++--- go/vt/vtgate/engine/insert_common.go | 14 ++++++++------ go/vt/vtgate/engine/insert_select.go | 6 +++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 31ca635d8ec..92ae25c10d7 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -90,12 +90,12 @@ func (ins *Insert) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVa } func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { - insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) + insertID, err := ins.processGenerateFromValues(ctx, vcursor, ins, bindVars) if err != nil { return nil, err } - return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, ins.Query, insertID) + return ins.executeUnshardedTableQuery(ctx, vcursor, ins, bindVars, ins.Query, insertID) } func (ins *Insert) insertIntoShardedTable( @@ -103,7 +103,7 @@ func (ins *Insert) insertIntoShardedTable( vcursor VCursor, bindVars map[string]*querypb.BindVariable, ) (*sqltypes.Result, error) { - insertID, err := ins.processGenerateFromValues(ctx, vcursor, bindVars) + insertID, err := ins.processGenerateFromValues(ctx, vcursor, ins, bindVars) if err != nil { return nil, err } diff --git a/go/vt/vtgate/engine/insert_common.go b/go/vt/vtgate/engine/insert_common.go index 1de57135c7d..5374f777ea1 100644 --- a/go/vt/vtgate/engine/insert_common.go +++ b/go/vt/vtgate/engine/insert_common.go @@ -140,7 +140,7 @@ func (ic *InsertCommon) GetFields(context.Context, VCursor, map[string]*querypb. return nil, vterrors.VT13001("unexpected fields call for insert query") } -func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, query string, insertID int64) (*sqltypes.Result, error) { +func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, loggingPrimitive Primitive, bindVars map[string]*querypb.BindVariable, query string, insertID int64) (*sqltypes.Result, error) { rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) if err != nil { return nil, err @@ -152,7 +152,7 @@ func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor if err != nil { return nil, err } - qr, err := execShard(ctx, nil, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) + qr, err := execShard(ctx, loggingPrimitive, vcursor, query, bindVars, rss[0], true, !ins.PreventAutoCommit /* canAutocommit */) if err != nil { return nil, err } @@ -341,6 +341,7 @@ func (ic *InsertCommon) processUnowned(ctx context.Context, vcursor VCursor, vin func (ic *InsertCommon) processGenerateFromSelect( ctx context.Context, vcursor VCursor, + loggingPrimitive Primitive, rows []sqltypes.Row, ) (insertID int64, err error) { if ic.Generate == nil { @@ -363,7 +364,7 @@ func (ic *InsertCommon) processGenerateFromSelect( return 0, nil } - insertID, err = ic.execGenerate(ctx, vcursor, count) + insertID, err = ic.execGenerate(ctx, vcursor, loggingPrimitive, count) if err != nil { return 0, err } @@ -390,6 +391,7 @@ func (ic *InsertCommon) processGenerateFromSelect( func (ic *InsertCommon) processGenerateFromValues( ctx context.Context, vcursor VCursor, + loggingPrimitive Primitive, bindVars map[string]*querypb.BindVariable, ) (insertID int64, err error) { if ic.Generate == nil { @@ -413,7 +415,7 @@ func (ic *InsertCommon) processGenerateFromValues( // If generation is needed, generate the requested number of values (as one call). if count != 0 { - insertID, err = ic.execGenerate(ctx, vcursor, count) + insertID, err = ic.execGenerate(ctx, vcursor, loggingPrimitive, count) if err != nil { return 0, err } @@ -432,7 +434,7 @@ func (ic *InsertCommon) processGenerateFromValues( return insertID, nil } -func (ic *InsertCommon) execGenerate(ctx context.Context, vcursor VCursor, count int64) (int64, error) { +func (ic *InsertCommon) execGenerate(ctx context.Context, vcursor VCursor, loggingPrimitive Primitive, count int64) (int64, error) { // If generation is needed, generate the requested number of values (as one call). rss, _, err := vcursor.ResolveDestinations(ctx, ic.Generate.Keyspace.Name, nil, []key.Destination{key.DestinationAnyShard{}}) if err != nil { @@ -442,7 +444,7 @@ func (ic *InsertCommon) execGenerate(ctx context.Context, vcursor VCursor, count return 0, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "auto sequence generation can happen through single shard only, it is getting routed to %d shards", len(rss)) } bindVars := map[string]*querypb.BindVariable{nextValBV: sqltypes.Int64BindVariable(count)} - qr, err := vcursor.ExecuteStandalone(ctx, nil, ic.Generate.Query, bindVars, rss[0]) + qr, err := vcursor.ExecuteStandalone(ctx, loggingPrimitive, ic.Generate.Query, bindVars, rss[0]) if err != nil { return 0, err } diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index 198ecce67d6..d7919cb424e 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -130,7 +130,7 @@ func (ins *InsertSelect) execInsertUnsharded(ctx context.Context, vcursor VCurso func (ins *InsertSelect) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, irr insertRowsResult) (*sqltypes.Result, error) { query := ins.getInsertUnshardedQuery(irr.rows, bindVars) - return ins.executeUnshardedTableQuery(ctx, vcursor, bindVars, query, irr.insertID) + return ins.executeUnshardedTableQuery(ctx, vcursor, ins, bindVars, query, irr.insertID) } func (ins *InsertSelect) getInsertUnshardedQuery(rows []sqltypes.Row, bindVars map[string]*querypb.BindVariable) string { @@ -345,7 +345,7 @@ func (ins *InsertSelect) execSelect( return insertRowsResult{}, err } - insertID, err := ins.processGenerateFromSelect(ctx, vcursor, res.Rows) + insertID, err := ins.processGenerateFromSelect(ctx, vcursor, ins, res.Rows) if err != nil { return insertRowsResult{}, err } @@ -374,7 +374,7 @@ func (ins *InsertSelect) execSelectStreaming( mu.Lock() defer mu.Unlock() - insertID, err := ins.processGenerateFromSelect(ctx, vcursor, result.Rows) + insertID, err := ins.processGenerateFromSelect(ctx, vcursor, ins, result.Rows) if err != nil { return err } From 514cd984257a20ff1eb4f8d6144db1e33ba093e2 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Wed, 29 Nov 2023 15:37:56 +0530 Subject: [PATCH 14/15] addressed review comments Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/insert.go | 197 +++++++++--------- go/vt/vtgate/engine/insert_common.go | 11 +- go/vt/vtgate/engine/insert_select.go | 134 ++++++------ go/vt/vtgate/engine/insert_test.go | 54 ++--- go/vt/vtgate/planbuilder/insert.go | 2 +- .../planbuilder/operator_transformers.go | 4 +- 6 files changed, 209 insertions(+), 193 deletions(-) diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 92ae25c10d7..46043413732 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -38,7 +38,7 @@ var _ Primitive = (*Insert)(nil) // Insert represents the instructions to perform an insert operation. type Insert struct { - *InsertCommon + InsertCommon // Query specifies the query to be executed. // For InsertSharded plans, this value is unused, @@ -52,14 +52,57 @@ type Insert struct { // Insert.Values[i].Values[j].Values[k] represents the value pulled from row k for that column: (k < len(ins.rows)) VindexValues [][][]evalengine.Expr - // Prefix, Mid and Suffix are for sharded insert plans. - Prefix string - Mid sqlparser.Values - Suffix string + // Mid is the row values for the sharded insert plans. + Mid sqlparser.Values noInputs } +// newQueryInsert creates an Insert with a query string. +func newQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { + return &Insert{ + InsertCommon: InsertCommon{ + Opcode: opcode, + Keyspace: keyspace, + }, + Query: query, + } +} + +// newInsert creates a new Insert. +func newInsert( + opcode InsertOpcode, + ignore bool, + keyspace *vindexes.Keyspace, + vindexValues [][][]evalengine.Expr, + table *vindexes.Table, + prefix string, + mid sqlparser.Values, + suffix string, +) *Insert { + ins := &Insert{ + InsertCommon: InsertCommon{ + Opcode: opcode, + Keyspace: keyspace, + Ignore: ignore, + Prefix: prefix, + Suffix: suffix, + }, + VindexValues: vindexValues, + Mid: mid, + } + if table != nil { + ins.TableName = table.Name.String() + for _, colVindex := range table.ColumnVindexes { + if colVindex.IsPartialVindex() { + continue + } + ins.ColVindexes = append(ins.ColVindexes, colVindex) + } + } + return ins +} + // RouteType returns a description of the query routing type used by the primitive func (ins *Insert) RouteType() string { return insName[ins.Opcode] @@ -95,7 +138,7 @@ func (ins *Insert) insertIntoUnshardedTable(ctx context.Context, vcursor VCursor return nil, err } - return ins.executeUnshardedTableQuery(ctx, vcursor, ins, bindVars, ins.Query, insertID) + return ins.executeUnshardedTableQuery(ctx, vcursor, ins, bindVars, ins.Query, uint64(insertID)) } func (ins *Insert) insertIntoShardedTable( @@ -107,12 +150,12 @@ func (ins *Insert) insertIntoShardedTable( if err != nil { return nil, err } - rss, queries, err := ins.getInsertQueries(ctx, vcursor, bindVars) + rss, queries, err := ins.getInsertShardedQueries(ctx, vcursor, bindVars) if err != nil { return nil, err } - return ins.executeInsertQueries(ctx, vcursor, rss, queries, insertID) + return ins.executeInsertQueries(ctx, vcursor, rss, queries, uint64(insertID)) } func (ins *Insert) executeInsertQueries( @@ -120,7 +163,7 @@ func (ins *Insert) executeInsertQueries( vcursor VCursor, rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, - insertID int64, + insertID uint64, ) (*sqltypes.Result, error) { autocommit := (len(rss) == 1 || ins.MultiShardAutocommit) && vcursor.AutocommitApproval() err := allowOnlyPrimary(rss...) @@ -133,82 +176,51 @@ func (ins *Insert) executeInsertQueries( } if insertID != 0 { - result.InsertID = uint64(insertID) + result.InsertID = insertID } return result, nil } -// getInsertQueries performs all the vindex related work +// getInsertShardedQueries performs all the vindex related work // and returns a map of shard to queries. // Using the primary vindex, it computes the target keyspace ids. // For owned vindexes, it creates entries. // For unowned vindexes with no input values, it reverse maps. // For unowned vindexes with values, it validates. // If it's an IGNORE or ON DUPLICATE key insert, it drops unroutable rows. -func (ins *Insert) getInsertQueries( +func (ins *Insert) getInsertShardedQueries( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, ) ([]*srvtopo.ResolvedShard, []*querypb.BoundQuery, error) { + // vindexRowsValues builds the values of all vindex columns. // the 3-d structure indexes are colVindex, row, col. Note that // ins.Values indexes are colVindex, col, row. So, the conversion // involves a transpose. // The reason we need to transpose is that all the Vindex APIs // require inputs in that format. - vindexRowsValues := make([][]sqltypes.Row, len(ins.VindexValues)) - rowCount := 0 - env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) - colVindexes := ins.ColVindexes - for vIdx, vColValues := range ins.VindexValues { - if len(vColValues) != len(colVindexes[vIdx].Columns) { - return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] supplied vindex column values don't match vschema: %v %v", vColValues, colVindexes[vIdx].Columns) - } - for colIdx, colValues := range vColValues { - rowsResolvedValues := make(sqltypes.Row, 0, len(colValues)) - for _, colValue := range colValues { - result, err := env.Evaluate(colValue) - if err != nil { - return nil, nil, err - } - rowsResolvedValues = append(rowsResolvedValues, result.Value(vcursor.ConnCollation())) - } - // This is the first iteration: allocate for transpose. - if colIdx == 0 { - if len(rowsResolvedValues) == 0 { - return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] rowcount is zero for inserts: %v", rowsResolvedValues) - } - if rowCount == 0 { - rowCount = len(rowsResolvedValues) - } - if rowCount != len(rowsResolvedValues) { - return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] uneven row values for inserts: %d %d", rowCount, len(rowsResolvedValues)) - } - vindexRowsValues[vIdx] = make([]sqltypes.Row, rowCount) - } - // Perform the transpose. - for rowNum, colVal := range rowsResolvedValues { - vindexRowsValues[vIdx][rowNum] = append(vindexRowsValues[vIdx][rowNum], colVal) - } - } + vindexRowsValues, err := ins.buildVindexRowsValues(ctx, vcursor, bindVars) + if err != nil { + return nil, nil, err } // The output from the following 'process' functions is a list of // keyspace ids. For regular inserts, a failure to find a route // results in an error. For 'ignore' type inserts, the keyspace // id is returned as nil, which is used later to drop the corresponding rows. - if len(vindexRowsValues) == 0 || len(colVindexes) == 0 { + if len(vindexRowsValues) == 0 || len(ins.ColVindexes) == 0 { return nil, nil, vterrors.NewErrorf(vtrpcpb.Code_FAILED_PRECONDITION, vterrors.RequiresPrimaryKey, vterrors.PrimaryVindexNotSet, ins.TableName) } - keyspaceIDs, err := ins.processVindexes(ctx, vcursor, vindexRowsValues, colVindexes) + keyspaceIDs, err := ins.processVindexes(ctx, vcursor, vindexRowsValues) if err != nil { return nil, nil, err } // Build 3-d bindvars. Skip rows with nil keyspace ids in case // we're executing an insert ignore. - for vIdx, colVindex := range colVindexes { + for vIdx, colVindex := range ins.ColVindexes { for rowNum, rowColumnKeys := range vindexRowsValues[vIdx] { if keyspaceIDs[rowNum] == nil { // InsertIgnore: skip the row. @@ -281,6 +293,46 @@ func (ins *Insert) getInsertQueries( return rss, queries, nil } +func (ins *Insert) buildVindexRowsValues(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([][]sqltypes.Row, error) { + vindexRowsValues := make([][]sqltypes.Row, len(ins.VindexValues)) + rowCount := 0 + env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) + colVindexes := ins.ColVindexes + for vIdx, vColValues := range ins.VindexValues { + if len(vColValues) != len(colVindexes[vIdx].Columns) { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] supplied vindex column values don't match vschema: %v %v", vColValues, colVindexes[vIdx].Columns) + } + for colIdx, colValues := range vColValues { + rowsResolvedValues := make(sqltypes.Row, 0, len(colValues)) + for _, colValue := range colValues { + result, err := env.Evaluate(colValue) + if err != nil { + return nil, err + } + rowsResolvedValues = append(rowsResolvedValues, result.Value(vcursor.ConnCollation())) + } + // This is the first iteration: allocate for transpose. + if colIdx == 0 { + if len(rowsResolvedValues) == 0 { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] rowcount is zero for inserts: %v", rowsResolvedValues) + } + if rowCount == 0 { + rowCount = len(rowsResolvedValues) + } + if rowCount != len(rowsResolvedValues) { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] uneven row values for inserts: %d %d", rowCount, len(rowsResolvedValues)) + } + vindexRowsValues[vIdx] = make([]sqltypes.Row, rowCount) + } + // Perform the transpose. + for rowNum, colVal := range rowsResolvedValues { + vindexRowsValues[vIdx][rowNum] = append(vindexRowsValues[vIdx][rowNum], colVal) + } + } + } + return vindexRowsValues, nil +} + func (ins *Insert) description() PrimitiveDescription { other := ins.commonDesc() other["Query"] = ins.Query @@ -321,48 +373,3 @@ func (ins *Insert) description() PrimitiveDescription { func InsertVarName(col sqlparser.IdentifierCI, rowNum int) string { return fmt.Sprintf("_%s_%d", col.CompliantName(), rowNum) } - -// NewQueryInsert creates an Insert with a query string. -func NewQueryInsert(opcode InsertOpcode, keyspace *vindexes.Keyspace, query string) *Insert { - return &Insert{ - InsertCommon: &InsertCommon{ - Opcode: opcode, - Keyspace: keyspace, - }, - Query: query, - } -} - -// NewInsert creates a new Insert. -func NewInsert( - opcode InsertOpcode, - ignore bool, - keyspace *vindexes.Keyspace, - vindexValues [][][]evalengine.Expr, - table *vindexes.Table, - prefix string, - mid sqlparser.Values, - suffix string, -) *Insert { - ins := &Insert{ - InsertCommon: &InsertCommon{ - Opcode: opcode, - Keyspace: keyspace, - Ignore: ignore, - }, - VindexValues: vindexValues, - Prefix: prefix, - Mid: mid, - Suffix: suffix, - } - if table != nil { - ins.TableName = table.Name.String() - for _, colVindex := range table.ColumnVindexes { - if colVindex.IsPartialVindex() { - continue - } - ins.ColVindexes = append(ins.ColVindexes, colVindex) - } - } - return ins -} diff --git a/go/vt/vtgate/engine/insert_common.go b/go/vt/vtgate/engine/insert_common.go index 5374f777ea1..d0a14feec26 100644 --- a/go/vt/vtgate/engine/insert_common.go +++ b/go/vt/vtgate/engine/insert_common.go @@ -71,6 +71,10 @@ type ( // ColVindexes are the vindexes that will use the VindexValues ColVindexes []*vindexes.ColumnVindex + // Prefix, Suffix are for sharded insert plans. + Prefix string + Suffix string + // Insert needs tx handling txNeeded } @@ -140,7 +144,7 @@ func (ic *InsertCommon) GetFields(context.Context, VCursor, map[string]*querypb. return nil, vterrors.VT13001("unexpected fields call for insert query") } -func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, loggingPrimitive Primitive, bindVars map[string]*querypb.BindVariable, query string, insertID int64) (*sqltypes.Result, error) { +func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor VCursor, loggingPrimitive Primitive, bindVars map[string]*querypb.BindVariable, query string, insertID uint64) (*sqltypes.Result, error) { rss, _, err := vcursor.ResolveDestinations(ctx, ins.Keyspace.Name, nil, []key.Destination{key.DestinationAllShards{}}) if err != nil { return nil, err @@ -162,12 +166,13 @@ func (ins *InsertCommon) executeUnshardedTableQuery(ctx context.Context, vcursor // values, we don't return an error because this behavior // is required to support migration. if insertID != 0 { - qr.InsertID = uint64(insertID) + qr.InsertID = insertID } return qr, nil } -func (ins *InsertCommon) processVindexes(ctx context.Context, vcursor VCursor, vindexRowsValues [][]sqltypes.Row, colVindexes []*vindexes.ColumnVindex) ([]ksID, error) { +func (ins *InsertCommon) processVindexes(ctx context.Context, vcursor VCursor, vindexRowsValues [][]sqltypes.Row) ([]ksID, error) { + colVindexes := ins.ColVindexes keyspaceIDs, err := ins.processPrimary(ctx, vcursor, vindexRowsValues[0], colVindexes[0]) if err != nil { return nil, err diff --git a/go/vt/vtgate/engine/insert_select.go b/go/vt/vtgate/engine/insert_select.go index d7919cb424e..d36176922dc 100644 --- a/go/vt/vtgate/engine/insert_select.go +++ b/go/vt/vtgate/engine/insert_select.go @@ -39,7 +39,7 @@ var _ Primitive = (*InsertSelect)(nil) type ( // InsertSelect represents the instructions to perform an insert operation with input rows from a select. InsertSelect struct { - *InsertCommon + InsertCommon // Input is a select query plan to retrieve results for inserting data. Input Primitive @@ -47,13 +47,41 @@ type ( // VindexValueOffset stores the offset for each column in the ColumnVindex // that will appear in the result set of the select query. VindexValueOffset [][]int - - // Prefix, Suffix are for sharded insert plans. - Prefix string - Suffix string } ) +// newInsertSelect creates a new InsertSelect. +func newInsertSelect( + ignore bool, + keyspace *vindexes.Keyspace, + table *vindexes.Table, + prefix string, + suffix string, + vv [][]int, + input Primitive, +) *InsertSelect { + ins := &InsertSelect{ + InsertCommon: InsertCommon{ + Ignore: ignore, + Keyspace: keyspace, + Prefix: prefix, + Suffix: suffix, + }, + Input: input, + VindexValueOffset: vv, + } + if table != nil { + ins.TableName = table.Name.String() + for _, colVindex := range table.ColumnVindexes { + if colVindex.IsPartialVindex() { + continue + } + ins.ColVindexes = append(ins.ColVindexes, colVindex) + } + } + return ins +} + func (ins *InsertSelect) Inputs() ([]Primitive, []map[string]any) { return []Primitive{ins.Input}, nil } @@ -153,7 +181,7 @@ func (ins *InsertSelect) insertIntoShardedTable( bindVars map[string]*querypb.BindVariable, irr insertRowsResult, ) (*sqltypes.Result, error) { - rss, queries, err := ins.getInsertQueries(ctx, vcursor, bindVars, irr.rows) + rss, queries, err := ins.getInsertShardedQueries(ctx, vcursor, bindVars, irr.rows) if err != nil { return nil, err } @@ -171,7 +199,7 @@ func (ins *InsertSelect) executeInsertQueries( vcursor VCursor, rss []*srvtopo.ResolvedShard, queries []*querypb.BoundQuery, - insertID int64, + insertID uint64, ) (*sqltypes.Result, error) { autocommit := (len(rss) == 1 || ins.MultiShardAutocommit) && vcursor.AutocommitApproval() err := allowOnlyPrimary(rss...) @@ -184,40 +212,23 @@ func (ins *InsertSelect) executeInsertQueries( } if insertID != 0 { - result.InsertID = uint64(insertID) + result.InsertID = insertID } return result, nil } -func (ins *InsertSelect) getInsertQueries( +func (ins *InsertSelect) getInsertShardedQueries( ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, rows []sqltypes.Row, ) ([]*srvtopo.ResolvedShard, []*querypb.BoundQuery, error) { - colVindexes := ins.ColVindexes - if len(colVindexes) != len(ins.VindexValueOffset) { - return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "vindex value offsets and vindex info do not match") - } - - // Here we go over the incoming rows and extract values for the vindexes we need to update - vindexRowsValues := make([][]sqltypes.Row, len(colVindexes)) - for _, inputRow := range rows { - for colIdx := range colVindexes { - offsets := ins.VindexValueOffset[colIdx] - row := make(sqltypes.Row, 0, len(offsets)) - for _, offset := range offsets { - if offset == -1 { // value not provided from select query - row = append(row, sqltypes.NULL) - continue - } - row = append(row, inputRow[offset]) - } - vindexRowsValues[colIdx] = append(vindexRowsValues[colIdx], row) - } + vindexRowsValues, err := ins.buildVindexRowsValues(rows) + if err != nil { + return nil, nil, err } - keyspaceIDs, err := ins.processVindexes(ctx, vcursor, vindexRowsValues, colVindexes) + keyspaceIDs, err := ins.processVindexes(ctx, vcursor, vindexRowsValues) if err != nil { return nil, nil, err } @@ -269,6 +280,31 @@ func (ins *InsertSelect) getInsertQueries( return rss, queries, nil } +func (ins *InsertSelect) buildVindexRowsValues(rows []sqltypes.Row) ([][]sqltypes.Row, error) { + colVindexes := ins.ColVindexes + if len(colVindexes) != len(ins.VindexValueOffset) { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "vindex value offsets and vindex info do not match") + } + + // Here we go over the incoming rows and extract values for the vindexes we need to update + vindexRowsValues := make([][]sqltypes.Row, len(colVindexes)) + for _, inputRow := range rows { + for colIdx := range colVindexes { + offsets := ins.VindexValueOffset[colIdx] + row := make(sqltypes.Row, 0, len(offsets)) + for _, offset := range offsets { + if offset == -1 { // value not provided from select query + row = append(row, sqltypes.NULL) + continue + } + row = append(row, inputRow[offset]) + } + vindexRowsValues[colIdx] = append(vindexRowsValues[colIdx], row) + } + } + return vindexRowsValues, nil +} + func (ins *InsertSelect) execInsertSharded(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { result, err := ins.execSelect(ctx, vcursor, bindVars) if err != nil { @@ -332,7 +368,7 @@ func insertVarOffset(rowNum, colOffset int) string { type insertRowsResult struct { rows []sqltypes.Row - insertID int64 + insertID uint64 } func (ins *InsertSelect) execSelect( @@ -352,7 +388,7 @@ func (ins *InsertSelect) execSelect( return insertRowsResult{ rows: res.Rows, - insertID: insertID, + insertID: uint64(insertID), }, nil } @@ -381,39 +417,7 @@ func (ins *InsertSelect) execSelectStreaming( return callback(insertRowsResult{ rows: result.Rows, - insertID: insertID, + insertID: uint64(insertID), }) }) } - -// NewInsertSelect creates a new InsertSelect. -func NewInsertSelect( - ignore bool, - keyspace *vindexes.Keyspace, - table *vindexes.Table, - prefix string, - suffix string, - vv [][]int, - input Primitive, -) *InsertSelect { - ins := &InsertSelect{ - InsertCommon: &InsertCommon{ - Ignore: ignore, - Keyspace: keyspace, - }, - Input: input, - Prefix: prefix, - Suffix: suffix, - VindexValueOffset: vv, - } - if table != nil { - ins.TableName = table.Name.String() - for _, colVindex := range table.ColumnVindexes { - if colVindex.IsPartialVindex() { - continue - } - ins.ColVindexes = append(ins.ColVindexes, colVindex) - } - } - return ins -} diff --git a/go/vt/vtgate/engine/insert_test.go b/go/vt/vtgate/engine/insert_test.go index 103408b0c20..4ee8431f083 100644 --- a/go/vt/vtgate/engine/insert_test.go +++ b/go/vt/vtgate/engine/insert_test.go @@ -33,7 +33,7 @@ import ( ) func TestInsertUnsharded(t *testing.T) { - ins := NewQueryInsert( + ins := newQueryInsert( InsertUnsharded, &vindexes.Keyspace{ Name: "ks", @@ -68,7 +68,7 @@ func TestInsertUnsharded(t *testing.T) { } func TestInsertUnshardedGenerate(t *testing.T) { - ins := NewQueryInsert( + ins := newQueryInsert( InsertUnsharded, &vindexes.Keyspace{ Name: "ks", @@ -121,7 +121,7 @@ func TestInsertUnshardedGenerate(t *testing.T) { } func TestInsertUnshardedGenerate_Zeros(t *testing.T) { - ins := NewQueryInsert( + ins := newQueryInsert( InsertUnsharded, &vindexes.Keyspace{ Name: "ks", @@ -198,7 +198,7 @@ func TestInsertShardedSimple(t *testing.T) { ks := vs.Keyspaces["sharded"] // A single row insert should be autocommitted - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -232,7 +232,7 @@ func TestInsertShardedSimple(t *testing.T) { }) // Multiple rows are not autocommitted by default - ins = NewInsert( + ins = newInsert( InsertSharded, false, ks.Keyspace, @@ -272,7 +272,7 @@ func TestInsertShardedSimple(t *testing.T) { }) // Optional flag overrides autocommit - ins = NewInsert( + ins = newInsert( InsertSharded, false, ks.Keyspace, @@ -344,7 +344,7 @@ func TestInsertShardedFail(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -394,7 +394,7 @@ func TestInsertShardedGenerate(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -513,7 +513,7 @@ func TestInsertShardedOwned(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -623,7 +623,7 @@ func TestInsertShardedOwnedWithNull(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -700,7 +700,7 @@ func TestInsertShardedGeo(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -806,7 +806,7 @@ func TestInsertShardedIgnoreOwned(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, true, ks.Keyspace, @@ -962,7 +962,7 @@ func TestInsertShardedIgnoreOwnedWithNull(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, true, ks.Keyspace, @@ -1060,7 +1060,7 @@ func TestInsertShardedUnownedVerify(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -1188,7 +1188,7 @@ func TestInsertShardedIgnoreUnownedVerify(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, true, ks.Keyspace, @@ -1294,7 +1294,7 @@ func TestInsertShardedIgnoreUnownedVerifyFail(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -1371,7 +1371,7 @@ func TestInsertShardedUnownedReverseMap(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -1485,7 +1485,7 @@ func TestInsertShardedUnownedReverseMapSuccess(t *testing.T) { vs := vindexes.BuildVSchema(invschema) ks := vs.Keyspaces["sharded"] - ins := NewInsert( + ins := newInsert( InsertSharded, false, ks.Keyspace, @@ -1539,7 +1539,7 @@ func TestInsertSelectSimple(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect(false, ks.Keyspace, ks.Tables["t1"], "prefix ", " suffix", [][]int{{1}}, rb) + ins := newInsertSelect(false, ks.Keyspace, ks.Tables["t1"], "prefix ", " suffix", [][]int{{1}}, rb) vc := newDMLTestVCursor("-20", "20-") vc.shardForKsid = []string{"20-", "-20", "20-"} @@ -1622,7 +1622,7 @@ func TestInsertSelectOwned(t *testing.T) { Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( + ins := newInsertSelect( false, ks.Keyspace, ks.Tables["t1"], @@ -1723,7 +1723,7 @@ func TestInsertSelectGenerate(t *testing.T) { Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( + ins := newInsertSelect( false, ks.Keyspace, ks.Tables["t1"], @@ -1815,7 +1815,7 @@ func TestStreamingInsertSelectGenerate(t *testing.T) { Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( + ins := newInsertSelect( false, ks.Keyspace, ks.Tables["t1"], @@ -1911,7 +1911,7 @@ func TestInsertSelectGenerateNotProvided(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( + ins := newInsertSelect( false, ks.Keyspace, ks.Tables["t1"], @@ -1998,7 +1998,7 @@ func TestStreamingInsertSelectGenerateNotProvided(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( + ins := newInsertSelect( false, ks.Keyspace, ks.Tables["t1"], @@ -2095,7 +2095,7 @@ func TestInsertSelectUnowned(t *testing.T) { RoutingParameters: &RoutingParameters{ Opcode: Scatter, Keyspace: ks.Keyspace}} - ins := NewInsertSelect( + ins := newInsertSelect( false, ks.Keyspace, ks.Tables["t2"], @@ -2209,7 +2209,7 @@ func TestInsertSelectShardingCases(t *testing.T) { RoutingParameters: &RoutingParameters{Opcode: Unsharded, Keyspace: uks2.Keyspace}} // sks1 and sks2 - ins := NewInsertSelect( + ins := newInsertSelect( false, sks1.Keyspace, sks1.Tables["s1"], @@ -2286,7 +2286,7 @@ func TestInsertSelectShardingCases(t *testing.T) { `ExecuteMultiShard sks1.-20: prefix values (:_c0_0) suffix {_c0_0: type:INT64 value:"1"} true true`}) // uks1 and sks2 - ins = NewInsertSelect( + ins = newInsertSelect( false, uks1.Keyspace, nil, diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 52f3131e2a9..173c0213073 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -91,7 +91,7 @@ func errOutIfPlanCannotBeConstructed(ctx *plancontext.PlanningContext, vTbl *vin func insertUnshardedShortcut(stmt *sqlparser.Insert, ks *vindexes.Keyspace, tables []*vindexes.Table) logicalPlan { eIns := &engine.Insert{ - InsertCommon: &engine.InsertCommon{ + InsertCommon: engine.InsertCommon{ Opcode: engine.InsertUnsharded, Keyspace: ks, TableName: tables[0].Name.String(), diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 6afb5721988..5f965b55ad9 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -111,7 +111,7 @@ func transformInsertionSelection(ctx *plancontext.PlanningContext, op *operators ins := dmlOp.(*operators.Insert) eins := &engine.InsertSelect{ - InsertCommon: &engine.InsertCommon{ + InsertCommon: engine.InsertCommon{ Keyspace: rb.Routing.Keyspace(), TableName: ins.VTable.Name.String(), Ignore: ins.Ignore, @@ -549,7 +549,7 @@ func buildInsertLogicalPlan( ) (logicalPlan, error) { ins := op.(*operators.Insert) - ic := &engine.InsertCommon{ + ic := engine.InsertCommon{ Opcode: mapToInsertOpCode(rb.Routing.OpCode()), Keyspace: rb.Routing.Keyspace(), TableName: ins.VTable.Name.String(), From e50243a7b502445b1387ca928210a897c570af79 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Wed, 29 Nov 2023 16:16:47 +0530 Subject: [PATCH 15/15] sizegen update Signed-off-by: Harshit Gangal --- go/vt/vtgate/engine/cached_size.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index eb2dc2211d8..807c7604412 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -386,10 +386,10 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(112) + size += int64(192) } - // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon - size += cached.InsertCommon.CachedSize(true) + // field InsertCommon vitess.io/vitess/go/vt/vtgate/engine.InsertCommon + size += cached.InsertCommon.CachedSize(false) // field Query string size += hack.RuntimeAllocSize(int64(len(cached.Query))) // field VindexValues [][][]vitess.io/vitess/go/vt/vtgate/evalengine.Expr @@ -411,8 +411,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } } } - // field Prefix string - size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) // field Mid vitess.io/vitess/go/vt/sqlparser.Values { size += hack.RuntimeAllocSize(int64(cap(cached.Mid)) * int64(24)) @@ -427,8 +425,6 @@ func (cached *Insert) CachedSize(alloc bool) int64 { } } } - // field Suffix string - size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) return size } func (cached *InsertCommon) CachedSize(alloc bool) int64 { @@ -437,7 +433,7 @@ func (cached *InsertCommon) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(96) + size += int64(128) } // field Keyspace *vitess.io/vitess/go/vt/vtgate/vindexes.Keyspace size += cached.Keyspace.CachedSize(true) @@ -452,6 +448,10 @@ func (cached *InsertCommon) CachedSize(alloc bool) int64 { size += elem.CachedSize(true) } } + // field Prefix string + size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) + // field Suffix string + size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) return size } func (cached *InsertSelect) CachedSize(alloc bool) int64 { @@ -460,10 +460,10 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(80) + size += int64(176) } - // field InsertCommon *vitess.io/vitess/go/vt/vtgate/engine.InsertCommon - size += cached.InsertCommon.CachedSize(true) + // field InsertCommon vitess.io/vitess/go/vt/vtgate/engine.InsertCommon + size += cached.InsertCommon.CachedSize(false) // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { size += cc.CachedSize(true) @@ -477,10 +477,6 @@ func (cached *InsertSelect) CachedSize(alloc bool) int64 { } } } - // field Prefix string - size += hack.RuntimeAllocSize(int64(len(cached.Prefix))) - // field Suffix string - size += hack.RuntimeAllocSize(int64(len(cached.Suffix))) return size }