From 7da5132029a84b4764e1db2370cb048c0709e4ba Mon Sep 17 00:00:00 2001 From: Armand Parajon Date: Tue, 10 Oct 2023 15:32:27 -0700 Subject: [PATCH 1/3] Add backfill support for non-unique lookups Signed-off-by: Armand Parajon --- go/vt/vtgate/planbuilder/plan_test_vindex.go | 17 +++++++++++--- .../planbuilder/testdata/select_cases.json | 22 +++++++++++++++++++ .../planbuilder/testdata/vschemas/schema.json | 14 ++++++++++++ go/vt/vtgate/vindexes/lookup.go | 5 +++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/planbuilder/plan_test_vindex.go b/go/vt/vtgate/planbuilder/plan_test_vindex.go index 432ef7b8479..30d72f8c03a 100644 --- a/go/vt/vtgate/planbuilder/plan_test_vindex.go +++ b/go/vt/vtgate/planbuilder/plan_test_vindex.go @@ -72,7 +72,10 @@ func newLookupIndex(name string, _ map[string]string) (vindexes.Vindex, error) { var _ vindexes.Lookup = (*lookupIndex)(nil) // nameLkpIndex satisfies Lookup, NonUnique. -type nameLkpIndex struct{ name string } +type nameLkpIndex struct { + name string + inBackfill bool +} func (v *nameLkpIndex) String() string { return v.name } func (*nameLkpIndex) Cost() int { return 3 } @@ -102,13 +105,21 @@ func (*nameLkpIndex) Query() (string, []string) { func (*nameLkpIndex) MapResult([]sqltypes.Value, []*sqltypes.Result) ([]key.Destination, error) { return nil, nil } -func newNameLkpIndex(name string, _ map[string]string) (vindexes.Vindex, error) { - return &nameLkpIndex{name: name}, nil + +func (v *nameLkpIndex) IsBackfilling() bool { return v.inBackfill } + +func newNameLkpIndex(name string, m map[string]string) (vindexes.Vindex, error) { + vdx := &nameLkpIndex{name: name} + if val, ok := m["write_only"]; ok { + vdx.inBackfill = val == "true" + } + return vdx, nil } var _ vindexes.Vindex = (*nameLkpIndex)(nil) var _ vindexes.Lookup = (*nameLkpIndex)(nil) var _ vindexes.LookupPlanable = (*nameLkpIndex)(nil) +var _ vindexes.LookupBackfill = (*nameLkpIndex)(nil) // costlyIndex satisfies Lookup, NonUnique. type costlyIndex struct{ name string } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index f5090e40880..4dda5c83284 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -4664,6 +4664,28 @@ ] } }, + { + "comment": "name is in backfill vindex - not selected for vindex lookup", + "query": "select * from customer where name = 'x'", + "plan": { + "QueryType": "SELECT", + "Original": "select * from customer where name = 'x'", + "Instructions": { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from customer where 1 != 1", + "Query": "select * from customer where name = 'x'", + "Table": "customer" + }, + "TablesUsed": [ + "user.customer" + ] + } + }, { "comment": "email vindex is costly than phone vindex - but phone vindex is backfiling hence ignored", "query": "select * from customer where email = 'a@mail.com' and phone = 123456", diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index 36aa30381e2..7887bde729d 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -166,6 +166,16 @@ "to": "keyspace_id", "cost": "300" } + }, + "lkp_bf_vdx": { + "type": "name_lkp_test", + "owner": "customer", + "params": { + "table": "lkp_shard_vdx", + "from": " ", + "to": "keyspace_id", + "write_only": "true" + } } }, "tables": { @@ -476,6 +486,10 @@ { "column": "phone", "name": "unq_lkp_bf_vdx" + }, + { + "column": "name", + "name": "lkp_bf_vdx" } ] }, diff --git a/go/vt/vtgate/vindexes/lookup.go b/go/vt/vtgate/vindexes/lookup.go index b3e14fa01f6..33462470010 100644 --- a/go/vt/vtgate/vindexes/lookup.go +++ b/go/vt/vtgate/vindexes/lookup.go @@ -181,6 +181,11 @@ func (ln *LookupNonUnique) MarshalJSON() ([]byte, error) { return json.Marshal(ln.lkp) } +// IsBackfilling implements the LookupBackfill interface +func (ln *LookupNonUnique) IsBackfilling() bool { + return ln.writeOnly +} + // Query implements the LookupPlanable interface func (ln *LookupNonUnique) Query() (selQuery string, arguments []string) { return ln.lkp.query() From 110f0c90076b70bfb48c714883891a894b66f846 Mon Sep 17 00:00:00 2001 From: Armand Parajon Date: Tue, 14 Nov 2023 17:48:21 -0800 Subject: [PATCH 2/3] Make ConsistentLookup implement IsBackfilling Signed-off-by: Armand Parajon --- go/vt/vtgate/vindexes/consistent_lookup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtgate/vindexes/consistent_lookup.go b/go/vt/vtgate/vindexes/consistent_lookup.go index d51d9d2e386..b6852a6f48d 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup.go +++ b/go/vt/vtgate/vindexes/consistent_lookup.go @@ -489,7 +489,7 @@ func (lu *clCommon) GetCommitOrder() vtgatepb.CommitOrder { } // IsBackfilling implements the LookupBackfill interface -func (lu *ConsistentLookupUnique) IsBackfilling() bool { +func (lu *clCommon) IsBackfilling() bool { return lu.writeOnly } From 7fd1740841917a2ed7af543cfad8d171757a54f2 Mon Sep 17 00:00:00 2001 From: Armand Parajon Date: Wed, 22 Nov 2023 20:37:40 -0800 Subject: [PATCH 3/3] Fix query format Signed-off-by: Armand Parajon --- go/vt/vtgate/planbuilder/testdata/select_cases.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index 4dda5c83284..a6f18c4d528 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -4678,7 +4678,7 @@ "Sharded": true }, "FieldQuery": "select * from customer where 1 != 1", - "Query": "select * from customer where name = 'x'", + "Query": "select * from customer where `name` = 'x'", "Table": "customer" }, "TablesUsed": [