select name, 'inject <script>alert();</script>' from test_table limit 1000
`),
`
1
`,
`
none
`,
`
1000
`,
diff --git a/go/vt/vttablet/tabletserver/schema/cached_size.go b/go/vt/vttablet/tabletserver/schema/cached_size.go
index 8ca75534a62..4db9f313644 100644
--- a/go/vt/vttablet/tabletserver/schema/cached_size.go
+++ b/go/vt/vttablet/tabletserver/schema/cached_size.go
@@ -25,7 +25,7 @@ func (cached *MessageInfo) CachedSize(alloc bool) int64 {
}
size := int64(0)
if alloc {
- size += int64(80)
+ size += int64(96)
}
// field Fields []*vitess.io/vitess/go/vt/proto/query.Field
{
diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go
index aadba5739c8..fb4a9d9367a 100644
--- a/go/vt/vttablet/tabletserver/schema/engine.go
+++ b/go/vt/vttablet/tabletserver/schema/engine.go
@@ -31,6 +31,7 @@ import (
"vitess.io/vitess/go/acl"
"vitess.io/vitess/go/constants/sidecar"
"vitess.io/vitess/go/mysql"
+ "vitess.io/vitess/go/mysql/collations/charset"
"vitess.io/vitess/go/mysql/replication"
"vitess.io/vitess/go/mysql/sqlerror"
"vitess.io/vitess/go/sqltypes"
@@ -180,14 +181,15 @@ func (se *Engine) syncSidecarDB(ctx context.Context, conn *dbconnpool.DBConnecti
// EnsureConnectionAndDB ensures that we can connect to mysql.
// If tablet type is primary and there is no db, then the database is created.
// This function can be called before opening the Engine.
-func (se *Engine) EnsureConnectionAndDB(tabletType topodatapb.TabletType) error {
+func (se *Engine) EnsureConnectionAndDB(tabletType topodatapb.TabletType, serving bool) error {
ctx := tabletenv.LocalContext()
// We use AllPrivs since syncSidecarDB() might need to upgrade the schema
conn, err := dbconnpool.NewDBConnection(ctx, se.env.Config().DB.AllPrivsWithDB())
if err == nil {
se.dbCreationFailed = false
// upgrade sidecar db if required, for a tablet with an existing database
- if tabletType == topodatapb.TabletType_PRIMARY {
+ // only run DDL updates when a PRIMARY is transitioning to serving state.
+ if tabletType == topodatapb.TabletType_PRIMARY && serving {
if err := se.syncSidecarDB(ctx, conn); err != nil {
conn.Close()
return err
@@ -196,7 +198,7 @@ func (se *Engine) EnsureConnectionAndDB(tabletType topodatapb.TabletType) error
conn.Close()
return nil
}
- if tabletType != topodatapb.TabletType_PRIMARY {
+ if tabletType != topodatapb.TabletType_PRIMARY || !serving {
return err
}
if merr, isSQLErr := err.(*sqlerror.SQLError); !isSQLErr || merr.Num != sqlerror.ERBadDb {
@@ -423,10 +425,55 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
return err
}
+ var innodbTablesStats map[string]*Table
+ if includeStats {
+ if innodbTableSizesQuery := conn.Conn.BaseShowInnodbTableSizes(); innodbTableSizesQuery != "" {
+ // Since the InnoDB table size query is available to us on this MySQL version, we should use it.
+ // We therefore don't want to query for table sizes in getTableData()
+ includeStats = false
+
+ innodbResults, err := conn.Conn.Exec(ctx, innodbTableSizesQuery, maxTableCount, false)
+ if err != nil {
+ return vterrors.Wrapf(err, "in Engine.reload(), reading innodb tables")
+ }
+ innodbTablesStats = make(map[string]*Table, len(innodbResults.Rows))
+ for _, row := range innodbResults.Rows {
+ innodbTableName := row[0].ToString() // In the form of encoded `schema/table`
+ fileSize, _ := row[1].ToCastUint64()
+ allocatedSize, _ := row[2].ToCastUint64()
+
+ if _, ok := innodbTablesStats[innodbTableName]; !ok {
+ innodbTablesStats[innodbTableName] = &Table{}
+ }
+ // There could be multiple appearances of the same table in the result set:
+ // A table that has FULLTEXT indexes will appear once for the table itself,
+ // with total size of row data, and once for the aggregates size of all
+ // FULLTEXT indexes. We aggregate the sizes of all appearances of the same table.
+ table := innodbTablesStats[innodbTableName]
+ table.FileSize += fileSize
+ table.AllocatedSize += allocatedSize
+
+ if originalTableName, _, found := strings.Cut(innodbTableName, "#p#"); found {
+ // innodbTableName is encoded any special characters are turned into some @0-f0-f0-f value.
+ // Therefore this "#p#" here is a clear indication that we are looking at a partitioned table.
+ // We turn `my@002ddb/tbl_part#p#p0` into `my@002ddb/tbl_part`
+ // and aggregate the total partition sizes.
+ if _, ok := innodbTablesStats[originalTableName]; !ok {
+ innodbTablesStats[originalTableName] = &Table{}
+ originalTable := innodbTablesStats[originalTableName]
+ originalTable.FileSize += fileSize
+ originalTable.AllocatedSize += allocatedSize
+ }
+ }
+ }
+ // See testing in TestEngineReload
+ }
+ }
tableData, err := getTableData(ctx, conn.Conn, includeStats)
if err != nil {
return vterrors.Wrapf(err, "in Engine.reload(), reading tables")
}
+
// On the primary tablet, we also check the data we have stored in our schema tables to see what all needs reloading.
shouldUseDatabase := se.isServingPrimary && se.schemaCopy
@@ -461,8 +508,14 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
changedTables := make(map[string]*Table)
// created and altered contain the names of created and altered tables for broadcast.
var created, altered []*Table
+ databaseName := se.cp.DBName()
for _, row := range tableData.Rows {
tableName := row[0].ToString()
+ var innodbTable *Table
+ if innodbTablesStats != nil {
+ innodbTableName := fmt.Sprintf("%s/%s", charset.TablenameToFilename(databaseName), charset.TablenameToFilename(tableName))
+ innodbTable = innodbTablesStats[innodbTableName]
+ }
curTables[tableName] = true
createTime, _ := row[2].ToCastInt64()
var fileSize, allocatedSize uint64
@@ -473,6 +526,9 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
// publish the size metrics
se.tableFileSizeGauge.Set(tableName, int64(fileSize))
se.tableAllocatedSizeGauge.Set(tableName, int64(allocatedSize))
+ } else if innodbTable != nil {
+ se.tableFileSizeGauge.Set(tableName, int64(innodbTable.FileSize))
+ se.tableAllocatedSizeGauge.Set(tableName, int64(innodbTable.AllocatedSize))
}
// Table schemas are cached by tabletserver. For each table we cache `information_schema.tables.create_time` (`tbl.CreateTime`).
@@ -500,6 +556,9 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
if includeStats {
tbl.FileSize = fileSize
tbl.AllocatedSize = allocatedSize
+ } else if innodbTable != nil {
+ tbl.FileSize = innodbTable.FileSize
+ tbl.AllocatedSize = innodbTable.AllocatedSize
}
continue
}
@@ -519,6 +578,9 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
if includeStats {
table.FileSize = fileSize
table.AllocatedSize = allocatedSize
+ } else if innodbTable != nil {
+ table.FileSize = innodbTable.FileSize
+ table.AllocatedSize = innodbTable.AllocatedSize
}
table.CreateTime = createTime
changedTables[tableName] = table
@@ -803,6 +865,13 @@ func (se *Engine) broadcast(created, altered, dropped []*Table, udfsChanged bool
}
}
+// BroadcastForTesting is meant to be a testing function that triggers a broadcast call.
+func (se *Engine) BroadcastForTesting(created, altered, dropped []*Table, udfsChanged bool) {
+ se.mu.Lock()
+ defer se.mu.Unlock()
+ se.broadcast(created, altered, dropped, udfsChanged)
+}
+
// GetTable returns the info for a table.
func (se *Engine) GetTable(tableName sqlparser.IdentifierCS) *Table {
se.mu.Lock()
@@ -889,6 +958,7 @@ func NewEngineForTests() *Engine {
tables: make(map[string]*Table),
historian: newHistorian(false, 0, nil),
env: tabletenv.NewEnv(vtenv.NewTestEnv(), tabletenv.NewDefaultConfig(), "SchemaEngineForTests"),
+ notifiers: make(map[string]notifier),
}
return se
}
diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go
index 1fd6098b384..3faa7a10554 100644
--- a/go/vt/vttablet/tabletserver/schema/engine_test.go
+++ b/go/vt/vttablet/tabletserver/schema/engine_test.go
@@ -42,6 +42,7 @@ import (
"vitess.io/vitess/go/stats"
"vitess.io/vitess/go/test/utils"
"vitess.io/vitess/go/vt/dbconfigs"
+ topodatapb "vitess.io/vitess/go/vt/proto/topodata"
"vitess.io/vitess/go/vt/sqlparser"
"vitess.io/vitess/go/vt/vtenv"
"vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
@@ -53,11 +54,15 @@ import (
)
const baseShowTablesWithSizesPattern = `SELECT t\.table_name.*SUM\(i\.file_size\).*`
+const baseInnoDBTableSizesPattern = `(?s).*SELECT.*its\.space = it\.space.*SUM\(its\.file_size\).*`
var mustMatch = utils.MustMatchFn(".Mutex")
-func TestOpenAndReload(t *testing.T) {
- db := fakesqldb.New(t)
+// TestOpenAndReloadLegacy
+//
+// Runs with 5.7 env
+func TestOpenAndReloadLegacy(t *testing.T) {
+ db := fakesqldb.NewWithEnv(t, vtenv.NewLegacyTestEnv())
defer db.Close()
schematest.AddDefaultQueries(db)
@@ -86,7 +91,7 @@ func TestOpenAndReload(t *testing.T) {
))
firstReadRowsValue := 12
AddFakeInnoDBReadRowsResult(db, firstReadRowsValue)
- se := newEngine(10*time.Second, 10*time.Second, 0, db)
+ se := newEngine(10*time.Second, 10*time.Second, 0, db, vtenv.NewLegacyTestEnv())
se.Open()
defer se.Close()
@@ -95,6 +100,19 @@ func TestOpenAndReload(t *testing.T) {
assert.Equal(t, int64(0), se.tableFileSizeGauge.Counts()["msg"])
assert.Equal(t, int64(0), se.tableAllocatedSizeGauge.Counts()["msg"])
+ t.Run("EnsureConnectionAndDB", func(t *testing.T) {
+ // Verify that none of the following configurations run any schema change detection queries -
+ // 1. REPLICA serving
+ // 2. REPLICA non-serving
+ // 3. PRIMARY serving
+ err := se.EnsureConnectionAndDB(topodatapb.TabletType_REPLICA, true)
+ require.NoError(t, err)
+ err = se.EnsureConnectionAndDB(topodatapb.TabletType_PRIMARY, false)
+ require.NoError(t, err)
+ err = se.EnsureConnectionAndDB(topodatapb.TabletType_REPLICA, false)
+ require.NoError(t, err)
+ })
+
// Advance time some more.
db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields(
"t",
@@ -278,23 +296,279 @@ func TestOpenAndReload(t *testing.T) {
assert.Equal(t, want, se.GetSchema())
}
-func TestReloadWithSwappedTables(t *testing.T) {
+func TestOpenAndReload(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
schematest.AddDefaultQueries(db)
db.RejectQueryPattern(baseShowTablesWithSizesPattern, "Opening schema engine should query tables without size information")
+ db.RejectQueryPattern(baseInnoDBTableSizesPattern, "Opening schema engine should query tables without size information")
+
+ db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesFields,
+ RowsAffected: 0,
+ InsertID: 0,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseShowTablesRow("test_table_01", false, ""),
+ mysql.BaseShowTablesRow("test_table_02", false, ""),
+ mysql.BaseShowTablesRow("test_table_03", false, ""),
+ mysql.BaseShowTablesRow("seq", false, "vitess_sequence"),
+ mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
+ },
+ SessionStateChanges: "",
+ StatusFlags: 0,
+ })
+
+ // advance to one second after the default 1427325875.
+ db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields(
+ "t",
+ "int64"),
+ "1427325876",
+ ))
+ firstReadRowsValue := 12
+ AddFakeInnoDBReadRowsResult(db, firstReadRowsValue)
+ se := newEngine(10*time.Second, 10*time.Second, 0, db, nil)
+ se.Open()
+ defer se.Close()
+
+ want := initialSchema()
+ mustMatch(t, want, se.GetSchema())
+ assert.Equal(t, int64(0), se.tableFileSizeGauge.Counts()["msg"])
+ assert.Equal(t, int64(0), se.tableAllocatedSizeGauge.Counts()["msg"])
+
+ t.Run("EnsureConnectionAndDB", func(t *testing.T) {
+ // Verify that none of the following configurations run any schema change detection queries -
+ // 1. REPLICA serving
+ // 2. REPLICA non-serving
+ // 3. PRIMARY serving
+ err := se.EnsureConnectionAndDB(topodatapb.TabletType_REPLICA, true)
+ require.NoError(t, err)
+ err = se.EnsureConnectionAndDB(topodatapb.TabletType_PRIMARY, false)
+ require.NoError(t, err)
+ err = se.EnsureConnectionAndDB(topodatapb.TabletType_REPLICA, false)
+ require.NoError(t, err)
+ })
+
+ // Advance time some more.
+ db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields(
+ "t",
+ "int64"),
+ "1427325877",
+ ))
+ assert.EqualValues(t, firstReadRowsValue, se.innoDbReadRowsCounter.Get())
+ // Modify test_table_03
+ // Add test_table_04
+ // Drop msg
db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
Fields: mysql.BaseShowTablesFields,
RowsAffected: 0,
InsertID: 0,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseShowTablesRow("test_table_01", false, ""),
+ mysql.BaseShowTablesRow("test_table_02", false, ""),
+ {
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_03")), // table_name
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(t.create_time)
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
+ },
+ mysql.BaseShowTablesRow("test_table_04", false, ""),
+ mysql.BaseShowTablesRow("seq", false, "vitess_sequence"),
+ },
+ SessionStateChanges: "",
+ StatusFlags: 0,
+ })
+ // Modify test_table_03
+ // Add test_table_04
+ // Drop msg
+ db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{
+ Fields: mysql.BaseInnoDBTableSizesFields,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_01"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_02"),
+ {
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/test_table_03")), // table_name
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
+ },
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_04"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "seq"),
+ },
+ })
+ db.RejectQueryPattern(baseShowTablesWithSizesPattern, "Opening schema engine should query tables without size information")
+
+ db.MockQueriesForTable("test_table_03", &sqltypes.Result{
+ Fields: []*querypb.Field{{
+ Name: "pk1",
+ Type: sqltypes.Int32,
+ }, {
+ Name: "pk2",
+ Type: sqltypes.Int32,
+ }, {
+ Name: "val",
+ Type: sqltypes.Int32,
+ }},
+ })
+
+ db.MockQueriesForTable("test_table_04", &sqltypes.Result{
+ Fields: []*querypb.Field{{
+ Name: "pk",
+ Type: sqltypes.Int32,
+ }},
+ })
+
+ db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{
+ Fields: mysql.ShowPrimaryFields,
+ Rows: [][]sqltypes.Value{
+ mysql.ShowPrimaryRow("test_table_01", "pk"),
+ mysql.ShowPrimaryRow("test_table_02", "pk"),
+ mysql.ShowPrimaryRow("test_table_03", "pk1"),
+ mysql.ShowPrimaryRow("test_table_03", "pk2"),
+ mysql.ShowPrimaryRow("test_table_04", "pk"),
+ mysql.ShowPrimaryRow("seq", "id"),
+ },
+ })
+ secondReadRowsValue := 123
+ AddFakeInnoDBReadRowsResult(db, secondReadRowsValue)
+
+ firstTime := true
+ notifier := func(full map[string]*Table, created, altered, dropped []*Table, _ bool) {
+ if firstTime {
+ firstTime = false
+ createTables := extractNamesFromTablesList(created)
+ sort.Strings(createTables)
+ assert.Equal(t, []string{"dual", "msg", "seq", "test_table_01", "test_table_02", "test_table_03"}, createTables)
+ assert.Equal(t, []*Table(nil), altered)
+ assert.Equal(t, []*Table(nil), dropped)
+ } else {
+ assert.Equal(t, []string{"test_table_04"}, extractNamesFromTablesList(created))
+ assert.Equal(t, []string{"test_table_03"}, extractNamesFromTablesList(altered))
+ assert.Equal(t, []string{"msg"}, extractNamesFromTablesList(dropped))
+ }
+ }
+ se.RegisterNotifier("test", notifier, true)
+ err := se.Reload(context.Background())
+ require.NoError(t, err)
+
+ assert.EqualValues(t, secondReadRowsValue, se.innoDbReadRowsCounter.Get())
+
+ want["seq"].FileSize = 100
+ want["seq"].AllocatedSize = 150
+
+ want["test_table_01"].FileSize = 100
+ want["test_table_01"].AllocatedSize = 150
+
+ want["test_table_02"].FileSize = 100
+ want["test_table_02"].AllocatedSize = 150
+
+ want["test_table_03"] = &Table{
+ Name: sqlparser.NewIdentifierCS("test_table_03"),
+ Fields: []*querypb.Field{{
+ Name: "pk1",
+ Type: sqltypes.Int32,
+ }, {
+ Name: "pk2",
+ Type: sqltypes.Int32,
+ }, {
+ Name: "val",
+ Type: sqltypes.Int32,
+ }},
+ PKColumns: []int{0, 1},
+ CreateTime: 1427325877,
+ FileSize: 128,
+ AllocatedSize: 256,
+ }
+ want["test_table_04"] = &Table{
+ Name: sqlparser.NewIdentifierCS("test_table_04"),
+ Fields: []*querypb.Field{{
+ Name: "pk",
+ Type: sqltypes.Int32,
+ }},
+ PKColumns: []int{0},
+ CreateTime: 1427325875,
+ FileSize: 100,
+ AllocatedSize: 150,
+ }
+ delete(want, "msg")
+ assert.Equal(t, want, se.GetSchema())
+ assert.Equal(t, int64(0), se.tableAllocatedSizeGauge.Counts()["msg"])
+ assert.Equal(t, int64(0), se.tableFileSizeGauge.Counts()["msg"])
+
+ // ReloadAt tests
+ pos1, err := replication.DecodePosition("MariaDB/0-41983-20")
+ require.NoError(t, err)
+ pos2, err := replication.DecodePosition("MariaDB/0-41983-40")
+ require.NoError(t, err)
+ se.UnregisterNotifier("test")
+
+ err = se.ReloadAt(context.Background(), replication.Position{})
+ require.NoError(t, err)
+ assert.Equal(t, want, se.GetSchema())
+
+ err = se.ReloadAt(context.Background(), pos1)
+ require.NoError(t, err)
+ assert.Equal(t, want, se.GetSchema())
+
+ db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesWithSizesFields,
Rows: [][]sqltypes.Value{
mysql.BaseShowTablesWithSizesRow("test_table_01", false, ""),
mysql.BaseShowTablesWithSizesRow("test_table_02", false, ""),
- mysql.BaseShowTablesWithSizesRow("test_table_03", false, ""),
+ mysql.BaseShowTablesWithSizesRow("test_table_04", false, ""),
mysql.BaseShowTablesWithSizesRow("seq", false, "vitess_sequence"),
- mysql.BaseShowTablesWithSizesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
+ },
+ })
+
+ db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesFields,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseShowTablesRow("test_table_01", false, ""),
+ mysql.BaseShowTablesRow("test_table_02", false, ""),
+ mysql.BaseShowTablesRow("test_table_04", false, ""),
+ mysql.BaseShowTablesRow("seq", false, "vitess_sequence"),
+ },
+ })
+
+ db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{
+ Fields: mysql.ShowPrimaryFields,
+ Rows: [][]sqltypes.Value{
+ mysql.ShowPrimaryRow("test_table_01", "pk"),
+ mysql.ShowPrimaryRow("test_table_02", "pk"),
+ mysql.ShowPrimaryRow("test_table_04", "pk"),
+ mysql.ShowPrimaryRow("seq", "id"),
+ },
+ })
+ err = se.ReloadAt(context.Background(), pos1)
+ require.NoError(t, err)
+ assert.Equal(t, want, se.GetSchema())
+
+ delete(want, "test_table_03")
+ err = se.ReloadAt(context.Background(), pos2)
+ require.NoError(t, err)
+ assert.Equal(t, want, se.GetSchema())
+}
+
+func TestReloadWithSwappedTables(t *testing.T) {
+ db := fakesqldb.New(t)
+ defer db.Close()
+ schematest.AddDefaultQueries(db)
+
+ db.RejectQueryPattern(baseShowTablesWithSizesPattern, "Opening schema engine should query tables without size information")
+ db.RejectQueryPattern(baseInnoDBTableSizesPattern, "Opening schema engine should query tables without size information")
+
+ db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesFields,
+ RowsAffected: 0,
+ InsertID: 0,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseShowTablesRow("test_table_01", false, ""),
+ mysql.BaseShowTablesRow("test_table_02", false, ""),
+ mysql.BaseShowTablesRow("test_table_03", false, ""),
+ mysql.BaseShowTablesRow("seq", false, "vitess_sequence"),
+ mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
},
SessionStateChanges: "",
StatusFlags: 0,
@@ -302,7 +576,7 @@ func TestReloadWithSwappedTables(t *testing.T) {
firstReadRowsValue := 12
AddFakeInnoDBReadRowsResult(db, firstReadRowsValue)
- se := newEngine(10*time.Second, 10*time.Second, 0, db)
+ se := newEngine(10*time.Second, 10*time.Second, 0, db, nil)
se.Open()
defer se.Close()
want := initialSchema()
@@ -315,24 +589,43 @@ func TestReloadWithSwappedTables(t *testing.T) {
"int64"),
"1427325876",
))
- db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{
- Fields: mysql.BaseShowTablesWithSizesFields,
+ db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{
+ Fields: mysql.BaseInnoDBTableSizesFields,
Rows: [][]sqltypes.Value{
- mysql.BaseShowTablesWithSizesRow("test_table_01", false, ""),
- mysql.BaseShowTablesWithSizesRow("test_table_02", false, ""),
- mysql.BaseShowTablesWithSizesRow("test_table_03", false, ""),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_01"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_02"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_03"),
{
- sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_04")),
- sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")),
- sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(create_time)
- sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")),
- sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
- sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/test_table_04")), // table_name
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
},
- mysql.BaseShowTablesWithSizesRow("seq", false, "vitess_sequence"),
- mysql.BaseShowTablesWithSizesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "seq"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "msg"),
},
})
+ db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesFields,
+ RowsAffected: 0,
+ InsertID: 0,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseShowTablesRow("test_table_01", false, ""),
+ mysql.BaseShowTablesRow("test_table_02", false, ""),
+ mysql.BaseShowTablesRow("test_table_03", false, ""),
+ {
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_04")), // table_name
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(t.create_time)
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
+ },
+ mysql.BaseShowTablesRow("seq", false, "vitess_sequence"),
+ mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
+ },
+ SessionStateChanges: "",
+ StatusFlags: 0,
+ })
db.MockQueriesForTable("test_table_04", &sqltypes.Result{
Fields: []*querypb.Field{{
Name: "mypk",
@@ -389,11 +682,28 @@ func TestReloadWithSwappedTables(t *testing.T) {
"int64"),
"1427325877",
))
- db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{
- Fields: mysql.BaseShowTablesWithSizesFields,
+ db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{
+ Fields: mysql.BaseInnoDBTableSizesFields,
Rows: [][]sqltypes.Value{
- mysql.BaseShowTablesWithSizesRow("test_table_01", false, ""),
- mysql.BaseShowTablesWithSizesRow("test_table_02", false, ""),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_01"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_02"),
+ {
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/test_table_03")), // table_name
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
+ },
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_04"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "seq"),
+ mysql.BaseInnoDBTableSizesRow("fakesqldb", "msg"),
+ },
+ })
+ db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesFields,
+ RowsAffected: 0,
+ InsertID: 0,
+ Rows: [][]sqltypes.Value{
+ mysql.BaseShowTablesRow("test_table_01", false, ""),
+ mysql.BaseShowTablesRow("test_table_02", false, ""),
{
sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_03")),
sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")),
@@ -402,10 +712,12 @@ func TestReloadWithSwappedTables(t *testing.T) {
sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
},
- mysql.BaseShowTablesWithSizesRow("test_table_04", false, ""),
- mysql.BaseShowTablesWithSizesRow("seq", false, "vitess_sequence"),
- mysql.BaseShowTablesWithSizesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
+ mysql.BaseShowTablesRow("test_table_04", false, ""),
+ mysql.BaseShowTablesRow("seq", false, "vitess_sequence"),
+ mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"),
},
+ SessionStateChanges: "",
+ StatusFlags: 0,
})
db.MockQueriesForTable("test_table_03", &sqltypes.Result{
Fields: []*querypb.Field{{
@@ -468,7 +780,7 @@ func TestOpenFailedDueToExecErr(t *testing.T) {
schematest.AddDefaultQueries(db)
want := "injected error"
db.AddRejectedQuery(mysql.BaseShowTables, errors.New(want))
- se := newEngine(1*time.Second, 1*time.Second, 0, db)
+ se := newEngine(1*time.Second, 1*time.Second, 0, db, nil)
err := se.Open()
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("se.Open: %v, want %s", err, want)
@@ -499,7 +811,7 @@ func TestOpenFailedDueToLoadTableErr(t *testing.T) {
db.AddRejectedQuery("SELECT * FROM `fakesqldb`.`test_view` WHERE 1 != 1", sqlerror.NewSQLErrorFromError(errors.New("The user specified as a definer ('root'@'%') does not exist (errno 1449) (sqlstate HY000)")))
AddFakeInnoDBReadRowsResult(db, 0)
- se := newEngine(1*time.Second, 1*time.Second, 0, db)
+ se := newEngine(1*time.Second, 1*time.Second, 0, db, nil)
err := se.Open()
// failed load should return an error because of test_table
assert.ErrorContains(t, err, "Row count exceeded")
@@ -534,7 +846,7 @@ func TestOpenNoErrorDueToInvalidViews(t *testing.T) {
db.AddRejectedQuery("SELECT `col1`, `col2` FROM `fakesqldb`.`bar_view` WHERE 1 != 1", sqlerror.NewSQLError(sqlerror.ERWrongFieldWithGroup, sqlerror.SSClientError, "random error for table bar_view"))
AddFakeInnoDBReadRowsResult(db, 0)
- se := newEngine(1*time.Second, 1*time.Second, 0, db)
+ se := newEngine(1*time.Second, 1*time.Second, 0, db, nil)
err := se.Open()
require.NoError(t, err)
@@ -550,7 +862,7 @@ func TestExportVars(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
schematest.AddDefaultQueries(db)
- se := newEngine(1*time.Second, 1*time.Second, 0, db)
+ se := newEngine(1*time.Second, 1*time.Second, 0, db, nil)
se.Open()
defer se.Close()
expvar.Do(func(kv expvar.KeyValue) {
@@ -562,7 +874,7 @@ func TestStatsURL(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
schematest.AddDefaultQueries(db)
- se := newEngine(1*time.Second, 1*time.Second, 0, db)
+ se := newEngine(1*time.Second, 1*time.Second, 0, db, nil)
se.Open()
defer se.Close()
@@ -592,7 +904,7 @@ func TestSchemaEngineCloseTickRace(t *testing.T) {
})
AddFakeInnoDBReadRowsResult(db, 12)
// Start the engine with a small reload tick
- se := newEngine(100*time.Millisecond, 1*time.Second, 0, db)
+ se := newEngine(100*time.Millisecond, 1*time.Second, 0, db, nil)
err := se.Open()
require.NoError(t, err)
@@ -619,15 +931,20 @@ func TestSchemaEngineCloseTickRace(t *testing.T) {
}
}
-func newEngine(reloadTime time.Duration, idleTimeout time.Duration, schemaMaxAgeSeconds int64, db *fakesqldb.DB) *Engine {
+func newEngine(reloadTime time.Duration, idleTimeout time.Duration, schemaMaxAgeSeconds int64, db *fakesqldb.DB, env *vtenv.Environment) *Engine {
cfg := tabletenv.NewDefaultConfig()
cfg.SchemaReloadInterval = reloadTime
cfg.OltpReadPool.IdleTimeout = idleTimeout
cfg.OlapReadPool.IdleTimeout = idleTimeout
cfg.TxPool.IdleTimeout = idleTimeout
cfg.SchemaVersionMaxAgeSeconds = schemaMaxAgeSeconds
- se := NewEngine(tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "SchemaTest"))
- se.InitDBConfig(newDBConfigs(db).DbaWithDB())
+ dbConfigs := newDBConfigs(db)
+ cfg.DB = dbConfigs
+ if env == nil {
+ env = vtenv.NewTestEnv()
+ }
+ se := NewEngine(tabletenv.NewEnv(env, cfg, "SchemaTest"))
+ se.InitDBConfig(dbConfigs.DbaWithDB())
return se
}
@@ -737,6 +1054,7 @@ func initialSchema() map[string]*Table {
BatchSize: 1,
CacheSize: 10,
PollInterval: 30 * time.Second,
+ IDType: sqltypes.Int64,
},
},
}
@@ -1015,7 +1333,6 @@ func TestEngineGetTableData(t *testing.T) {
name string
expectedQueries map[string]*sqltypes.Result
queriesToReject map[string]error
- includeStats bool
expectedError string
}{
{
@@ -1023,13 +1340,6 @@ func TestEngineGetTableData(t *testing.T) {
expectedQueries: map[string]*sqltypes.Result{
conn.BaseShowTables(): {},
},
- includeStats: false,
- }, {
- name: "Success with include stats",
- expectedQueries: map[string]*sqltypes.Result{
- conn.BaseShowTablesWithSizes(): {},
- },
- includeStats: true,
}, {
name: "Error in query",
queriesToReject: map[string]error{
@@ -1051,7 +1361,7 @@ func TestEngineGetTableData(t *testing.T) {
defer db.DeleteRejectedQuery(query)
}
- _, err = getTableData(context.Background(), conn, tt.includeStats)
+ _, err = getTableData(context.Background(), conn, false)
if tt.expectedError != "" {
require.ErrorContains(t, err, tt.expectedError)
return
@@ -1178,194 +1488,204 @@ func TestEngineGetDroppedTables(t *testing.T) {
// TestEngineReload tests the entire functioning of engine.Reload testing all the queries that we end up running against MySQL
// while simulating the responses and verifies the final list of created, altered and dropped tables.
func TestEngineReload(t *testing.T) {
- db := fakesqldb.New(t)
- cfg := tabletenv.NewDefaultConfig()
- cfg.DB = newDBConfigs(db)
- cfg.SignalWhenSchemaChange = true
- env := tabletenv.NewEnv(vtenv.NewTestEnv(), nil, "TestEngineReload")
- conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil, env)
- require.NoError(t, err)
-
- se := newEngine(10*time.Second, 10*time.Second, 0, db)
- se.conns.Open(se.cp, se.cp, se.cp)
- se.isOpen = true
- se.notifiers = make(map[string]notifier)
- se.MakePrimary(true)
+ envs := []*vtenv.Environment{
+ vtenv.NewTestEnv(),
+ vtenv.NewLegacyTestEnv(),
+ }
+ for _, venv := range envs {
+ t.Run(venv.MySQLVersion(), func(t *testing.T) {
+ db := fakesqldb.NewWithEnv(t, venv)
+ cfg := tabletenv.NewDefaultConfig()
+ cfg.DB = newDBConfigs(db)
+ cfg.SignalWhenSchemaChange = true
+
+ env := tabletenv.NewEnv(venv, nil, "TestEngineReload")
+ conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil, env)
+ require.NoError(t, err)
- // If we have to skip the meta check, then there is nothing to do
- se.SkipMetaCheck = true
- err = se.reload(context.Background(), false)
- require.NoError(t, err)
+ se := newEngine(10*time.Second, 10*time.Second, 0, db, venv)
+ se.conns.Open(se.cp, se.cp, se.cp)
+ se.isOpen = true
+ se.notifiers = make(map[string]notifier)
+ se.MakePrimary(true)
- se.SkipMetaCheck = false
- se.lastChange = 987654321
+ // If we have to skip the meta check, then there is nothing to do
+ se.SkipMetaCheck = true
+ err = se.reload(context.Background(), false)
+ require.NoError(t, err)
- // Initial tables in the schema engine
- se.tables = map[string]*Table{
- "t1": {
- Name: sqlparser.NewIdentifierCS("t1"),
- Type: NoType,
- CreateTime: 123456789,
- },
- "t2": {
- Name: sqlparser.NewIdentifierCS("t2"),
- Type: NoType,
- CreateTime: 123456789,
- },
- "t4": {
- Name: sqlparser.NewIdentifierCS("t4"),
- Type: NoType,
- CreateTime: 123456789,
- },
- "v1": {
- Name: sqlparser.NewIdentifierCS("v1"),
- Type: View,
- CreateTime: 123456789,
- },
- "v2": {
- Name: sqlparser.NewIdentifierCS("v2"),
- Type: View,
- CreateTime: 123456789,
- },
- "v4": {
- Name: sqlparser.NewIdentifierCS("v4"),
- Type: View,
- CreateTime: 123456789,
- },
- }
- // MySQL unix timestamp query.
- db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields("UNIX_TIMESTAMP", "int64"), "987654326"))
- // Table t2 is updated, T2 is created and t4 is deleted.
- // View v2 is updated, V2 is created and v4 is deleted.
- db.AddQuery(conn.BaseShowTables(), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|table_type|unix_timestamp(create_time)|table_comment",
- "varchar|varchar|int64|varchar"),
- "t1|BASE_TABLE|123456789|",
- "t2|BASE_TABLE|123456790|",
- "T2|BASE_TABLE|123456789|",
- "v1|VIEW|123456789|",
- "v2|VIEW|123456789|",
- "V2|VIEW|123456789|",
- ))
+ se.SkipMetaCheck = false
+ se.lastChange = 987654321
- // Detecting view changes.
- // According to the database, v2, V2, v4, and v5 require updating.
- db.AddQuery(fmt.Sprintf(detectViewChange, sidecar.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
- "v2",
- "V2",
- "v4",
- "v5",
- ))
+ // Initial tables in the schema engine
+ se.tables = map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 123456789,
+ },
+ "t2": {
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: NoType,
+ CreateTime: 123456789,
+ },
+ "t4": {
+ Name: sqlparser.NewIdentifierCS("t4"),
+ Type: NoType,
+ CreateTime: 123456789,
+ },
+ "v1": {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 123456789,
+ },
+ "v2": {
+ Name: sqlparser.NewIdentifierCS("v2"),
+ Type: View,
+ CreateTime: 123456789,
+ },
+ "v4": {
+ Name: sqlparser.NewIdentifierCS("v4"),
+ Type: View,
+ CreateTime: 123456789,
+ },
+ }
+ // MySQL unix timestamp query.
+ db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields("UNIX_TIMESTAMP", "int64"), "987654326"))
+ // Table t2 is updated, T2 is created and t4 is deleted.
+ // View v2 is updated, V2 is created and v4 is deleted.
+ db.AddQuery(conn.BaseShowTables(), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|table_type|unix_timestamp(create_time)|table_comment",
+ "varchar|varchar|int64|varchar"),
+ "t1|BASE_TABLE|123456789|",
+ "t2|BASE_TABLE|123456790|",
+ "T2|BASE_TABLE|123456789|",
+ "v1|VIEW|123456789|",
+ "v2|VIEW|123456789|",
+ "V2|VIEW|123456789|",
+ ))
+ // Detecting view changes.
+ // According to the database, v2, V2, v4, and v5 require updating.
+ db.AddQuery(fmt.Sprintf(detectViewChange, sidecar.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
+ "v2",
+ "V2",
+ "v4",
+ "v5",
+ ))
- // Finding mismatches in the tables.
- // t5 exists in the database.
- db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.`tables`", sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64"),
- "t1|123456789",
- "t2|123456789",
- "t4|123456789",
- "t5|123456789",
- ))
+ // Finding mismatches in the tables.
+ // t5 exists in the database.
+ db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.`tables`", sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64"),
+ "t1|123456789",
+ "t2|123456789",
+ "t4|123456789",
+ "t5|123456789",
+ ))
- // Read Innodb_rows_read.
- db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(sqltypes.MakeTestFields("Variable_name|Value", "varchar|int64"),
- "Innodb_rows_read|35"))
+ // Read Innodb_rows_read.
+ db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(sqltypes.MakeTestFields("Variable_name|Value", "varchar|int64"),
+ "Innodb_rows_read|35"))
- // Queries to load the tables' information.
- for _, tableName := range []string{"t2", "T2", "v2", "V2"} {
- db.AddQuery(fmt.Sprintf(`SELECT COLUMN_NAME as column_name
+ // Queries to load the tables' information.
+ for _, tableName := range []string{"t2", "T2", "v2", "V2"} {
+ db.AddQuery(fmt.Sprintf(`SELECT COLUMN_NAME as column_name
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = 'fakesqldb' AND TABLE_NAME = '%s'
ORDER BY ORDINAL_POSITION`, tableName),
- sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"),
- "col1"))
- db.AddQuery(fmt.Sprintf("SELECT `col1` FROM `fakesqldb`.`%v` WHERE 1 != 1", tableName), sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "varchar")))
- }
-
- // Primary key information.
- db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(mysql.ShowPrimaryFields,
- "t1|col1",
- "t2|col1",
- "T2|col1",
- ))
-
- // Queries for reloading the tables' information.
- {
- for _, tableName := range []string{"t2", "T2"} {
- db.AddQuery(fmt.Sprintf(`show create table %s`, tableName),
- sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar"),
- fmt.Sprintf("%v|create_table_%v", tableName, tableName)))
- }
- db.AddQuery("begin", &sqltypes.Result{})
- db.AddQuery("commit", &sqltypes.Result{})
- db.AddQuery("rollback", &sqltypes.Result{})
- // We are adding both the variants of the delete statements that we can see in the test, since the deleted tables are initially stored as a map, the order is not defined.
- db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t5', 't4', 'T2', 't2')", &sqltypes.Result{})
- db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t4', 't5', 'T2', 't2')", &sqltypes.Result{})
- db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 't2', 'create_table_t2', 123456790)", &sqltypes.Result{})
- db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 'T2', 'create_table_T2', 123456789)", &sqltypes.Result{})
- }
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"),
+ "col1"))
+ db.AddQuery(fmt.Sprintf("SELECT `col1` FROM `fakesqldb`.`%v` WHERE 1 != 1", tableName), sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "varchar")))
+ }
- // Queries for reloading the views' information.
- {
- for _, tableName := range []string{"v2", "V2"} {
- db.AddQuery(fmt.Sprintf(`show create table %s`, tableName),
- sqltypes.MakeTestResult(sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar"),
- fmt.Sprintf("%v|create_table_%v|utf8mb4|utf8mb4_0900_ai_ci", tableName, tableName)))
- }
- // We are adding both the variants of the select statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined.
- db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v4', 'v5', 'V2', 'v2')",
- sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
- "v2|select_v2",
- "V2|select_V2",
- ))
- db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v5', 'v4', 'V2', 'v2')",
- sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
- "v2|select_v2",
- "V2|select_V2",
+ // Primary key information.
+ db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(mysql.ShowPrimaryFields,
+ "t1|col1",
+ "t2|col1",
+ "T2|col1",
))
- // We are adding both the variants of the delete statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined.
- db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v4', 'v5', 'V2', 'v2')", &sqltypes.Result{})
- db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v5', 'v4', 'V2', 'v2')", &sqltypes.Result{})
- db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'v2', 'create_table_v2', 'select_v2')", &sqltypes.Result{})
- db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'V2', 'create_table_V2', 'select_V2')", &sqltypes.Result{})
- }
+ // Queries for reloading the tables' information.
+ {
+ for _, tableName := range []string{"t2", "T2"} {
+ db.AddQuery(fmt.Sprintf(`show create table %s`, tableName),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar"),
+ fmt.Sprintf("%v|create_table_%v", tableName, tableName)))
+ }
+ db.AddQuery("begin", &sqltypes.Result{})
+ db.AddQuery("commit", &sqltypes.Result{})
+ db.AddQuery("rollback", &sqltypes.Result{})
+ // We are adding both the variants of the delete statements that we can see in the test, since the deleted tables are initially stored as a map, the order is not defined.
+ db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t5', 't4', 'T2', 't2')", &sqltypes.Result{})
+ db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t4', 't5', 'T2', 't2')", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 't2', 'create_table_t2', 123456790)", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 'T2', 'create_table_T2', 123456789)", &sqltypes.Result{})
+ }
- // adding query pattern for udfs
- udfQueryPattern := "SELECT name.*"
- db.AddQueryPattern(udfQueryPattern, &sqltypes.Result{})
+ // Queries for reloading the views' information.
+ {
+ for _, tableName := range []string{"v2", "V2"} {
+ db.AddQuery(fmt.Sprintf(`show create table %s`, tableName),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar"),
+ fmt.Sprintf("%v|create_table_%v|utf8mb4|utf8mb4_0900_ai_ci", tableName, tableName)))
+ }
+ // We are adding both the variants of the select statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined.
+ db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v4', 'v5', 'V2', 'v2')",
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
+ "v2|select_v2",
+ "V2|select_V2",
+ ))
+ db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v5', 'v4', 'V2', 'v2')",
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
+ "v2|select_v2",
+ "V2|select_V2",
+ ))
+
+ // We are adding both the variants of the delete statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined.
+ db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v4', 'v5', 'V2', 'v2')", &sqltypes.Result{})
+ db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v5', 'v4', 'V2', 'v2')", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'v2', 'create_table_v2', 'select_v2')", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'V2', 'create_table_V2', 'select_V2')", &sqltypes.Result{})
+ }
- // Verify the list of created, altered and dropped tables seen.
- se.RegisterNotifier("test", func(full map[string]*Table, created, altered, dropped []*Table, _ bool) {
- require.ElementsMatch(t, extractNamesFromTablesList(created), []string{"T2", "V2"})
- require.ElementsMatch(t, extractNamesFromTablesList(altered), []string{"t2", "v2"})
- require.ElementsMatch(t, extractNamesFromTablesList(dropped), []string{"t4", "v4", "t5", "v5"})
- }, false)
+ // adding query pattern for udfs
+ udfQueryPattern := "SELECT name.*mysql.func.*"
+ db.AddQueryPattern(udfQueryPattern, &sqltypes.Result{})
- // Run the reload.
- err = se.reload(context.Background(), false)
- require.NoError(t, err)
- require.NoError(t, db.LastError())
- require.Zero(t, se.throttledLogger.GetLastLogTime())
+ // Verify the list of created, altered and dropped tables seen.
+ se.RegisterNotifier("test", func(full map[string]*Table, created, altered, dropped []*Table, _ bool) {
+ require.ElementsMatch(t, extractNamesFromTablesList(created), []string{"T2", "V2"})
+ require.ElementsMatch(t, extractNamesFromTablesList(altered), []string{"t2", "v2"})
+ require.ElementsMatch(t, extractNamesFromTablesList(dropped), []string{"t4", "v4", "t5", "v5"})
+ }, false)
- // Now if we remove the query pattern for udfs, schema engine shouldn't fail.
- // Instead we should see a log message with the error.
- db.RemoveQueryPattern(udfQueryPattern)
- se.UnregisterNotifier("test")
- err = se.reload(context.Background(), false)
- require.NoError(t, err)
- // Check for the udf error being logged. The last log time should be less than a second.
- require.Less(t, time.Since(se.throttledLogger.GetLastLogTime()), 1*time.Second)
+ // Run the reload.
+ err = se.reload(context.Background(), false)
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ require.Zero(t, se.throttledLogger.GetLastLogTime())
+
+ // Now if we remove the query pattern for udfs, schema engine shouldn't fail.
+ // Instead we should see a log message with the error.
+ db.RemoveQueryPattern(udfQueryPattern)
+ se.UnregisterNotifier("test")
+ err = se.reload(context.Background(), false)
+ require.NoError(t, err)
+ // Check for the udf error being logged. The last log time should be less than a second.
+ require.Less(t, time.Since(se.throttledLogger.GetLastLogTime()), 1*time.Second)
+ })
+ }
}
-// TestEngineReload tests the vreplication specific GetTableForPos function to ensure
+// TestGetTableForPosLegacy tests the vreplication specific GetTableForPos function to ensure
// that it conforms to the intended/expected behavior in various scenarios.
// This more specifically tests the behavior of the function when the historian is
// disabled or otherwise unable to get a table schema for the given position. When it
// CAN, that is tested indepenently in the historian tests.
-func TestGetTableForPos(t *testing.T) {
+//
+// Runs with 5.7 env
+func TestGetTableForPosLegacy(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- fakedb := fakesqldb.New(t)
+ fakedb := fakesqldb.NewWithEnv(t, vtenv.NewLegacyTestEnv())
cfg := tabletenv.NewDefaultConfig()
cfg.DB = newDBConfigs(fakedb)
table := sqlparser.NewIdentifierCS("t1")
@@ -1383,7 +1703,7 @@ func TestGetTableForPos(t *testing.T) {
}
// Don't do any automatic / TTL based cache refreshes.
- se := newEngine(1*time.Hour, 1*time.Hour, 0, fakedb)
+ se := newEngine(1*time.Hour, 1*time.Hour, 0, fakedb, vtenv.NewLegacyTestEnv())
se.conns.Open(se.cp, se.cp, se.cp)
se.isOpen = true
se.notifiers = make(map[string]notifier)
@@ -1545,3 +1865,197 @@ func TestGetTableForPos(t *testing.T) {
})
}
}
+
+// TestGetTableForPos tests the vreplication specific GetTableForPos function to ensure
+// that it conforms to the intended/expected behavior in various scenarios.
+// This more specifically tests the behavior of the function when the historian is
+// disabled or otherwise unable to get a table schema for the given position. When it
+// CAN, that is tested indepenently in the historian tests.
+func TestGetTableForPos(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ fakedb := fakesqldb.New(t)
+ cfg := tabletenv.NewDefaultConfig()
+ cfg.DB = newDBConfigs(fakedb)
+ table := sqlparser.NewIdentifierCS("t1")
+ column := "col1"
+ tableSchema := fmt.Sprintf("create table %s (%s varchar(50), primary key(col1))", table.String(), column)
+ tableMt := &binlogdatapb.MinimalTable{
+ Name: table.String(),
+ Fields: []*querypb.Field{
+ {
+ Name: column,
+ Type: sqltypes.VarChar,
+ },
+ },
+ PKColumns: []int64{0}, // First column: col1
+ }
+
+ // Don't do any automatic / TTL based cache refreshes.
+ se := newEngine(1*time.Hour, 1*time.Hour, 0, fakedb, nil)
+ se.conns.Open(se.cp, se.cp, se.cp)
+ se.isOpen = true
+ se.notifiers = make(map[string]notifier)
+ se.MakePrimary(true)
+ se.historian.enabled = false
+
+ addExpectedReloadQueries := func(db *fakesqldb.DB) {
+ db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields(
+ "UNIX_TIMESTAMP()",
+ "int64"),
+ fmt.Sprintf("%d", time.Now().Unix()),
+ ))
+ db.AddQuery(fmt.Sprintf(detectViewChange, sidecar.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar")))
+ db.AddQuery(fmt.Sprintf(readTableCreateTimes, sidecar.GetIdentifier()),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64")))
+ db.AddQuery(fmt.Sprintf(detectUdfChange, sidecar.GetIdentifier()), &sqltypes.Result{})
+ db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{
+ Fields: mysql.BaseInnoDBTableSizesFields,
+ Rows: [][]sqltypes.Value{
+ {
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/"+table.String())), // table_name
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size
+ },
+ },
+ })
+ db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{
+ Fields: mysql.BaseShowTablesFields,
+ Rows: [][]sqltypes.Value{
+ {
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte(table.String())), // table_name
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type
+ sqltypes.MakeTrusted(sqltypes.Int64, []byte(fmt.Sprintf("%d", time.Now().Unix()-1000))), // unix_timestamp(t.create_time)
+ sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment
+ },
+ },
+ SessionStateChanges: "",
+ StatusFlags: 0,
+ })
+ db.RejectQueryPattern(baseShowTablesWithSizesPattern, "we should expect to get sizes by InnoDBTableSizes")
+ db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{
+ Fields: mysql.ShowPrimaryFields,
+ Rows: [][]sqltypes.Value{
+ mysql.ShowPrimaryRow(table.String(), column),
+ },
+ })
+ db.AddQueryPattern(fmt.Sprintf(mysql.GetColumnNamesQueryPatternForTable, table.String()),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"), column))
+ db.AddQuery(fmt.Sprintf("SELECT `%s` FROM `fakesqldb`.`%v` WHERE 1 != 1", column, table.String()),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields(column, "varchar")))
+ db.AddQuery(fmt.Sprintf(`show create table %s`, table.String()),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table|Create Table", "varchar|varchar"), table.String(), tableSchema))
+ db.AddQuery("begin", &sqltypes.Result{})
+ db.AddQuery(fmt.Sprintf("delete from %s.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('%s')",
+ sidecar.GetIdentifier(), table.String()), &sqltypes.Result{})
+ db.AddQuery(fmt.Sprintf("insert into %s.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), '%s', '%s', %d)",
+ sidecar.GetIdentifier(), table.String(), tableSchema, time.Now().Unix()), &sqltypes.Result{RowsAffected: 1})
+ db.AddQuery("rollback", &sqltypes.Result{})
+ }
+
+ type testcase struct {
+ name string
+ initialCacheState map[string]*Table
+ expectedQueriesFunc func(db *fakesqldb.DB)
+ expectFunc func()
+ }
+ tests := []testcase{
+ {
+ name: "GetTableForPos with cache uninitialized",
+ initialCacheState: make(map[string]*Table), // empty
+ expectedQueriesFunc: func(db *fakesqldb.DB) {
+ // We do a reload to initialize the cache.
+ addExpectedReloadQueries(db)
+ },
+ expectFunc: func() {
+ tbl, err := se.GetTableForPos(ctx, table, "")
+ require.NoError(t, err)
+ require.Equal(t, tableMt, tbl)
+ },
+ },
+ {
+ name: "GetTableForPos with cache uninitialized, table not found",
+ initialCacheState: make(map[string]*Table), // empty
+ expectedQueriesFunc: func(db *fakesqldb.DB) {
+ // We do a reload to initialize the cache and in doing so get the missing table.
+ addExpectedReloadQueries(db)
+ },
+ expectFunc: func() {
+ tbl, err := se.GetTableForPos(ctx, sqlparser.NewIdentifierCS("nobueno"), "")
+ require.EqualError(t, err, "table nobueno not found in vttablet schema")
+ require.Nil(t, tbl)
+ },
+ },
+ {
+ name: "GetTableForPos with cache initialized, table not found",
+ initialCacheState: map[string]*Table{"t2": {Name: sqlparser.NewIdentifierCS("t2")}},
+ expectedQueriesFunc: func(db *fakesqldb.DB) {
+ // We do a reload to try and get this missing table and any other recently created ones.
+ addExpectedReloadQueries(db)
+ },
+ expectFunc: func() {
+ tbl, err := se.GetTableForPos(ctx, table, "")
+ require.NoError(t, err)
+ require.Equal(t, tableMt, tbl)
+ },
+ },
+ {
+ name: "GetTableForPos with cache initialized, table found",
+ initialCacheState: map[string]*Table{table.String(): {Name: table}},
+ expectedQueriesFunc: func(db *fakesqldb.DB) {
+ // We only reload the column and PK info for the table in our cache. A new column
+ // called col2 has been added to the table schema and it is the new PK.
+ newTableSchema := fmt.Sprintf("create table %s (%s varchar(50), col2 varchar(50), primary key(col2))", table.String(), column)
+ db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{
+ Fields: mysql.ShowPrimaryFields,
+ Rows: [][]sqltypes.Value{
+ mysql.ShowPrimaryRow(table.String(), "col2"),
+ },
+ })
+ db.AddQueryPattern(fmt.Sprintf(mysql.GetColumnNamesQueryPatternForTable, table.String()),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"), column, "col2"))
+ db.AddQuery(fmt.Sprintf("SELECT `%s`, `%s` FROM `fakesqldb`.`%v` WHERE 1 != 1",
+ column, "col2", table.String()), sqltypes.MakeTestResult(sqltypes.MakeTestFields(fmt.Sprintf("%s|%s", column, "col2"), "varchar|varchar")))
+ db.AddQuery(fmt.Sprintf(`show create table %s`, table.String()),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table|Create Table", "varchar|varchar"), table.String(), newTableSchema))
+ db.AddQuery("begin", &sqltypes.Result{})
+ db.AddQuery(fmt.Sprintf("delete from %s.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('%s')",
+ sidecar.GetIdentifier(), table.String()), &sqltypes.Result{})
+ db.AddQuery(fmt.Sprintf("insert into %s.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), '%s', '%s', %d)",
+ sidecar.GetIdentifier(), table.String(), newTableSchema, time.Now().Unix()), &sqltypes.Result{})
+ db.AddQuery("rollback", &sqltypes.Result{})
+ },
+ expectFunc: func() {
+ tbl, err := se.GetTableForPos(ctx, table, "MySQL56/1497ddb0-7cb9-11ed-a1eb-0242ac120002:1-891")
+ require.NoError(t, err)
+ require.NotNil(t, tbl)
+ require.Equal(t, &binlogdatapb.MinimalTable{
+ Name: table.String(),
+ Fields: []*querypb.Field{
+ {
+ Name: column,
+ Type: sqltypes.VarChar,
+ },
+ {
+ Name: "col2",
+ Type: sqltypes.VarChar,
+ },
+ },
+ PKColumns: []int64{1}, // Second column: col2
+ }, tbl)
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ fakedb.DeleteAllQueries()
+ AddFakeInnoDBReadRowsResult(fakedb, int(rand.Int32N(1000000)))
+ tc.expectedQueriesFunc(fakedb)
+ se.tables = tc.initialCacheState
+ tc.expectFunc()
+ fakedb.VerifyAllExecutedOrFail()
+ require.NoError(t, fakedb.LastError())
+ })
+ }
+}
diff --git a/go/vt/vttablet/tabletserver/schema/load_table.go b/go/vt/vttablet/tabletserver/schema/load_table.go
index 6022f8724eb..2a2be33a208 100644
--- a/go/vt/vttablet/tabletserver/schema/load_table.go
+++ b/go/vt/vttablet/tabletserver/schema/load_table.go
@@ -157,6 +157,14 @@ func loadMessageInfo(ta *Table, comment string, collationEnv *collations.Environ
ta.MessageInfo.Fields = getDefaultMessageFields(ta.Fields, hiddenCols)
}
+ ta.MessageInfo.IDType = sqltypes.VarBinary
+ for _, field := range ta.MessageInfo.Fields {
+ if field.Name == "id" {
+ ta.MessageInfo.IDType = field.Type
+ break
+ }
+ }
+
return nil
}
diff --git a/go/vt/vttablet/tabletserver/schema/load_table_test.go b/go/vt/vttablet/tabletserver/schema/load_table_test.go
index 6416e2e306e..e451624c0d7 100644
--- a/go/vt/vttablet/tabletserver/schema/load_table_test.go
+++ b/go/vt/vttablet/tabletserver/schema/load_table_test.go
@@ -149,6 +149,7 @@ func TestLoadTableMessage(t *testing.T) {
BatchSize: 1,
CacheSize: 10,
PollInterval: 30 * time.Second,
+ IDType: sqltypes.Int64,
},
}
assert.Equal(t, want, table)
diff --git a/go/vt/vttablet/tabletserver/schema/main_test.go b/go/vt/vttablet/tabletserver/schema/main_test.go
index 7eaca5f18e5..8b705b9bfbd 100644
--- a/go/vt/vttablet/tabletserver/schema/main_test.go
+++ b/go/vt/vttablet/tabletserver/schema/main_test.go
@@ -34,11 +34,11 @@ func getTestSchemaEngine(t *testing.T, schemaMaxAgeSeconds int64) (*Engine, *fak
"int64"),
"1427325876",
))
- db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{})
+ db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{})
db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{})
db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{})
AddFakeInnoDBReadRowsResult(db, 1)
- se := newEngine(10*time.Second, 10*time.Second, schemaMaxAgeSeconds, db)
+ se := newEngine(10*time.Second, 10*time.Second, schemaMaxAgeSeconds, db, nil)
require.NoError(t, se.Open())
cancel := func() {
defer db.Close()
diff --git a/go/vt/vttablet/tabletserver/schema/schema.go b/go/vt/vttablet/tabletserver/schema/schema.go
index 4b3d9c88fb5..e800477da3b 100644
--- a/go/vt/vttablet/tabletserver/schema/schema.go
+++ b/go/vt/vttablet/tabletserver/schema/schema.go
@@ -20,6 +20,7 @@ import (
"sync"
"time"
+ "vitess.io/vitess/go/sqltypes"
"vitess.io/vitess/go/vt/log"
"vitess.io/vitess/go/vt/sqlparser"
@@ -125,6 +126,9 @@ type MessageInfo struct {
// MaxBackoff specifies the longest duration message manager
// should wait before rescheduling a message
MaxBackoff time.Duration
+
+ // IDType specifies the type of the ID column
+ IDType sqltypes.Type
}
// NewTable creates a new Table.
diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go
index 3fe78457b60..cae6a237dc8 100644
--- a/go/vt/vttablet/tabletserver/state_manager.go
+++ b/go/vt/vttablet/tabletserver/state_manager.go
@@ -140,7 +140,7 @@ type stateManager struct {
type (
schemaEngine interface {
- EnsureConnectionAndDB(topodatapb.TabletType) error
+ EnsureConnectionAndDB(topodatapb.TabletType, bool) error
Open() error
MakeNonPrimary()
MakePrimary(bool)
@@ -447,7 +447,7 @@ func (sm *stateManager) verifyTargetLocked(ctx context.Context, target *querypb.
func (sm *stateManager) servePrimary() error {
sm.watcher.Close()
- if err := sm.connect(topodatapb.TabletType_PRIMARY); err != nil {
+ if err := sm.connect(topodatapb.TabletType_PRIMARY, true); err != nil {
return err
}
@@ -476,7 +476,7 @@ func (sm *stateManager) unservePrimary() error {
sm.watcher.Close()
- if err := sm.connect(topodatapb.TabletType_PRIMARY); err != nil {
+ if err := sm.connect(topodatapb.TabletType_PRIMARY, false); err != nil {
return err
}
@@ -500,7 +500,7 @@ func (sm *stateManager) serveNonPrimary(wantTabletType topodatapb.TabletType) er
sm.se.MakeNonPrimary()
sm.hs.MakeNonPrimary()
- if err := sm.connect(wantTabletType); err != nil {
+ if err := sm.connect(wantTabletType, true); err != nil {
return err
}
@@ -518,7 +518,7 @@ func (sm *stateManager) unserveNonPrimary(wantTabletType topodatapb.TabletType)
sm.se.MakeNonPrimary()
sm.hs.MakeNonPrimary()
- if err := sm.connect(wantTabletType); err != nil {
+ if err := sm.connect(wantTabletType, false); err != nil {
return err
}
@@ -528,8 +528,8 @@ func (sm *stateManager) unserveNonPrimary(wantTabletType topodatapb.TabletType)
return nil
}
-func (sm *stateManager) connect(tabletType topodatapb.TabletType) error {
- if err := sm.se.EnsureConnectionAndDB(tabletType); err != nil {
+func (sm *stateManager) connect(tabletType topodatapb.TabletType, serving bool) error {
+ if err := sm.se.EnsureConnectionAndDB(tabletType, serving); err != nil {
return err
}
if err := sm.se.Open(); err != nil {
diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go
index f70e77de710..df819c6f05c 100644
--- a/go/vt/vttablet/tabletserver/state_manager_test.go
+++ b/go/vt/vttablet/tabletserver/state_manager_test.go
@@ -809,7 +809,7 @@ type testSchemaEngine struct {
failMySQL bool
}
-func (te *testSchemaEngine) EnsureConnectionAndDB(tabletType topodatapb.TabletType) error {
+func (te *testSchemaEngine) EnsureConnectionAndDB(topodatapb.TabletType, bool) error {
if te.failMySQL {
te.failMySQL = false
return errors.New("intentional error")
diff --git a/go/vt/vttablet/tabletserver/stateful_connection.go b/go/vt/vttablet/tabletserver/stateful_connection.go
index 91d51677241..10fc763984f 100644
--- a/go/vt/vttablet/tabletserver/stateful_connection.go
+++ b/go/vt/vttablet/tabletserver/stateful_connection.go
@@ -313,11 +313,12 @@ func (sc *StatefulConnection) getUsername() string {
return callerid.GetUsername(sc.reservedProps.ImmediateCaller)
}
-func (sc *StatefulConnection) ApplySetting(ctx context.Context, setting *smartconnpool.Setting) error {
+// ApplySetting returns whether the settings where applied or not. It also returns an error, if encountered.
+func (sc *StatefulConnection) ApplySetting(ctx context.Context, setting *smartconnpool.Setting) (bool, error) {
if sc.dbConn.Conn.Setting() == setting {
- return nil
+ return false, nil
}
- return sc.dbConn.Conn.ApplySetting(ctx, setting)
+ return true, sc.dbConn.Conn.ApplySetting(ctx, setting)
}
func (sc *StatefulConnection) resetExpiryTime() {
diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go
index 158f40d5202..994999f2368 100644
--- a/go/vt/vttablet/tabletserver/tabletenv/config.go
+++ b/go/vt/vttablet/tabletserver/tabletenv/config.go
@@ -156,8 +156,12 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) {
fs.BoolVar(¤tConfig.WatchReplication, "watch_replication_stream", false, "When enabled, vttablet will stream the MySQL replication stream from the local server, and use it to update schema when it sees a DDL.")
fs.BoolVar(¤tConfig.TrackSchemaVersions, "track_schema_versions", false, "When enabled, vttablet will store versions of schemas at each position that a DDL is applied and allow retrieval of the schema corresponding to a position")
fs.Int64Var(¤tConfig.SchemaVersionMaxAgeSeconds, "schema-version-max-age-seconds", 0, "max age of schema version records to kept in memory by the vreplication historian")
- fs.BoolVar(¤tConfig.TwoPCEnable, "twopc_enable", defaultConfig.TwoPCEnable, "if the flag is on, 2pc is enabled. Other 2pc flags must be supplied.")
- SecondsVar(fs, ¤tConfig.TwoPCAbandonAge, "twopc_abandon_age", defaultConfig.TwoPCAbandonAge, "time in seconds. Any unresolved transaction older than this time will be sent to the coordinator to be resolved.")
+
+ _ = fs.Bool("twopc_enable", true, "TwoPC is enabled")
+ _ = fs.MarkDeprecated("twopc_enable", "TwoPC is always enabled, the transaction abandon age can be configured")
+ flagutil.FloatDuration(fs, ¤tConfig.TwoPCAbandonAge, "twopc_abandon_age", defaultConfig.TwoPCAbandonAge,
+ "Any unresolved transaction older than this time will be sent to the coordinator to be resolved. NOTE: Providing time as seconds (float64) is deprecated. Use time.Duration format (e.g., '1s', '2m', '1h').")
+
// Tx throttler config
flagutil.DualFormatBoolVar(fs, ¤tConfig.EnableTxThrottler, "enable_tx_throttler", defaultConfig.EnableTxThrottler, "If true replication-lag-based throttling on transactions will be enabled.")
flagutil.DualFormatVar(fs, currentConfig.TxThrottlerConfig, "tx_throttler_config", "The configuration of the transaction throttler as a text-formatted throttlerdata.Configuration protocol buffer message.")
@@ -331,12 +335,11 @@ type TabletConfig struct {
ExternalConnections map[string]*dbconfigs.DBConfigs `json:"externalConnections,omitempty"`
- SanitizeLogMessages bool `json:"-"`
- StrictTableACL bool `json:"-"`
- EnableTableACLDryRun bool `json:"-"`
- TableACLExemptACL string `json:"-"`
- TwoPCEnable bool `json:"-"`
- TwoPCAbandonAge Seconds `json:"-"`
+ SanitizeLogMessages bool `json:"-"`
+ StrictTableACL bool `json:"-"`
+ EnableTableACLDryRun bool `json:"-"`
+ TableACLExemptACL string `json:"-"`
+ TwoPCAbandonAge time.Duration `json:"-"`
EnableTxThrottler bool `json:"-"`
TxThrottlerConfig *TxThrottlerConfigFlag `json:"-"`
@@ -1054,6 +1057,8 @@ var defaultConfig = TabletConfig{
},
EnablePerWorkloadTableMetrics: false,
+
+ TwoPCAbandonAge: 15 * time.Minute,
}
// defaultTxThrottlerConfig returns the default TxThrottlerConfigFlag object based on
diff --git a/go/vt/vttablet/tabletserver/tabletenv/seconds.go b/go/vt/vttablet/tabletserver/tabletenv/seconds.go
deleted file mode 100644
index ae11121f2de..00000000000
--- a/go/vt/vttablet/tabletserver/tabletenv/seconds.go
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
-Copyright 2020 The Vitess Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tabletenv
-
-import (
- "time"
-
- "github.com/spf13/pflag"
-)
-
-// Seconds provides convenience functions for extracting
-// duration from float64 seconds values.
-type Seconds float64
-
-// SecondsVar is like a flag.Float64Var, but it works for Seconds.
-func SecondsVar(fs *pflag.FlagSet, p *Seconds, name string, value Seconds, usage string) {
- fs.Float64Var((*float64)(p), name, float64(value), usage)
-}
-
-// Get converts Seconds to time.Duration
-func (s Seconds) Get() time.Duration {
- return time.Duration(s * Seconds(1*time.Second))
-}
-
-// Set sets the value from time.Duration
-func (s *Seconds) Set(d time.Duration) {
- *s = Seconds(d) / Seconds(1*time.Second)
-}
diff --git a/go/vt/vttablet/tabletserver/tabletenv/seconds_test.go b/go/vt/vttablet/tabletserver/tabletenv/seconds_test.go
deleted file mode 100644
index dc09a3f419f..00000000000
--- a/go/vt/vttablet/tabletserver/tabletenv/seconds_test.go
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
-Copyright 2020 The Vitess Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package tabletenv
-
-import (
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "vitess.io/vitess/go/yaml2"
-)
-
-func TestSecondsYaml(t *testing.T) {
- type testSecond struct {
- Value Seconds `json:"value"`
- }
-
- ts := testSecond{
- Value: 1,
- }
- gotBytes, err := yaml2.Marshal(&ts)
- require.NoError(t, err)
- wantBytes := "value: 1\n"
- assert.Equal(t, wantBytes, string(gotBytes))
-
- var gotts testSecond
- err = yaml2.Unmarshal([]byte(wantBytes), &gotts)
- require.NoError(t, err)
- assert.Equal(t, ts, gotts)
-}
-
-func TestSecondsGetSet(t *testing.T) {
- var val Seconds
- val.Set(2 * time.Second)
- assert.Equal(t, Seconds(2), val)
- assert.Equal(t, 2*time.Second, val.Get())
-}
diff --git a/go/vt/vttablet/tabletserver/tabletenv/stats.go b/go/vt/vttablet/tabletserver/tabletenv/stats.go
index 1ad93532719..52bb6a5a3b0 100644
--- a/go/vt/vttablet/tabletserver/tabletenv/stats.go
+++ b/go/vt/vttablet/tabletserver/tabletenv/stats.go
@@ -34,7 +34,6 @@ type Stats struct {
ErrorCounters *stats.CountersWithSingleLabel
InternalErrors *stats.CountersWithSingleLabel
Warnings *stats.CountersWithSingleLabel
- Unresolved *stats.GaugesWithSingleLabel // For now, only Prepares are tracked
UserTableQueryCount *stats.CountersWithMultiLabels // Per CallerID/table counts
UserTableQueryTimesNs *stats.CountersWithMultiLabels // Per CallerID/table latencies
UserTransactionCount *stats.CountersWithMultiLabels // Per CallerID transaction counts
@@ -49,6 +48,11 @@ type Stats struct {
UserReservedTimesNs *stats.CountersWithSingleLabel // Per CallerID reserved connection duration
QueryTimingsByTabletType *servenv.TimingsWrapper // Query timings split by current tablet type
+
+ // Atomic Transactions
+ Unresolved *stats.GaugesWithSingleLabel
+ CommitPreparedFail *stats.CountersWithSingleLabel
+ RedoPreparedFail *stats.CountersWithSingleLabel
}
// NewStats instantiates a new set of stats scoped by exporter.
@@ -83,7 +87,6 @@ func NewStats(exporter *servenv.Exporter) *Stats {
),
InternalErrors: exporter.NewCountersWithSingleLabel("InternalErrors", "Internal component errors", "type", "Task", "StrayTransactions", "Panic", "HungQuery", "Schema", "TwopcCommit", "TwopcResurrection", "WatchdogFail", "Messages"),
Warnings: exporter.NewCountersWithSingleLabel("Warnings", "Warnings", "type", "ResultsExceeded"),
- Unresolved: exporter.NewGaugesWithSingleLabel("Unresolved", "Unresolved items", "item_type", "Prepares"),
UserTableQueryCount: exporter.NewCountersWithMultiLabels("UserTableQueryCount", "Queries received for each CallerID/table combination", []string{"TableName", "CallerID", "Type"}),
UserTableQueryTimesNs: exporter.NewCountersWithMultiLabels("UserTableQueryTimesNs", "Total latency for each CallerID/table combination", []string{"TableName", "CallerID", "Type"}),
UserTransactionCount: exporter.NewCountersWithMultiLabels("UserTransactionCount", "transactions received for each CallerID", []string{"CallerID", "Conclusion"}),
@@ -98,6 +101,10 @@ func NewStats(exporter *servenv.Exporter) *Stats {
UserReservedTimesNs: exporter.NewCountersWithSingleLabel("UserReservedTimesNs", "Total reserved connection latency for each CallerID", "CallerID"),
QueryTimingsByTabletType: exporter.NewTimings("QueryTimingsByTabletType", "Query timings broken down by active tablet type", "TabletType"),
+
+ Unresolved: exporter.NewGaugesWithSingleLabel("UnresolvedTransaction", "Current unresolved transactions", "ManagerType"),
+ CommitPreparedFail: exporter.NewCountersWithSingleLabel("CommitPreparedFail", "failed prepared transactions commit", "FailureType"),
+ RedoPreparedFail: exporter.NewCountersWithSingleLabel("RedoPreparedFail", "failed prepared transactions on redo", "FailureType"),
}
stats.QPSRates = exporter.NewRates("QPS", stats.QueryTimings, 15*60/5, 5*time.Second)
return stats
diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go
index 4936516be72..847de25eb02 100644
--- a/go/vt/vttablet/tabletserver/tabletserver.go
+++ b/go/vt/vttablet/tabletserver/tabletserver.go
@@ -510,7 +510,14 @@ func (tsv *TabletServer) Begin(ctx context.Context, target *querypb.Target, opti
return tsv.begin(ctx, target, nil, 0, nil, options)
}
-func (tsv *TabletServer) begin(ctx context.Context, target *querypb.Target, savepointQueries []string, reservedID int64, settings []string, options *querypb.ExecuteOptions) (state queryservice.TransactionState, err error) {
+func (tsv *TabletServer) begin(
+ ctx context.Context,
+ target *querypb.Target,
+ postBeginQueries []string,
+ reservedID int64,
+ settings []string,
+ options *querypb.ExecuteOptions,
+) (state queryservice.TransactionState, err error) {
state.TabletAlias = tsv.alias
err = tsv.execRequest(
ctx, tsv.loadQueryTimeoutWithOptions(options),
@@ -528,12 +535,43 @@ func (tsv *TabletServer) begin(ctx context.Context, target *querypb.Target, save
return err
}
}
- transactionID, beginSQL, sessionStateChanges, err := tsv.te.Begin(ctx, savepointQueries, reservedID, connSetting, options)
+ transactionID, beginSQL, sessionStateChanges, err := tsv.te.Begin(ctx, reservedID, connSetting, options)
state.TransactionID = transactionID
state.SessionStateChanges = sessionStateChanges
logStats.TransactionID = transactionID
logStats.ReservedID = reservedID
+ if err != nil {
+ return err
+ }
+
+ targetType, err := tsv.resolveTargetType(ctx, target)
+ if err != nil {
+ return err
+ }
+ for _, query := range postBeginQueries {
+ plan, err := tsv.qe.GetPlan(ctx, logStats, query, true)
+ if err != nil {
+ return err
+ }
+
+ qre := &QueryExecutor{
+ ctx: ctx,
+ query: query,
+ connID: transactionID,
+ options: options,
+ plan: plan,
+ logStats: logStats,
+ tsv: tsv,
+ targetTabletType: targetType,
+ setting: connSetting,
+ }
+ _, err = qre.Execute()
+ if err != nil {
+ return err
+ }
+ }
+
// Record the actual statements that were executed in the logStats.
// If nothing was actually executed, don't count the operation in
// the tablet metrics, and clear out the logStats Method so that
@@ -552,7 +590,7 @@ func (tsv *TabletServer) begin(ctx context.Context, target *querypb.Target, save
return err
},
)
- return state, err
+ return
}
func (tsv *TabletServer) getPriorityFromOptions(options *querypb.ExecuteOptions) int {
@@ -665,7 +703,7 @@ func (tsv *TabletServer) Prepare(ctx context.Context, target *querypb.Target, tr
"Prepare", "prepare", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
return txe.Prepare(transactionID, dtid)
},
)
@@ -678,7 +716,7 @@ func (tsv *TabletServer) CommitPrepared(ctx context.Context, target *querypb.Tar
"CommitPrepared", "commit_prepared", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
if DebugTwoPc {
commitPreparedDelayForTest(tsv)
}
@@ -694,7 +732,7 @@ func (tsv *TabletServer) RollbackPrepared(ctx context.Context, target *querypb.T
"RollbackPrepared", "rollback_prepared", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
return txe.RollbackPrepared(dtid, originalID)
},
)
@@ -727,7 +765,7 @@ func (tsv *TabletServer) CreateTransaction(ctx context.Context, target *querypb.
"CreateTransaction", "create_transaction", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
return txe.CreateTransaction(dtid, participants)
},
)
@@ -735,16 +773,18 @@ func (tsv *TabletServer) CreateTransaction(ctx context.Context, target *querypb.
// StartCommit atomically commits the transaction along with the
// decision to commit the associated 2pc transaction.
-func (tsv *TabletServer) StartCommit(ctx context.Context, target *querypb.Target, transactionID int64, dtid string) (err error) {
- return tsv.execRequest(
+func (tsv *TabletServer) StartCommit(ctx context.Context, target *querypb.Target, transactionID int64, dtid string) (state querypb.StartCommitState, err error) {
+ err = tsv.execRequest(
ctx, tsv.loadQueryTimeout(),
"StartCommit", "start_commit", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
- return txe.StartCommit(transactionID, dtid)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
+ state, err = txe.StartCommit(transactionID, dtid)
+ return err
},
)
+ return state, err
}
// SetRollback transitions the 2pc transaction to the Rollback state.
@@ -755,7 +795,7 @@ func (tsv *TabletServer) SetRollback(ctx context.Context, target *querypb.Target
"SetRollback", "set_rollback", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
return txe.SetRollback(dtid, transactionID)
},
)
@@ -769,7 +809,7 @@ func (tsv *TabletServer) ConcludeTransaction(ctx context.Context, target *queryp
"ConcludeTransaction", "conclude_transaction", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
return txe.ConcludeTransaction(dtid)
},
)
@@ -782,7 +822,7 @@ func (tsv *TabletServer) ReadTransaction(ctx context.Context, target *querypb.Ta
"ReadTransaction", "read_transaction", nil,
target, nil, true, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
metadata, err = txe.ReadTransaction(dtid)
return err
},
@@ -790,6 +830,21 @@ func (tsv *TabletServer) ReadTransaction(ctx context.Context, target *querypb.Ta
return metadata, err
}
+// GetTransactionInfo returns the data for the specified dtid.
+func (tsv *TabletServer) GetTransactionInfo(ctx context.Context, target *querypb.Target, dtid string) (resp *tabletmanagerdatapb.GetTransactionInfoResponse, err error) {
+ err = tsv.execRequest(
+ ctx, tsv.loadQueryTimeout(),
+ "GetTransactionInfo", "get_transaction_info", nil,
+ target, nil, true, /* allowOnShutdown */
+ func(ctx context.Context, logStats *tabletenv.LogStats) error {
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
+ resp, err = txe.GetTransactionInfo(dtid)
+ return err
+ },
+ )
+ return resp, err
+}
+
// UnresolvedTransactions returns the unresolved distributed transaction record.
func (tsv *TabletServer) UnresolvedTransactions(ctx context.Context, target *querypb.Target, abandonAgeSeconds int64) (transactions []*querypb.TransactionMetadata, err error) {
err = tsv.execRequest(
@@ -797,7 +852,7 @@ func (tsv *TabletServer) UnresolvedTransactions(ctx context.Context, target *que
"UnresolvedTransactions", "unresolved_transaction", nil,
target, nil, false, /* allowOnShutdown */
func(ctx context.Context, logStats *tabletenv.LogStats) error {
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, logStats)
+ txe := NewDTExecutor(ctx, logStats, tsv.te, tsv.qe, tsv.getShard)
transactions, err = txe.UnresolvedTransactions(time.Duration(abandonAgeSeconds) * time.Second)
return err
},
@@ -978,7 +1033,7 @@ func (tsv *TabletServer) streamExecute(ctx context.Context, target *querypb.Targ
}
// BeginExecute combines Begin and Execute.
-func (tsv *TabletServer) BeginExecute(ctx context.Context, target *querypb.Target, preQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, reservedID int64, options *querypb.ExecuteOptions) (queryservice.TransactionState, *sqltypes.Result, error) {
+func (tsv *TabletServer) BeginExecute(ctx context.Context, target *querypb.Target, postBeginQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, reservedID int64, options *querypb.ExecuteOptions) (queryservice.TransactionState, *sqltypes.Result, error) {
// Disable hot row protection in case of reserve connection.
if tsv.enableHotRowProtection && reservedID == 0 {
@@ -991,7 +1046,7 @@ func (tsv *TabletServer) BeginExecute(ctx context.Context, target *querypb.Targe
}
}
- state, err := tsv.begin(ctx, target, preQueries, reservedID, nil, options)
+ state, err := tsv.begin(ctx, target, postBeginQueries, reservedID, nil, options)
if err != nil {
return state, nil, err
}
@@ -1004,14 +1059,14 @@ func (tsv *TabletServer) BeginExecute(ctx context.Context, target *querypb.Targe
func (tsv *TabletServer) BeginStreamExecute(
ctx context.Context,
target *querypb.Target,
- preQueries []string,
+ postBeginQueries []string,
sql string,
bindVariables map[string]*querypb.BindVariable,
reservedID int64,
options *querypb.ExecuteOptions,
callback func(*sqltypes.Result) error,
) (queryservice.TransactionState, error) {
- state, err := tsv.begin(ctx, target, preQueries, reservedID, nil, options)
+ state, err := tsv.begin(ctx, target, postBeginQueries, reservedID, nil, options)
if err != nil {
return state, err
}
@@ -1237,8 +1292,8 @@ func (tsv *TabletServer) VStreamResults(ctx context.Context, target *querypb.Tar
}
// ReserveBeginExecute implements the QueryService interface
-func (tsv *TabletServer) ReserveBeginExecute(ctx context.Context, target *querypb.Target, preQueries []string, postBeginQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, options *querypb.ExecuteOptions) (state queryservice.ReservedTransactionState, result *sqltypes.Result, err error) {
- state, result, err = tsv.beginExecuteWithSettings(ctx, target, preQueries, postBeginQueries, sql, bindVariables, options)
+func (tsv *TabletServer) ReserveBeginExecute(ctx context.Context, target *querypb.Target, settings []string, postBeginQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, options *querypb.ExecuteOptions) (state queryservice.ReservedTransactionState, result *sqltypes.Result, err error) {
+ state, result, err = tsv.beginExecuteWithSettings(ctx, target, settings, postBeginQueries, sql, bindVariables, options)
// If there is an error and the error message is about allowing query in reserved connection only,
// then we do not return an error from here and continue to use the reserved connection path.
// This is specially for get_lock function call from vtgate that needs a reserved connection.
@@ -1266,12 +1321,35 @@ func (tsv *TabletServer) ReserveBeginExecute(ctx context.Context, target *queryp
return err
}
defer tsv.stats.QueryTimingsByTabletType.Record(targetType.String(), time.Now())
- connID, sessionStateChanges, err = tsv.te.ReserveBegin(ctx, options, preQueries, postBeginQueries)
+ connID, sessionStateChanges, err = tsv.te.ReserveBegin(ctx, options, settings)
+ logStats.TransactionID = connID
+ logStats.ReservedID = connID
if err != nil {
return err
}
- logStats.TransactionID = connID
- logStats.ReservedID = connID
+
+ for _, query := range postBeginQueries {
+ plan, err := tsv.qe.GetPlan(ctx, logStats, query, true)
+ if err != nil {
+ return err
+ }
+
+ qre := &QueryExecutor{
+ ctx: ctx,
+ query: query,
+ connID: connID,
+ options: options,
+ plan: plan,
+ logStats: logStats,
+ tsv: tsv,
+ targetTabletType: targetType,
+ }
+ _, err = qre.Execute()
+ if err != nil {
+ return err
+ }
+ }
+
return nil
},
)
@@ -1292,13 +1370,13 @@ func (tsv *TabletServer) ReserveBeginStreamExecute(
ctx context.Context,
target *querypb.Target,
settings []string,
- savepointQueries []string,
+ postBeginQueries []string,
sql string,
bindVariables map[string]*querypb.BindVariable,
options *querypb.ExecuteOptions,
callback func(*sqltypes.Result) error,
) (state queryservice.ReservedTransactionState, err error) {
- txState, err := tsv.begin(ctx, target, savepointQueries, 0, settings, options)
+ txState, err := tsv.begin(ctx, target, postBeginQueries, 0, settings, options)
if err != nil {
return txToReserveState(txState), err
}
@@ -1308,9 +1386,9 @@ func (tsv *TabletServer) ReserveBeginStreamExecute(
}
// ReserveExecute implements the QueryService interface
-func (tsv *TabletServer) ReserveExecute(ctx context.Context, target *querypb.Target, preQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, transactionID int64, options *querypb.ExecuteOptions) (state queryservice.ReservedState, result *sqltypes.Result, err error) {
+func (tsv *TabletServer) ReserveExecute(ctx context.Context, target *querypb.Target, settings []string, sql string, bindVariables map[string]*querypb.BindVariable, transactionID int64, options *querypb.ExecuteOptions) (state queryservice.ReservedState, result *sqltypes.Result, err error) {
- result, err = tsv.executeWithSettings(ctx, target, preQueries, sql, bindVariables, transactionID, options)
+ result, err = tsv.executeWithSettings(ctx, target, settings, sql, bindVariables, transactionID, options)
// If there is an error and the error message is about allowing query in reserved connection only,
// then we do not return an error from here and continue to use the reserved connection path.
// This is specially for get_lock function call from vtgate that needs a reserved connection.
@@ -1335,7 +1413,7 @@ func (tsv *TabletServer) ReserveExecute(ctx context.Context, target *querypb.Tar
return err
}
defer tsv.stats.QueryTimingsByTabletType.Record(targetType.String(), time.Now())
- state.ReservedID, err = tsv.te.Reserve(ctx, options, transactionID, preQueries)
+ state.ReservedID, err = tsv.te.Reserve(ctx, options, transactionID, settings)
if err != nil {
return err
}
@@ -1357,14 +1435,14 @@ func (tsv *TabletServer) ReserveExecute(ctx context.Context, target *querypb.Tar
func (tsv *TabletServer) ReserveStreamExecute(
ctx context.Context,
target *querypb.Target,
- preQueries []string,
+ settings []string,
sql string,
bindVariables map[string]*querypb.BindVariable,
transactionID int64,
options *querypb.ExecuteOptions,
callback func(*sqltypes.Result) error,
) (state queryservice.ReservedState, err error) {
- return state, tsv.streamExecute(ctx, target, sql, bindVariables, transactionID, 0, preQueries, options, callback)
+ return state, tsv.streamExecute(ctx, target, sql, bindVariables, transactionID, 0, settings, options, callback)
}
// Release implements the QueryService interface
@@ -1404,8 +1482,8 @@ func (tsv *TabletServer) executeWithSettings(ctx context.Context, target *queryp
return tsv.execute(ctx, target, sql, bindVariables, transactionID, 0, settings, options)
}
-func (tsv *TabletServer) beginExecuteWithSettings(ctx context.Context, target *querypb.Target, settings []string, savepointQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, options *querypb.ExecuteOptions) (queryservice.ReservedTransactionState, *sqltypes.Result, error) {
- txState, err := tsv.begin(ctx, target, savepointQueries, 0, settings, options)
+func (tsv *TabletServer) beginExecuteWithSettings(ctx context.Context, target *querypb.Target, settings []string, postBeginQueries []string, sql string, bindVariables map[string]*querypb.BindVariable, options *querypb.ExecuteOptions) (queryservice.ReservedTransactionState, *sqltypes.Result, error) {
+ txState, err := tsv.begin(ctx, target, postBeginQueries, 0, settings, options)
if err != nil {
return txToReserveState(txState), nil, err
}
@@ -1620,7 +1698,7 @@ func convertErrorCode(err error) vtrpcpb.Code {
sqlerror.ERTooLongString, sqlerror.ERDelayedInsertTableLocked, sqlerror.ERDupUnique, sqlerror.ERRequiresPrimaryKey, sqlerror.ERCantDoThisDuringAnTransaction, sqlerror.ERReadOnlyTransaction,
sqlerror.ERCannotAddForeign, sqlerror.ERNoReferencedRow, sqlerror.ERRowIsReferenced, sqlerror.ERCantUpdateWithReadLock, sqlerror.ERNoDefault, sqlerror.EROperandColumns,
sqlerror.ERSubqueryNo1Row, sqlerror.ERNonUpdateableTable, sqlerror.ERFeatureDisabled, sqlerror.ERDuplicatedValueInType, sqlerror.ERRowIsReferenced2,
- sqlerror.ErNoReferencedRow2, sqlerror.ERWarnDataOutOfRange:
+ sqlerror.ErNoReferencedRow2, sqlerror.ERWarnDataOutOfRange, sqlerror.ERInnodbIndexCorrupt:
errCode = vtrpcpb.Code_FAILED_PRECONDITION
case sqlerror.EROptionPreventsStatement:
errCode = vtrpcpb.Code_CLUSTER_EVENT
@@ -1645,7 +1723,7 @@ func convertErrorCode(err error) vtrpcpb.Code {
sqlerror.ERCantAggregate3Collations, sqlerror.ERCantAggregateNCollations, sqlerror.ERVariableIsNotStruct, sqlerror.ERUnknownCollation, sqlerror.ERWrongNameForIndex,
sqlerror.ERWrongNameForCatalog, sqlerror.ERBadFTColumn, sqlerror.ERTruncatedWrongValue, sqlerror.ERTooMuchAutoTimestampCols, sqlerror.ERInvalidOnUpdate, sqlerror.ERUnknownTimeZone,
sqlerror.ERInvalidCharacterString, sqlerror.ERIllegalReference, sqlerror.ERDerivedMustHaveAlias, sqlerror.ERTableNameNotAllowedHere, sqlerror.ERDataTooLong, sqlerror.ERDataOutOfRange,
- sqlerror.ERTruncatedWrongValueForField, sqlerror.ERIllegalValueForType:
+ sqlerror.ERTruncatedWrongValueForField, sqlerror.ERIllegalValueForType, sqlerror.ERWrongValue, sqlerror.ERWrongParamcountToNativeFct:
errCode = vtrpcpb.Code_INVALID_ARGUMENT
case sqlerror.ERSpecifiedAccessDenied:
errCode = vtrpcpb.Code_PERMISSION_DENIED
@@ -1804,7 +1882,7 @@ func (tsv *TabletServer) registerQueryListHandlers(queryLists []*QueryList) {
func (tsv *TabletServer) registerTwopczHandler() {
tsv.exporter.HandleFunc("/twopcz", func(w http.ResponseWriter, r *http.Request) {
ctx := tabletenv.LocalContext()
- txe := NewDTExecutor(ctx, tsv.te, tsv.qe, tabletenv.NewLogStats(ctx, "twopcz"))
+ txe := NewDTExecutor(ctx, tabletenv.NewLogStats(ctx, "twopcz"), tsv.te, tsv.qe, tsv.getShard)
twopczHandler(txe, w, r)
})
}
@@ -1963,7 +2041,11 @@ func (tsv *TabletServer) StreamPoolSize() int {
// SetTxPoolSize changes the tx pool size to the specified value.
func (tsv *TabletServer) SetTxPoolSize(ctx context.Context, val int) error {
- return tsv.te.txPool.scp.conns.SetCapacity(ctx, int64(val))
+ // TxPool manages two pools, one for normal connections and one for CLIENT_FOUND_ROWS capability enabled on the connections.
+ if err := tsv.te.txPool.scp.conns.SetCapacity(ctx, int64(val)); err != nil {
+ return err
+ }
+ return tsv.te.txPool.scp.foundRowsPool.SetCapacity(ctx, int64(val))
}
// TxPoolSize returns the tx pool size.
@@ -2093,3 +2175,7 @@ func skipQueryPlanCache(options *querypb.ExecuteOptions) bool {
}
return options.SkipQueryPlanCache || options.HasCreatedTempTables
}
+
+func (tsv *TabletServer) getShard() string {
+ return tsv.sm.Target().Shard
+}
diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go
index 97d948a4d0a..621941224a9 100644
--- a/go/vt/vttablet/tabletserver/tabletserver_test.go
+++ b/go/vt/vttablet/tabletserver/tabletserver_test.go
@@ -304,8 +304,8 @@ func TestTabletServerCreateTransaction(t *testing.T) {
defer closer()
target := querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- db.AddQueryPattern(fmt.Sprintf("insert into _vt\\.dt_state\\(dtid, state, time_created\\) values \\('aa', %d,.*", int(querypb.TransactionState_PREPARE)), &sqltypes.Result{})
- db.AddQueryPattern("insert into _vt\\.dt_participant\\(dtid, id, keyspace, shard\\) values \\('aa', 1,.*", &sqltypes.Result{})
+ db.AddQueryPattern(fmt.Sprintf("insert into _vt\\.dt_state\\(dtid, state, time_created\\) values \\(_binary'aa', %d,.*", int(querypb.TransactionState_PREPARE)), &sqltypes.Result{})
+ db.AddQueryPattern("insert into _vt\\.dt_participant\\(dtid, id, keyspace, shard\\) values \\(_binary'aa', 1,.*", &sqltypes.Result{})
err := tsv.CreateTransaction(ctx, &target, "aa", []*querypb.Target{{
Keyspace: "t1",
Shard: "0",
@@ -320,16 +320,18 @@ func TestTabletServerStartCommit(t *testing.T) {
defer closer()
target := querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- commitTransition := fmt.Sprintf("update _vt.dt_state set state = %d where dtid = 'aa' and state = %d", int(querypb.TransactionState_COMMIT), int(querypb.TransactionState_PREPARE))
+ commitTransition := fmt.Sprintf("update _vt.dt_state set state = %d where dtid = _binary'aa' and state = %d", int(querypb.TransactionState_COMMIT), int(querypb.TransactionState_PREPARE))
db.AddQuery(commitTransition, &sqltypes.Result{RowsAffected: 1})
txid := newTxForPrep(ctx, tsv)
- err := tsv.StartCommit(ctx, &target, txid, "aa")
+ state, err := tsv.StartCommit(ctx, &target, txid, "aa")
require.NoError(t, err)
+ assert.Equal(t, querypb.StartCommitState_Success, state, "StartCommit state")
db.AddQuery(commitTransition, &sqltypes.Result{})
txid = newTxForPrep(ctx, tsv)
- err = tsv.StartCommit(ctx, &target, txid, "aa")
+ state, err = tsv.StartCommit(ctx, &target, txid, "aa")
assert.EqualError(t, err, "could not transition to COMMIT: aa", "Prepare err")
+ assert.Equal(t, querypb.StartCommitState_Fail, state, "StartCommit state")
}
func TestTabletserverSetRollback(t *testing.T) {
@@ -339,7 +341,7 @@ func TestTabletserverSetRollback(t *testing.T) {
defer closer()
target := querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- rollbackTransition := fmt.Sprintf("update _vt.dt_state set state = %d where dtid = 'aa' and state = %d", int(querypb.TransactionState_ROLLBACK), int(querypb.TransactionState_PREPARE))
+ rollbackTransition := fmt.Sprintf("update _vt.dt_state set state = %d where dtid = _binary'aa' and state = %d", int(querypb.TransactionState_ROLLBACK), int(querypb.TransactionState_PREPARE))
db.AddQuery(rollbackTransition, &sqltypes.Result{RowsAffected: 1})
txid := newTxForPrep(ctx, tsv)
err := tsv.SetRollback(ctx, &target, "aa", txid)
@@ -358,7 +360,7 @@ func TestTabletServerReadTransaction(t *testing.T) {
defer closer()
target := querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = 'aa'", &sqltypes.Result{})
+ db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = _binary'aa'", &sqltypes.Result{})
got, err := tsv.ReadTransaction(ctx, &target, "aa")
require.NoError(t, err)
want := &querypb.TransactionMetadata{}
@@ -376,8 +378,8 @@ func TestTabletServerReadTransaction(t *testing.T) {
sqltypes.NewVarBinary("1"),
}},
}
- db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = 'aa'", txResult)
- db.AddQuery("select keyspace, shard from _vt.dt_participant where dtid = 'aa'", &sqltypes.Result{
+ db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = _binary'aa'", txResult)
+ db.AddQuery("select keyspace, shard from _vt.dt_participant where dtid = _binary'aa'", &sqltypes.Result{
Fields: []*querypb.Field{
{Type: sqltypes.VarBinary},
{Type: sqltypes.VarBinary},
@@ -420,7 +422,7 @@ func TestTabletServerReadTransaction(t *testing.T) {
sqltypes.NewVarBinary("1"),
}},
}
- db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = 'aa'", txResult)
+ db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = _binary'aa'", txResult)
want.State = querypb.TransactionState_COMMIT
got, err = tsv.ReadTransaction(ctx, &target, "aa")
require.NoError(t, err)
@@ -438,7 +440,7 @@ func TestTabletServerReadTransaction(t *testing.T) {
sqltypes.NewVarBinary("1"),
}},
}
- db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = 'aa'", txResult)
+ db.AddQuery("select dtid, state, time_created from _vt.dt_state where dtid = _binary'aa'", txResult)
want.State = querypb.TransactionState_ROLLBACK
got, err = tsv.ReadTransaction(ctx, &target, "aa")
require.NoError(t, err)
@@ -452,8 +454,8 @@ func TestTabletServerConcludeTransaction(t *testing.T) {
defer closer()
target := querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- db.AddQuery("delete from _vt.dt_state where dtid = 'aa'", &sqltypes.Result{})
- db.AddQuery("delete from _vt.dt_participant where dtid = 'aa'", &sqltypes.Result{})
+ db.AddQuery("delete from _vt.dt_state where dtid = _binary'aa'", &sqltypes.Result{})
+ db.AddQuery("delete from _vt.dt_participant where dtid = _binary'aa'", &sqltypes.Result{})
err := tsv.ConcludeTransaction(ctx, &target, "aa")
require.NoError(t, err)
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_cache.go b/go/vt/vttablet/tabletserver/throttle/base/metric_cache.go
index 8695cb83229..faad65ca79e 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/metric_cache.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/metric_cache.go
@@ -49,6 +49,7 @@ import (
"github.com/patrickmn/go-cache"
"vitess.io/vitess/go/stats"
+ "vitess.io/vitess/go/vt/vttablet/tmclient"
)
// MetricsQueryType indicates the type of metrics query on MySQL backend. See following.
@@ -142,13 +143,13 @@ func (metric *ThrottleMetric) WithError(err error) *ThrottleMetric {
// ReadThrottleMetrics returns a metric for the given probe. Either by explicit query
// or via SHOW REPLICA STATUS
-func ReadThrottleMetrics(ctx context.Context, probe *Probe, metricsFunc func(context.Context) ThrottleMetrics) ThrottleMetrics {
+func ReadThrottleMetrics(ctx context.Context, probe *Probe, tmClient tmclient.TabletManagerClient, metricsFunc func(context.Context, tmclient.TabletManagerClient) ThrottleMetrics) ThrottleMetrics {
if metrics := getCachedThrottleMetrics(probe); metrics != nil {
return metrics
}
started := time.Now()
- throttleMetrics := metricsFunc(ctx)
+ throttleMetrics := metricsFunc(ctx, tmClient)
go func(metrics ThrottleMetrics, started time.Time) {
stats.GetOrNewGauge("ThrottlerProbesLatency", "probes latency").Set(time.Since(started).Nanoseconds())
diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_name.go b/go/vt/vttablet/tabletserver/throttle/base/metric_name.go
index 98e1288fb23..43bd2d17a8c 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/metric_name.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/metric_name.go
@@ -60,11 +60,14 @@ func (names MetricNames) Unique() MetricNames {
}
const (
- DefaultMetricName MetricName = "default"
- LagMetricName MetricName = "lag"
- ThreadsRunningMetricName MetricName = "threads_running"
- CustomMetricName MetricName = "custom"
- LoadAvgMetricName MetricName = "loadavg"
+ DefaultMetricName MetricName = "default"
+ LagMetricName MetricName = "lag"
+ ThreadsRunningMetricName MetricName = "threads_running"
+ CustomMetricName MetricName = "custom"
+ LoadAvgMetricName MetricName = "loadavg"
+ HistoryListLengthMetricName MetricName = "history_list_length"
+ MysqldLoadAvgMetricName MetricName = "mysqld-loadavg"
+ MysqldDatadirUsedRatioMetricName MetricName = "mysqld-datadir-used-ratio"
)
func (metric MetricName) DefaultScope() Scope {
diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go b/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go
index 9867ca18db3..c2e2b44b36f 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go
@@ -21,6 +21,9 @@ import (
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "vitess.io/vitess/go/textutil"
)
func TestAggregateName(t *testing.T) {
@@ -238,4 +241,27 @@ func TestKnownMetricNames(t *testing.T) {
assert.Contains(t, KnownMetricNames, LoadAvgMetricName)
assert.Contains(t, KnownMetricNames, CustomMetricName)
assert.Contains(t, KnownMetricNames, DefaultMetricName)
+ assert.Contains(t, KnownMetricNames, HistoryListLengthMetricName)
+ assert.Contains(t, KnownMetricNames, MysqldLoadAvgMetricName)
+ assert.Contains(t, KnownMetricNames, MysqldDatadirUsedRatioMetricName)
+}
+
+func TestKnownMetricNamesPascalCase(t *testing.T) {
+ expectCases := map[MetricName]string{
+ LagMetricName: "Lag",
+ ThreadsRunningMetricName: "ThreadsRunning",
+ LoadAvgMetricName: "Loadavg",
+ HistoryListLengthMetricName: "HistoryListLength",
+ CustomMetricName: "Custom",
+ DefaultMetricName: "Default",
+ MysqldLoadAvgMetricName: "MysqldLoadavg",
+ MysqldDatadirUsedRatioMetricName: "MysqldDatadirUsedRatio",
+ }
+ for _, metricName := range KnownMetricNames {
+ t.Run(metricName.String(), func(t *testing.T) {
+ expect, ok := expectCases[metricName]
+ require.True(t, ok)
+ assert.Equal(t, expect, textutil.PascalCase(metricName.String()))
+ })
+ }
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric.go
index 220dfa6bf60..88fbe2bdd13 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/self_metric.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric.go
@@ -21,15 +21,24 @@ import (
"fmt"
"strconv"
+ "vitess.io/vitess/go/vt/topo"
"vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
+ "vitess.io/vitess/go/vt/vttablet/tmclient"
)
+type SelfMetricReadParams struct {
+ Throttler metricsPublisher
+ Conn *connpool.Conn
+ TmClient tmclient.TabletManagerClient
+ TabletInfo *topo.TabletInfo
+}
+
type SelfMetric interface {
Name() MetricName
DefaultScope() Scope
DefaultThreshold() float64
RequiresConn() bool
- Read(ctx context.Context, throttler ThrottlerMetricsPublisher, conn *connpool.Conn) *ThrottleMetric
+ Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric
}
var (
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_custom_query.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_custom_query.go
index 585e63ea285..88f789e5dcd 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/self_metric_custom_query.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_custom_query.go
@@ -18,8 +18,6 @@ package base
import (
"context"
-
- "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
)
var _ SelfMetric = registerSelfMetric(&CustomQuerySelfMetric{})
@@ -43,6 +41,6 @@ func (m *CustomQuerySelfMetric) RequiresConn() bool {
return true
}
-func (m *CustomQuerySelfMetric) Read(ctx context.Context, throttler ThrottlerMetricsPublisher, conn *connpool.Conn) *ThrottleMetric {
- return ReadSelfMySQLThrottleMetric(ctx, conn, throttler.GetCustomMetricsQuery())
+func (m *CustomQuerySelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ return ReadSelfMySQLThrottleMetric(ctx, params.Conn, params.Throttler.GetCustomMetricsQuery())
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_default.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_default.go
index 8bce295da7c..97309fa6ea9 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/self_metric_default.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_default.go
@@ -19,8 +19,6 @@ package base
import (
"context"
"fmt"
-
- "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
)
var _ SelfMetric = registerSelfMetric(&DefaultSelfMetric{})
@@ -44,7 +42,7 @@ func (m *DefaultSelfMetric) RequiresConn() bool {
return false
}
-func (m *DefaultSelfMetric) Read(ctx context.Context, throttler ThrottlerMetricsPublisher, conn *connpool.Conn) *ThrottleMetric {
+func (m *DefaultSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
return &ThrottleMetric{
Err: fmt.Errorf("unexpected direct call to DefaultSelfMetric.Read"),
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go
new file mode 100644
index 00000000000..2696b1750ea
--- /dev/null
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package base
+
+import (
+ "context"
+ "math"
+ "sync/atomic"
+ "time"
+)
+
+var (
+ historyListLengthQuery = "select count as history_len from information_schema.INNODB_METRICS where name = 'trx_rseg_history_len'"
+
+ cachedHistoryListLengthMetric atomic.Pointer[ThrottleMetric]
+ historyListLengthCacheDuration = 5 * time.Second
+ historyListLengthDefaultThreshold = math.Pow10(9)
+)
+
+var _ SelfMetric = registerSelfMetric(&HistoryListLengthSelfMetric{})
+
+type HistoryListLengthSelfMetric struct {
+}
+
+func (m *HistoryListLengthSelfMetric) Name() MetricName {
+ return HistoryListLengthMetricName
+}
+
+func (m *HistoryListLengthSelfMetric) DefaultScope() Scope {
+ return SelfScope
+}
+
+func (m *HistoryListLengthSelfMetric) DefaultThreshold() float64 {
+ return historyListLengthDefaultThreshold
+}
+
+func (m *HistoryListLengthSelfMetric) RequiresConn() bool {
+ return true
+}
+
+func (m *HistoryListLengthSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ // This function will be called sequentially, and therefore does not need strong mutex protection. Still, we use atomics
+ // to ensure correctness in case an external goroutine tries to read the metric concurrently.
+ metric := cachedHistoryListLengthMetric.Load()
+ if metric != nil {
+ return metric
+ }
+ metric = ReadSelfMySQLThrottleMetric(ctx, params.Conn, historyListLengthQuery)
+ cachedHistoryListLengthMetric.Store(metric)
+ time.AfterFunc(historyListLengthCacheDuration, func() {
+ cachedHistoryListLengthMetric.Store(nil)
+ })
+ return metric
+}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_lag.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_lag.go
index dc25ee5622a..3d0e4beebe1 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/self_metric_lag.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_lag.go
@@ -23,7 +23,6 @@ import (
"vitess.io/vitess/go/constants/sidecar"
"vitess.io/vitess/go/vt/sqlparser"
- "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
)
var (
@@ -65,6 +64,6 @@ func (m *LagSelfMetric) RequiresConn() bool {
return true
}
-func (m *LagSelfMetric) Read(ctx context.Context, throttler ThrottlerMetricsPublisher, conn *connpool.Conn) *ThrottleMetric {
- return ReadSelfMySQLThrottleMetric(ctx, conn, m.GetQuery())
+func (m *LagSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ return ReadSelfMySQLThrottleMetric(ctx, params.Conn, m.GetQuery())
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_loadavg.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_loadavg.go
index 40a2878421a..2d880169020 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/self_metric_loadavg.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_loadavg.go
@@ -18,20 +18,16 @@ package base
import (
"context"
- "fmt"
- "os"
"runtime"
- "strconv"
- "strings"
+ "sync/atomic"
+ "time"
- "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
+ "vitess.io/vitess/go/osutil"
)
var (
- loadavgOnlyAvailableOnLinuxMetric = &ThrottleMetric{
- Scope: SelfScope,
- Err: fmt.Errorf("loadavg metric is only available on Linux"),
- }
+ cachedLoadAvgMetric atomic.Pointer[ThrottleMetric]
+ loadAvgCacheDuration = 1 * time.Second
)
var _ SelfMetric = registerSelfMetric(&LoadAvgSelfMetric{})
@@ -55,27 +51,26 @@ func (m *LoadAvgSelfMetric) RequiresConn() bool {
return false
}
-func (m *LoadAvgSelfMetric) Read(ctx context.Context, throttler ThrottlerMetricsPublisher, conn *connpool.Conn) *ThrottleMetric {
- if runtime.GOOS != "linux" {
- return loadavgOnlyAvailableOnLinuxMetric
+func (m *LoadAvgSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ // This function will be called sequentially, and therefore does not need strong mutex protection. Still, we use atomics
+ // to ensure correctness in case an external goroutine tries to read the metric concurrently.
+ metric := cachedLoadAvgMetric.Load()
+ if metric != nil {
+ return metric
}
- metric := &ThrottleMetric{
+ metric = &ThrottleMetric{
Scope: SelfScope,
}
- {
- content, err := os.ReadFile("/proc/loadavg")
- if err != nil {
- return metric.WithError(err)
- }
- fields := strings.Fields(string(content))
- if len(fields) == 0 {
- return metric.WithError(fmt.Errorf("unexpected /proc/loadavg content"))
- }
- loadAvg, err := strconv.ParseFloat(fields[0], 64)
- if err != nil {
- return metric.WithError(err)
- }
- metric.Value = loadAvg / float64(runtime.NumCPU())
+ val, err := osutil.LoadAvg()
+ if err != nil {
+ return metric.WithError(err)
}
+ metric.Value = val / float64(runtime.NumCPU())
+
+ cachedLoadAvgMetric.Store(metric)
+ time.AfterFunc(loadAvgCacheDuration, func() {
+ cachedLoadAvgMetric.Store(nil)
+ })
+
return metric
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld.go
new file mode 100644
index 00000000000..321837d86b4
--- /dev/null
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld.go
@@ -0,0 +1,156 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package base
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync/atomic"
+ "time"
+
+ "vitess.io/vitess/go/timer"
+
+ tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
+)
+
+var (
+ mysqlHostMetricsRpcTimeout = 5 * time.Second
+ mysqlHostMetricsRateLimit = 10 * time.Second
+ mysqlHostMetricsRateLimiter atomic.Pointer[timer.RateLimiter]
+ lastMySQLHostMetricsResponse atomic.Pointer[tabletmanagerdatapb.MysqlHostMetricsResponse]
+)
+
+// getMysqlMetricsRateLimiter returns a rate limiter that is active until the given context is cancelled.
+// This function will be called sequentially, but nonetheless it offers _some_ concurrent safety. Namely,
+// that a created rate limiter is guaranteed to be cleaned up
+func getMysqlMetricsRateLimiter(ctx context.Context, rateLimit time.Duration) *timer.RateLimiter {
+ rateLimiter := mysqlHostMetricsRateLimiter.Load()
+ if rateLimiter == nil {
+ rateLimiter = timer.NewRateLimiter(rateLimit)
+ go func() {
+ defer mysqlHostMetricsRateLimiter.Store(nil)
+ defer rateLimiter.Stop()
+ <-ctx.Done()
+ }()
+ mysqlHostMetricsRateLimiter.Store(rateLimiter)
+ }
+ return rateLimiter
+}
+
+// readMysqlHostMetrics reads MySQL host metrics sporadically from the tablet manager (which in turn reads
+// them from mysql deamon). The metrics are then cached, whether successful or not.
+// This idea is that is is very wasteful to read these metrics for every single query. E.g. right now the throttler
+// can issue 4 reads per second, which is wasteful to go through two RPCs to get the disk space usage for example. Even the load
+// average on the MySQL server is not that susceptible to change.
+func readMysqlHostMetrics(ctx context.Context, params *SelfMetricReadParams) error {
+ if params.TmClient == nil {
+ return fmt.Errorf("tmClient is nil")
+ }
+ if params.TabletInfo == nil {
+ return fmt.Errorf("tabletInfo is nil")
+ }
+ rateLimiter := getMysqlMetricsRateLimiter(ctx, mysqlHostMetricsRateLimit)
+ err := rateLimiter.Do(func() error {
+ ctx, cancel := context.WithTimeout(ctx, mysqlHostMetricsRpcTimeout)
+ defer cancel()
+
+ resp, err := params.TmClient.MysqlHostMetrics(ctx, params.TabletInfo.Tablet, &tabletmanagerdatapb.MysqlHostMetricsRequest{})
+ if err != nil {
+ return err
+ }
+ lastMySQLHostMetricsResponse.Store(resp)
+ return nil
+ })
+ return err
+}
+
+// getMysqlHostMetric gets a metric from the last read MySQL host metrics. The metric will either be directly read from
+// tablet manager (which then reads it from the mysql deamon), or from the cache.
+func getMysqlHostMetric(ctx context.Context, params *SelfMetricReadParams, mysqlHostMetricName string) *ThrottleMetric {
+ metric := &ThrottleMetric{
+ Scope: SelfScope,
+ }
+ if err := readMysqlHostMetrics(ctx, params); err != nil {
+ return metric.WithError(err)
+ }
+ resp := lastMySQLHostMetricsResponse.Load()
+ if resp == nil {
+ return metric.WithError(ErrNoResultYet)
+ }
+ mysqlMetric := resp.HostMetrics.Metrics[mysqlHostMetricName]
+ if mysqlMetric == nil {
+ return metric.WithError(ErrNoSuchMetric)
+ }
+ metric.Value = mysqlMetric.Value
+ if mysqlMetric.Error != nil {
+ metric.Err = errors.New(mysqlMetric.Error.Message)
+ }
+ return metric
+}
+
+var _ SelfMetric = registerSelfMetric(&MysqldLoadAvgSelfMetric{})
+var _ SelfMetric = registerSelfMetric(&MysqldDatadirUsedRatioSelfMetric{})
+
+// MysqldLoadAvgSelfMetric stands for the load average per cpu, on the MySQL host.
+type MysqldLoadAvgSelfMetric struct {
+}
+
+func (m *MysqldLoadAvgSelfMetric) Name() MetricName {
+ return MysqldLoadAvgMetricName
+}
+
+func (m *MysqldLoadAvgSelfMetric) DefaultScope() Scope {
+ return SelfScope
+}
+
+func (m *MysqldLoadAvgSelfMetric) DefaultThreshold() float64 {
+ return 1.0
+}
+
+func (m *MysqldLoadAvgSelfMetric) RequiresConn() bool {
+ return false
+}
+
+func (m *MysqldLoadAvgSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ return getMysqlHostMetric(ctx, params, "loadavg")
+}
+
+// MysqldDatadirUsedRatioSelfMetric stands for the disk space usage of the mount where MySQL's datadir is located.
+// Range: 0.0 (empty) - 1.0 (full)
+type MysqldDatadirUsedRatioSelfMetric struct {
+}
+
+func (m *MysqldDatadirUsedRatioSelfMetric) Name() MetricName {
+ return MysqldDatadirUsedRatioMetricName
+}
+
+func (m *MysqldDatadirUsedRatioSelfMetric) DefaultScope() Scope {
+ return SelfScope
+}
+
+func (m *MysqldDatadirUsedRatioSelfMetric) DefaultThreshold() float64 {
+ return 0.98
+}
+
+func (m *MysqldDatadirUsedRatioSelfMetric) RequiresConn() bool {
+ return false
+}
+
+func (m *MysqldDatadirUsedRatioSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ return getMysqlHostMetric(ctx, params, "datadir-used-ratio")
+}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld_test.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld_test.go
new file mode 100644
index 00000000000..39d3f3f5ec2
--- /dev/null
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld_test.go
@@ -0,0 +1,72 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package base
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetMysqlMetricsRateLimiter(t *testing.T) {
+ rateLimit := 10 * time.Millisecond
+ for i := range 3 {
+ testName := fmt.Sprintf("iteration %d", i)
+ t.Run(testName, func(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ {
+ rateLimiter := mysqlHostMetricsRateLimiter.Load()
+ assert.Nil(t, rateLimiter)
+ }
+ rateLimiter := getMysqlMetricsRateLimiter(ctx, rateLimit)
+ assert.NotNil(t, rateLimiter)
+ for range 5 {
+ r := getMysqlMetricsRateLimiter(ctx, rateLimit)
+ // Returning the same rate limiter
+ assert.Equal(t, rateLimiter, r)
+ }
+ val := 0
+ incr := func() error {
+ val++
+ return nil
+ }
+ for range 10 {
+ rateLimiter.Do(incr)
+ time.Sleep(2 * rateLimit)
+ }
+ assert.EqualValues(t, 10, val)
+ cancel()
+ // There can be a race condition where the rate limiter still emits one final tick after the context is cancelled.
+ // So we wait enough time to ensure that tick is "wasted".
+ time.Sleep(2 * rateLimit)
+ // Now that the rate limited was stopped (we invoked `cancel()`), its `Do()` should not invoke the function anymore.
+ for range 7 {
+ rateLimiter.Do(incr)
+ time.Sleep(time.Millisecond)
+ }
+ assert.EqualValues(t, 10, val) // Same "10" value as before.
+ {
+ rateLimiter := mysqlHostMetricsRateLimiter.Load()
+ assert.Nil(t, rateLimiter)
+ }
+ })
+ }
+}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_threads_running.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_threads_running.go
index 08f7d408d1c..cb59547a768 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/self_metric_threads_running.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_threads_running.go
@@ -18,8 +18,6 @@ package base
import (
"context"
-
- "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
)
var (
@@ -47,6 +45,6 @@ func (m *ThreadsRunningSelfMetric) RequiresConn() bool {
return true
}
-func (m *ThreadsRunningSelfMetric) Read(ctx context.Context, throttler ThrottlerMetricsPublisher, conn *connpool.Conn) *ThrottleMetric {
- return ReadSelfMySQLThrottleMetric(ctx, conn, threadsRunningMetricQuery)
+func (m *ThreadsRunningSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
+ return ReadSelfMySQLThrottleMetric(ctx, params.Conn, threadsRunningMetricQuery)
}
diff --git a/go/vt/vttablet/tabletserver/throttle/base/throttler_metrics_publisher.go b/go/vt/vttablet/tabletserver/throttle/base/throttler_metrics_publisher.go
index 1d2d4d0652c..10020af27e6 100644
--- a/go/vt/vttablet/tabletserver/throttle/base/throttler_metrics_publisher.go
+++ b/go/vt/vttablet/tabletserver/throttle/base/throttler_metrics_publisher.go
@@ -16,8 +16,8 @@ limitations under the License.
package base
-// ThrottlerMetricsPublisher is implemented by throttler.Throttler and is used by SelfMetric
+// metricsPublisher is implemented by throttler.Throttler and is used by SelfMetric
// implementations to query the throttler.
-type ThrottlerMetricsPublisher interface {
+type metricsPublisher interface {
GetCustomMetricsQuery() string
}
diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go
index ccdfcb2ce23..d7f43d85e9d 100644
--- a/go/vt/vttablet/tabletserver/throttle/check.go
+++ b/go/vt/vttablet/tabletserver/throttle/check.go
@@ -188,9 +188,9 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba
// Out of abundance of caution, we will protect against such a scenario.
return
}
- stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheck%s%sTotal", textutil.SingleWordCamel(metricScope.String()), textutil.SingleWordCamel(metricName.String())), "").Add(1)
+ stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheck%s%sTotal", textutil.PascalCase(metricScope.String()), textutil.PascalCase(metricName.String())), "").Add(1)
if !metricCheckResult.IsOK() {
- stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheck%s%sError", textutil.SingleWordCamel(metricScope.String()), textutil.SingleWordCamel(metricName.String())), "").Add(1)
+ stats.GetOrNewCounter(fmt.Sprintf("ThrottlerCheck%s%sError", textutil.PascalCase(metricScope.String()), textutil.PascalCase(metricName.String())), "").Add(1)
}
}(metricCheckResult)
}
@@ -249,7 +249,7 @@ func (check *ThrottlerCheck) localCheck(ctx context.Context, aggregatedMetricNam
check.throttler.markMetricHealthy(aggregatedMetricName)
}
if timeSinceHealthy, found := check.throttler.timeSinceMetricHealthy(aggregatedMetricName); found {
- go stats.GetOrNewGauge(fmt.Sprintf("ThrottlerCheck%sSecondsSinceHealthy", textutil.SingleWordCamel(scope.String())), fmt.Sprintf("seconds since last healthy check for %v", scope)).Set(int64(timeSinceHealthy.Seconds()))
+ go stats.GetOrNewGauge(fmt.Sprintf("ThrottlerCheck%sSecondsSinceHealthy", textutil.PascalCase(scope.String())), fmt.Sprintf("seconds since last healthy check for %v", scope)).Set(int64(timeSinceHealthy.Seconds()))
}
return checkResult
@@ -261,7 +261,7 @@ func (check *ThrottlerCheck) reportAggregated(aggregatedMetricName string, metri
return
}
if value, err := metricResult.Get(); err == nil {
- stats.GetOrNewGaugeFloat64(fmt.Sprintf("ThrottlerAggregated%s%s", textutil.SingleWordCamel(scope.String()), textutil.SingleWordCamel(metricName.String())), fmt.Sprintf("aggregated value for %v", scope)).Set(value)
+ stats.GetOrNewGaugeFloat64(fmt.Sprintf("ThrottlerAggregated%s%s", textutil.PascalCase(scope.String()), textutil.PascalCase(metricName.String())), fmt.Sprintf("aggregated value for %v", scope)).Set(value)
}
}
diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go
index af7f59abb7e..839ba9d43b8 100644
--- a/go/vt/vttablet/tabletserver/throttle/throttler.go
+++ b/go/vt/vttablet/tabletserver/throttle/throttler.go
@@ -95,7 +95,6 @@ const (
DefaultThrottleRatio = 1.0
defaultReplicationLagQuery = "select unix_timestamp(now(6))-max(ts/1000000000) as replication_lag from %s.heartbeat"
- threadsRunningQuery = "show global status like 'threads_running'"
inventoryPrefix = "inventory/"
throttlerConfigPrefix = "config/"
@@ -137,6 +136,7 @@ type Throttler struct {
keyspace string
shard string
tabletAlias *topodatapb.TabletAlias
+ tabletInfo atomic.Pointer[topo.TabletInfo]
check *ThrottlerCheck
isEnabled atomic.Bool
@@ -190,7 +190,7 @@ type Throttler struct {
cancelEnableContext context.CancelFunc
throttledAppsMutex sync.Mutex
- readSelfThrottleMetrics func(context.Context) base.ThrottleMetrics // overwritten by unit test
+ readSelfThrottleMetrics func(context.Context, tmclient.TabletManagerClient) base.ThrottleMetrics // overwritten by unit test
}
// ThrottlerStatus published some status values from the throttler
@@ -262,8 +262,8 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv
}
throttler.StoreMetricsThreshold(base.RegisteredSelfMetrics[base.LagMetricName].DefaultThreshold())
- throttler.readSelfThrottleMetrics = func(ctx context.Context) base.ThrottleMetrics {
- return throttler.readSelfThrottleMetricsInternal(ctx)
+ throttler.readSelfThrottleMetrics = func(ctx context.Context, tmClient tmclient.TabletManagerClient) base.ThrottleMetrics {
+ return throttler.readSelfThrottleMetricsInternal(ctx, tmClient)
}
return throttler
}
@@ -338,6 +338,15 @@ func (throttler *Throttler) initConfig() {
// readThrottlerConfig proactively reads the throttler's config from SrvKeyspace in local topo
func (throttler *Throttler) readThrottlerConfig(ctx context.Context) (*topodatapb.ThrottlerConfig, error) {
+ // since we're reading from topo, let's seize this opportunity to read table info as well
+ if throttler.tabletInfo.Load() == nil {
+ if ti, err := throttler.ts.GetTablet(ctx, throttler.tabletAlias); err == nil {
+ throttler.tabletInfo.Store(ti)
+ } else {
+ log.Errorf("Throttler: error reading tablet info: %v", err)
+ }
+ }
+
srvks, err := throttler.ts.GetSrvKeyspace(ctx, throttler.tabletAlias.Cell, throttler.keyspace)
if err != nil {
return nil, err
@@ -804,7 +813,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) {
if throttler.IsOpen() {
// frequent
// Always collect self metrics:
- throttler.collectSelfMetrics(ctx)
+ throttler.collectSelfMetrics(ctx, tmClient)
if !throttler.isDormant() {
throttler.collectShardMetrics(ctx, tmClient)
}
@@ -869,7 +878,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) {
}()
}
-func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClient tmclient.TabletManagerClient, probe *base.Probe) (probeFunc func(context.Context) base.ThrottleMetrics) {
+func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, probe *base.Probe) (probeFunc func(context.Context, tmclient.TabletManagerClient) base.ThrottleMetrics) {
metricsWithError := func(err error) base.ThrottleMetrics {
metrics := base.ThrottleMetrics{}
for _, metricName := range base.KnownMetricNames {
@@ -882,7 +891,7 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie
}
return metrics
}
- return func(ctx context.Context) base.ThrottleMetrics {
+ return func(ctx context.Context, tmClient tmclient.TabletManagerClient) base.ThrottleMetrics {
// Some reasonable timeout, to ensure we release connections even if they're hanging (otherwise grpc-go keeps polling those connections forever)
ctx, cancel := context.WithTimeout(ctx, 4*activeCollectInterval)
defer cancel()
@@ -940,7 +949,7 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie
// readSelfThrottleMetricsInternal rreads all registsred self metrics on this tablet (or backend MySQL server).
// This is the actual place where metrics are read, to be later aggregated and/or propagated to other tablets.
-func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context) base.ThrottleMetrics {
+func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context, tmClient tmclient.TabletManagerClient) base.ThrottleMetrics {
result := make(base.ThrottleMetrics, len(base.RegisteredSelfMetrics))
writeMetric := func(metric *base.ThrottleMetric) {
select {
@@ -950,15 +959,20 @@ func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context)
}
}
readMetric := func(selfMetric base.SelfMetric) *base.ThrottleMetric {
- if !selfMetric.RequiresConn() {
- return selfMetric.Read(ctx, throttler, nil)
+ params := &base.SelfMetricReadParams{
+ Throttler: throttler,
+ TmClient: tmClient,
+ TabletInfo: throttler.tabletInfo.Load(),
}
- conn, err := throttler.pool.Get(ctx, nil)
- if err != nil {
- return &base.ThrottleMetric{Err: err}
+ if selfMetric.RequiresConn() {
+ conn, err := throttler.pool.Get(ctx, nil)
+ if err != nil {
+ return &base.ThrottleMetric{Err: err}
+ }
+ defer conn.Recycle()
+ params.Conn = conn.Conn
}
- defer conn.Recycle()
- return selfMetric.Read(ctx, throttler, conn.Conn)
+ return selfMetric.Read(ctx, params)
}
for metricName, selfMetric := range base.RegisteredSelfMetrics {
if metricName == base.DefaultMetricName {
@@ -975,7 +989,7 @@ func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context)
return result
}
-func (throttler *Throttler) collectSelfMetrics(ctx context.Context) {
+func (throttler *Throttler) collectSelfMetrics(ctx context.Context, tmClient tmclient.TabletManagerClient) {
probe := throttler.inventory.ClustersProbes[throttler.tabletAliasString()]
if probe == nil {
// probe not created yet
@@ -990,7 +1004,7 @@ func (throttler *Throttler) collectSelfMetrics(ctx context.Context) {
defer atomic.StoreInt64(&probe.QueryInProgress, 0)
// Throttler is probing its own tablet's metrics:
- _ = base.ReadThrottleMetrics(ctx, probe, throttler.readSelfThrottleMetrics)
+ _ = base.ReadThrottleMetrics(ctx, probe, tmClient, throttler.readSelfThrottleMetrics)
}()
}
@@ -1011,9 +1025,9 @@ func (throttler *Throttler) collectShardMetrics(ctx context.Context, tmClient tm
defer atomic.StoreInt64(&probe.QueryInProgress, 0)
// Throttler probing other tablets:
- throttleMetricFunc := throttler.generateTabletProbeFunction(base.ShardScope, tmClient, probe)
+ throttleMetricFunc := throttler.generateTabletProbeFunction(base.ShardScope, probe)
- throttleMetrics := base.ReadThrottleMetrics(ctx, probe, throttleMetricFunc)
+ throttleMetrics := base.ReadThrottleMetrics(ctx, probe, tmClient, throttleMetricFunc)
for _, metric := range throttleMetrics {
select {
case <-ctx.Done():
diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go
index fd7921899da..352e641fa35 100644
--- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go
+++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go
@@ -71,6 +71,24 @@ var (
Value: 2.718,
Err: nil,
},
+ base.HistoryListLengthMetricName: &base.ThrottleMetric{
+ Scope: base.SelfScope,
+ Alias: "",
+ Value: 5,
+ Err: nil,
+ },
+ base.MysqldLoadAvgMetricName: &base.ThrottleMetric{
+ Scope: base.SelfScope,
+ Alias: "",
+ Value: 0.3311,
+ Err: nil,
+ },
+ base.MysqldDatadirUsedRatioMetricName: &base.ThrottleMetric{
+ Scope: base.SelfScope,
+ Alias: "",
+ Value: 0.85,
+ Err: nil,
+ },
}
replicaMetrics = map[string]*MetricResult{
base.LagMetricName.String(): {
@@ -93,6 +111,21 @@ var (
ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
Value: 5.1,
},
+ base.HistoryListLengthMetricName.String(): {
+ StatusCode: http.StatusOK,
+ ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
+ Value: 6,
+ },
+ base.MysqldLoadAvgMetricName.String(): {
+ StatusCode: http.StatusOK,
+ ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
+ Value: 0.2211,
+ },
+ base.MysqldDatadirUsedRatioMetricName.String(): {
+ StatusCode: http.StatusOK,
+ ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
+ Value: 0.87,
+ },
}
nonPrimaryTabletType atomic.Int32
)
@@ -283,7 +316,7 @@ func newTestThrottler() *Throttler {
throttler.recentCheckDormantDiff = int64(throttler.dormantPeriod / recentCheckRateLimiterInterval)
throttler.recentCheckDiff = int64(3 * time.Second / recentCheckRateLimiterInterval)
- throttler.readSelfThrottleMetrics = func(ctx context.Context) base.ThrottleMetrics {
+ throttler.readSelfThrottleMetrics = func(ctx context.Context, tmClient tmclient.TabletManagerClient) base.ThrottleMetrics {
for _, metric := range selfMetrics {
go func() {
select {
@@ -1827,10 +1860,13 @@ func TestChecks(t *testing.T) {
assert.Equal(t, testAppName.String(), checkResult.AppName)
assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics))
- assert.EqualValues(t, 0.3, checkResult.Metrics[base.LagMetricName.String()].Value) // self lag value, because flags.Scope is set
- assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because flags.Scope is set
- assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because flags.Scope is set
- assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because flags.Scope is set
+ assert.EqualValues(t, 0.3, checkResult.Metrics[base.LagMetricName.String()].Value) // self lag value, because flags.Scope is set
+ assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because flags.Scope is set
+ assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because flags.Scope is set
+ assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because flags.Scope is set
+ assert.EqualValues(t, 5, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // self value, because flags.Scope is set
+ assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // self value, because flags.Scope is set
+ assert.EqualValues(t, 0.85, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // self value, because flags.Scope is set
for _, metric := range checkResult.Metrics {
assert.EqualValues(t, base.SelfScope.String(), metric.Scope)
}
@@ -1886,10 +1922,13 @@ func TestChecks(t *testing.T) {
assert.Equal(t, testAppName.String(), checkResult.AppName)
assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics))
- assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, because flags.Scope is set
- assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, because flags.Scope is set
- assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value, because flags.Scope is set
- assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set
+ assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, because flags.Scope is set
+ assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, because flags.Scope is set
+ assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value, because flags.Scope is set
+ assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set
+ assert.EqualValues(t, 6, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // shard value, because flags.Scope is set
+ assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set
+ assert.EqualValues(t, 0.87, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // shard value, because flags.Scope is set
for _, metric := range checkResult.Metrics {
assert.EqualValues(t, base.ShardScope.String(), metric.Scope)
}
@@ -1918,14 +1957,19 @@ func TestChecks(t *testing.T) {
assert.ErrorIs(t, checkResult.Error, base.ErrThresholdExceeded)
assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics))
- assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, because "shard" is the default scope for lag
- assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because "self" is the default scope for threads_running
- assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because "self" is the default scope for custom
- assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
+ assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, because "shard" is the default scope for lag
+ assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because "self" is the default scope for threads_running
+ assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because "self" is the default scope for custom
+ assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
+ assert.EqualValues(t, 5, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
+ assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
+ assert.EqualValues(t, 0.85, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
assert.EqualValues(t, base.ShardScope.String(), checkResult.Metrics[base.LagMetricName.String()].Scope)
assert.EqualValues(t, base.SelfScope.String(), checkResult.Metrics[base.ThreadsRunningMetricName.String()].Scope)
assert.EqualValues(t, base.SelfScope.String(), checkResult.Metrics[base.CustomMetricName.String()].Scope)
assert.EqualValues(t, base.SelfScope.String(), checkResult.Metrics[base.LoadAvgMetricName.String()].Scope)
+ assert.EqualValues(t, base.SelfScope.String(), checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Scope)
+ assert.EqualValues(t, base.SelfScope.String(), checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Scope)
})
})
t.Run("checks, defined scope masks explicit scope metrics", func(t *testing.T) {
@@ -1939,6 +1983,9 @@ func TestChecks(t *testing.T) {
base.MetricName("self/threads_running"),
base.MetricName("custom"),
base.MetricName("shard/loadavg"),
+ base.MetricName("shard/mysqld-loadavg"),
+ base.MetricName("self/history_list_length"),
+ base.MetricName("self/mysqld-datadir-used-ratio"),
base.MetricName("default"),
}
checkResult := throttler.Check(ctx, testAppName.String(), metricNames, flags)
@@ -1950,10 +1997,13 @@ func TestChecks(t *testing.T) {
assert.ErrorIs(t, checkResult.Error, base.ErrThresholdExceeded)
assert.Equal(t, len(metricNames), len(checkResult.Metrics))
- assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, even though scope name is in metric name
- assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, even though scope name is in metric name
- assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value because flags.Scope is set
- assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set
+ assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, even though scope name is in metric name
+ assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, even though scope name is in metric name
+ assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value because flags.Scope is set
+ assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set
+ assert.EqualValues(t, 6, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // shard value, even though scope name is in metric name
+ assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set
+ assert.EqualValues(t, 0.87, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // shard value, even though scope name is in metric name
for _, metric := range checkResult.Metrics {
assert.EqualValues(t, base.ShardScope.String(), metric.Scope)
}
@@ -2222,8 +2272,13 @@ func TestReplica(t *testing.T) {
base.DefaultMetricName:
assert.Error(t, metricResult.Error, "metricName=%v, value=%v, threshold=%v", metricName, metricResult.Value, metricResult.Threshold)
assert.ErrorIs(t, metricResult.Error, base.ErrThresholdExceeded)
- case base.ThreadsRunningMetricName:
+ case base.ThreadsRunningMetricName,
+ base.HistoryListLengthMetricName,
+ base.MysqldLoadAvgMetricName,
+ base.MysqldDatadirUsedRatioMetricName:
assert.NoError(t, metricResult.Error, "metricName=%v, value=%v, threshold=%v", metricName, metricResult.Value, metricResult.Threshold)
+ default:
+ assert.Fail(t, "unexpected metric", "name=%v", metricName)
}
}
})
diff --git a/go/vt/vttablet/tabletserver/twopc.go b/go/vt/vttablet/tabletserver/twopc.go
index 5eff30ce07e..e1212ebe627 100644
--- a/go/vt/vttablet/tabletserver/twopc.go
+++ b/go/vt/vttablet/tabletserver/twopc.go
@@ -27,6 +27,7 @@ import (
"vitess.io/vitess/go/vt/dbconnpool"
"vitess.io/vitess/go/vt/log"
querypb "vitess.io/vitess/go/vt/proto/query"
+ tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/sqlparser"
@@ -75,7 +76,9 @@ type TwoPC struct {
readPool *connpool.Pool
insertRedoTx *sqlparser.ParsedQuery
+ readRedoTx *sqlparser.ParsedQuery
insertRedoStmt *sqlparser.ParsedQuery
+ readRedoStmts *sqlparser.ParsedQuery
updateRedoTx *sqlparser.ParsedQuery
deleteRedoTx *sqlparser.ParsedQuery
deleteRedoStmt *sqlparser.ParsedQuery
@@ -105,9 +108,15 @@ func (tpc *TwoPC) initializeQueries() {
tpc.insertRedoTx = sqlparser.BuildParsedQuery(
"insert into %s.redo_state(dtid, state, time_created) values (%a, %a, %a)",
dbname, ":dtid", ":state", ":time_created")
+ tpc.readRedoTx = sqlparser.BuildParsedQuery(
+ "select state, time_created, message from %s.redo_state where dtid = %a",
+ dbname, ":dtid")
tpc.insertRedoStmt = sqlparser.BuildParsedQuery(
"insert into %s.redo_statement(dtid, id, statement) values %a",
dbname, ":vals")
+ tpc.readRedoStmts = sqlparser.BuildParsedQuery(
+ "select statement from %s.redo_statement where dtid = %a order by id",
+ dbname, ":dtid")
tpc.updateRedoTx = sqlparser.BuildParsedQuery(
"update %s.redo_state set state = %a, message = %a where dtid = %a",
dbname, ":state", ":message", ":dtid")
@@ -150,6 +159,18 @@ func (tpc *TwoPC) initializeQueries() {
dbname, ":time_created")
}
+// getStateString gets the redo state of the transaction as a string.
+func getStateString(st int) string {
+ switch st {
+ case RedoStateFailed:
+ return "FAILED"
+ case RedoStatePrepared:
+ return "PREPARED"
+ default:
+ return "UNKNOWN"
+ }
+}
+
// Open starts the TwoPC service.
func (tpc *TwoPC) Open(dbconfigs *dbconfigs.DBConfigs) error {
conn, err := dbconnpool.NewDBConnection(context.TODO(), dbconfigs.DbaWithDB())
@@ -171,7 +192,7 @@ func (tpc *TwoPC) Close() {
// SaveRedo saves the statements in the redo log using the supplied connection.
func (tpc *TwoPC) SaveRedo(ctx context.Context, conn *StatefulConnection, dtid string, queries []tx.Query) error {
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
"state": sqltypes.Int64BindVariable(RedoStatePrepared),
"time_created": sqltypes.Int64BindVariable(time.Now().UnixNano()),
}
@@ -202,7 +223,7 @@ func (tpc *TwoPC) SaveRedo(ctx context.Context, conn *StatefulConnection, dtid s
// UpdateRedo changes the state of the redo log for the dtid.
func (tpc *TwoPC) UpdateRedo(ctx context.Context, conn *StatefulConnection, dtid string, state int, message string) error {
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
"state": sqltypes.Int64BindVariable(int64(state)),
"message": sqltypes.StringBindVariable(message),
}
@@ -213,7 +234,7 @@ func (tpc *TwoPC) UpdateRedo(ctx context.Context, conn *StatefulConnection, dtid
// DeleteRedo deletes the redo log for the dtid.
func (tpc *TwoPC) DeleteRedo(ctx context.Context, conn *StatefulConnection, dtid string) error {
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
}
_, err := tpc.exec(ctx, conn, tpc.deleteRedoTx, bindVars)
if err != nil {
@@ -291,7 +312,7 @@ func (tpc *TwoPC) CountUnresolvedRedo(ctx context.Context, unresolvedTime time.T
// CreateTransaction saves the metadata of a 2pc transaction as Prepared.
func (tpc *TwoPC) CreateTransaction(ctx context.Context, conn *StatefulConnection, dtid string, participants []*querypb.Target) error {
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
"state": sqltypes.Int64BindVariable(int64(DTStatePrepare)),
"cur_time": sqltypes.Int64BindVariable(time.Now().UnixNano()),
}
@@ -324,7 +345,7 @@ func (tpc *TwoPC) CreateTransaction(ctx context.Context, conn *StatefulConnectio
// If the transaction is not a in the Prepare state, an error is returned.
func (tpc *TwoPC) Transition(ctx context.Context, conn *StatefulConnection, dtid string, state querypb.TransactionState) error {
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
"state": sqltypes.Int64BindVariable(int64(state)),
"prepare": sqltypes.Int64BindVariable(int64(querypb.TransactionState_PREPARE)),
}
@@ -341,7 +362,7 @@ func (tpc *TwoPC) Transition(ctx context.Context, conn *StatefulConnection, dtid
// DeleteTransaction deletes the metadata for the specified transaction.
func (tpc *TwoPC) DeleteTransaction(ctx context.Context, conn *StatefulConnection, dtid string) error {
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
}
_, err := tpc.exec(ctx, conn, tpc.deleteTransaction, bindVars)
if err != nil {
@@ -361,7 +382,7 @@ func (tpc *TwoPC) ReadTransaction(ctx context.Context, dtid string) (*querypb.Tr
result := &querypb.TransactionMetadata{}
bindVars := map[string]*querypb.BindVariable{
- "dtid": sqltypes.StringBindVariable(dtid),
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
}
qr, err := tpc.read(ctx, conn.Conn, tpc.readTransaction, bindVars)
if err != nil {
@@ -400,6 +421,47 @@ func (tpc *TwoPC) ReadTransaction(ctx context.Context, dtid string) (*querypb.Tr
return result, nil
}
+// GetTransactionInfo returns the data for the transaction.
+func (tpc *TwoPC) GetTransactionInfo(ctx context.Context, dtid string) (*tabletmanagerdatapb.GetTransactionInfoResponse, error) {
+ conn, err := tpc.readPool.Get(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer conn.Recycle()
+
+ result := &tabletmanagerdatapb.GetTransactionInfoResponse{}
+ bindVars := map[string]*querypb.BindVariable{
+ "dtid": sqltypes.BytesBindVariable([]byte(dtid)),
+ }
+ qr, err := tpc.read(ctx, conn.Conn, tpc.readRedoTx, bindVars)
+ if err != nil {
+ return nil, err
+ }
+ if len(qr.Rows) == 0 {
+ return result, nil
+ }
+ state, err := qr.Rows[0][0].ToInt()
+ if err != nil {
+ return nil, err
+ }
+ result.State = getStateString(state)
+ // A failure in time parsing will show up as a very old time,
+ // which is harmless.
+ tm, _ := qr.Rows[0][1].ToCastInt64()
+ result.TimeCreated = tm
+ result.Message = qr.Rows[0][2].ToString()
+
+ qr, err = tpc.read(ctx, conn.Conn, tpc.readRedoStmts, bindVars)
+ if err != nil {
+ return nil, err
+ }
+ result.Statements = make([]string, len(qr.Rows))
+ for idx, row := range qr.Rows {
+ result.Statements[idx] = row[0].ToString()
+ }
+ return result, nil
+}
+
// ReadAllTransactions returns info about all distributed transactions.
func (tpc *TwoPC) ReadAllTransactions(ctx context.Context) ([]*tx.DistributedTx, error) {
conn, err := tpc.readPool.Get(ctx, nil)
diff --git a/go/vt/vttablet/tabletserver/tx/api.go b/go/vt/vttablet/tabletserver/tx/api.go
index 48a1cc1107a..a7bc4389b89 100644
--- a/go/vt/vttablet/tabletserver/tx/api.go
+++ b/go/vt/vttablet/tabletserver/tx/api.go
@@ -21,19 +21,19 @@ import (
"strings"
"time"
+ "vitess.io/vitess/go/slice"
querypb "vitess.io/vitess/go/vt/proto/query"
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/servenv"
"vitess.io/vitess/go/vt/sqlparser"
+ "vitess.io/vitess/go/vt/vterrors"
)
type (
// ConnID as type int64
ConnID = int64
-
// DTID as type string
DTID = string
-
// EngineStateMachine is used to control the state the transactional engine -
// whether new connections and/or transactions are allowed or not.
EngineStateMachine interface {
@@ -42,10 +42,8 @@ type (
AcceptReadOnly() error
StopGently()
}
-
// ReleaseReason as type int
ReleaseReason int
-
// Properties contains all information that is related to the currently running
// transaction on the connection
Properties struct {
@@ -60,12 +58,17 @@ type (
Stats *servenv.TimingsWrapper
}
-)
-type Query struct {
- Sql string
- Tables []string
-}
+ // Query contains the query and involved tables executed inside transaction.
+ // A savepoint is represented by having only the Savepoint field set.
+ // This is used to rollback to a specific savepoint.
+ // The query log on commit, does not need to store the savepoint.
+ Query struct {
+ Savepoint string
+ Sql string
+ Tables []string
+ }
+)
const (
// TxClose - connection released on close.
@@ -130,6 +133,30 @@ func (p *Properties) RecordQueryDetail(query string, tables []string) {
})
}
+// RecordQueryDetail records the query and tables against this transaction.
+func (p *Properties) RecordSavePointDetail(savepoint string) {
+ if p == nil {
+ return
+ }
+ p.Queries = append(p.Queries, Query{
+ Savepoint: savepoint,
+ })
+}
+
+func (p *Properties) RollbackToSavepoint(savepoint string) error {
+ if p == nil {
+ return nil
+ }
+ for i, query := range p.Queries {
+ if query.Savepoint == savepoint {
+ p.Queries = p.Queries[:i]
+ return nil
+ }
+ }
+
+ return vterrors.VT13001(fmt.Sprintf("savepoint %s not found", savepoint))
+}
+
// RecordQuery records the query and extract tables against this transaction.
func (p *Properties) RecordQuery(query string, parser *sqlparser.Parser) {
if p == nil {
@@ -181,3 +208,12 @@ func (p *Properties) String(sanitize bool, parser *sqlparser.Parser) string {
printQueries(),
)
}
+
+func (p *Properties) GetQueries() []Query {
+ if p == nil {
+ return nil
+ }
+ return slice.Filter(p.Queries, func(q Query) bool {
+ return q.Sql != ""
+ })
+}
diff --git a/go/vt/vttablet/tabletserver/tx/api_test.go b/go/vt/vttablet/tabletserver/tx/api_test.go
new file mode 100644
index 00000000000..cefb04c0391
--- /dev/null
+++ b/go/vt/vttablet/tabletserver/tx/api_test.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tx
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "vitess.io/vitess/go/test/utils"
+)
+
+/*
+ TestRollbackToSavePointQueryDetails tests the rollback to savepoint query details
+
+s1
+q1
+s2
+r1
+q2
+q3
+s3
+s4
+q4
+q5
+r2 -- error
+r4
+*/
+func TestRollbackToSavePointQueryDetails(t *testing.T) {
+ p := &Properties{}
+ p.RecordSavePointDetail("s1")
+ p.RecordQueryDetail("select 1", nil)
+ p.RecordSavePointDetail("s2")
+ require.NoError(t, p.RollbackToSavepoint("s1"))
+ p.RecordQueryDetail("select 2", nil)
+ p.RecordQueryDetail("select 3", nil)
+ p.RecordSavePointDetail("s3")
+ p.RecordSavePointDetail("s4")
+ p.RecordQueryDetail("select 4", nil)
+ p.RecordQueryDetail("select 5", nil)
+ require.ErrorContains(t, p.RollbackToSavepoint("s2"), "savepoint s2 not found")
+ require.NoError(t, p.RollbackToSavepoint("s4"))
+
+ utils.MustMatch(t, p.GetQueries(), []Query{
+ {Sql: "select 2"},
+ {Sql: "select 3"},
+ })
+}
diff --git a/go/vt/vttablet/tabletserver/tx_engine.go b/go/vt/vttablet/tabletserver/tx_engine.go
index 42bec29dfa3..d0ca4ec498c 100644
--- a/go/vt/vttablet/tabletserver/tx_engine.go
+++ b/go/vt/vttablet/tabletserver/tx_engine.go
@@ -31,6 +31,7 @@ import (
querypb "vitess.io/vitess/go/vt/proto/query"
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/servenv"
+ "vitess.io/vitess/go/vt/sqlparser"
"vitess.io/vitess/go/vt/vterrors"
"vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
@@ -83,7 +84,6 @@ type TxEngine struct {
// 2. TabletControls have been set in the tablet record, and Query service is going to be disabled.
twopcAllowed []bool
shutdownGracePeriod time.Duration
- coordinatorAddress string
abandonAge time.Duration
ticks *timer.Timer
@@ -116,20 +116,19 @@ func NewTxEngine(env tabletenv.Env, dxNotifier func()) *TxEngine {
te.txPool = NewTxPool(env, limiter)
// We initially allow twoPC (handles vttablet restarts).
// We will disallow them for a few reasons -
- // 1. when a new tablet is promoted if semi-sync is turned off.
+ // 1. When a new tablet is promoted if semi-sync is turned off.
// 2. TabletControls have been set by a Resharding workflow.
te.twopcAllowed = make([]bool, TwoPCAllowed_Len)
for idx := range te.twopcAllowed {
te.twopcAllowed[idx] = true
}
- te.twopcEnabled = config.TwoPCEnable
- if te.twopcEnabled {
- if config.TwoPCAbandonAge <= 0 {
- log.Error("2PC abandon age not specified: Disabling 2PC")
- te.twopcEnabled = false
- }
+ te.twopcEnabled = true
+ if config.TwoPCAbandonAge <= 0 {
+ log.Error("2PC abandon age not specified: Disabling 2PC")
+ te.twopcEnabled = false
}
- te.abandonAge = config.TwoPCAbandonAge.Get()
+
+ te.abandonAge = config.TwoPCAbandonAge
te.ticks = timer.NewTimer(te.abandonAge / 2)
// Set the prepared pool capacity to something lower than
@@ -270,7 +269,7 @@ func (te *TxEngine) isTxPoolAvailable(addToWaitGroup func(int)) error {
// statement(s) used to execute the begin (if any).
//
// Subsequent statements can access the connection through the transaction id.
-func (te *TxEngine) Begin(ctx context.Context, savepointQueries []string, reservedID int64, setting *smartconnpool.Setting, options *querypb.ExecuteOptions) (int64, string, string, error) {
+func (te *TxEngine) Begin(ctx context.Context, reservedID int64, setting *smartconnpool.Setting, options *querypb.ExecuteOptions) (int64, string, string, error) {
span, ctx := trace.NewSpan(ctx, "TxEngine.Begin")
defer span.Finish()
@@ -285,7 +284,7 @@ func (te *TxEngine) Begin(ctx context.Context, savepointQueries []string, reserv
}
defer te.beginRequests.Done()
- conn, beginSQL, sessionStateChanges, err := te.txPool.Begin(ctx, options, te.state == AcceptingReadOnly, reservedID, savepointQueries, setting)
+ conn, beginSQL, sessionStateChanges, err := te.txPool.Begin(ctx, options, te.state == AcceptingReadOnly, reservedID, setting)
if err != nil {
return 0, "", "", err
}
@@ -454,6 +453,9 @@ func (te *TxEngine) prepareFromRedo() error {
allErrs = append(allErrs, vterrors.Wrapf(err, "dtid - %v", preparedTx.Dtid))
if prepFailed {
failedCounter++
+ te.env.Stats().RedoPreparedFail.Add("NonRetryable", 1)
+ } else {
+ te.env.Stats().RedoPreparedFail.Add("Retryable", 1)
}
} else {
preparedCounter++
@@ -480,9 +482,24 @@ func (te *TxEngine) prepareTx(ctx context.Context, preparedTx *tx.PreparedTx) (f
}
}()
+ // We need to check whether the first query is a SET query or not.
+ // If it is then we need to run it before we begin the transaction because
+ // some connection settings can't be modified after a transaction has started
+ // For example -
+ // mysql> begin;
+ // Query OK, 0 rows affected (0.00 sec)
+ // mysql> set @@transaction_isolation="read-committed";
+ // ERROR 1568 (25001): Transaction characteristics can't be changed while a transaction is in progress.
+ var settingsQuery string
+ firstQuery := preparedTx.Queries[0]
+ if sqlparser.Preview(firstQuery) == sqlparser.StmtSet {
+ settingsQuery = firstQuery
+ preparedTx.Queries = preparedTx.Queries[1:]
+ }
+
// We need to redo the prepared transactions using a dba user because MySQL might still be in read only mode.
var conn *StatefulConnection
- if conn, err = te.beginNewDbaConnection(ctx); err != nil {
+ if conn, err = te.beginNewDbaConnection(ctx, settingsQuery); err != nil {
return
}
@@ -516,7 +533,7 @@ func (te *TxEngine) checkErrorAndMarkFailed(ctx context.Context, dtid string, re
// Update the state of the transaction in the redo log.
// Retryable Error: Update the message with error message.
// Non-retryable Error: Along with message, update the state as RedoStateFailed.
- conn, _, _, err := te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
if err != nil {
log.Errorf("markFailed: Begin failed for dtid %s: %v", dtid, err)
return
@@ -580,14 +597,13 @@ func (te *TxEngine) startTransactionWatcher() {
ctx, cancel := context.WithTimeout(tabletenv.LocalContext(), te.abandonAge/4)
defer cancel()
- // Raise alerts on prepares that have been unresolved for too long.
- // Use 5x abandonAge to give opportunity for transaction coordinator to resolve these redo logs.
- count, err := te.twoPC.CountUnresolvedRedo(ctx, time.Now().Add(-te.abandonAge*5))
+ // Track unresolved redo logs.
+ count, err := te.twoPC.CountUnresolvedRedo(ctx, time.Now().Add(-te.abandonAge))
if err != nil {
te.env.Stats().InternalErrors.Add("RedoWatcherFail", 1)
log.Errorf("Error reading prepared transactions: %v", err)
}
- te.env.Stats().Unresolved.Set("Prepares", count)
+ te.env.Stats().Unresolved.Set("ResourceManager", count)
// Notify lingering distributed transactions.
count, err = te.twoPC.CountUnresolvedTransaction(ctx, time.Now().Add(-te.abandonAge))
@@ -596,6 +612,7 @@ func (te *TxEngine) startTransactionWatcher() {
log.Errorf("Error reading unresolved transactions: %v", err)
return
}
+ te.env.Stats().Unresolved.Set("MetadataManager", count)
if count > 0 {
te.dxNotify()
}
@@ -608,7 +625,7 @@ func (te *TxEngine) stopTransactionWatcher() {
}
// ReserveBegin creates a reserved connection, and in it opens a transaction
-func (te *TxEngine) ReserveBegin(ctx context.Context, options *querypb.ExecuteOptions, preQueries []string, savepointQueries []string) (int64, string, error) {
+func (te *TxEngine) ReserveBegin(ctx context.Context, options *querypb.ExecuteOptions, preQueries []string) (int64, string, error) {
span, ctx := trace.NewSpan(ctx, "TxEngine.ReserveBegin")
defer span.Finish()
err := te.isTxPoolAvailable(te.beginRequests.Add)
@@ -622,7 +639,7 @@ func (te *TxEngine) ReserveBegin(ctx context.Context, options *querypb.ExecuteOp
return 0, "", err
}
defer conn.UnlockUpdateTime()
- _, sessionStateChanges, err := te.txPool.begin(ctx, options, te.state == AcceptingReadOnly, conn, savepointQueries)
+ _, sessionStateChanges, err := te.txPool.begin(ctx, options, te.state == AcceptingReadOnly, conn)
if err != nil {
conn.Close()
conn.Release(tx.ConnInitFail)
@@ -707,12 +724,19 @@ func (te *TxEngine) Release(connID int64) error {
// beginNewDbaConnection gets a new dba connection and starts a transaction in it.
// This should only be used to redo prepared transactions. All the other writes should use the normal pool.
-func (te *TxEngine) beginNewDbaConnection(ctx context.Context) (*StatefulConnection, error) {
+func (te *TxEngine) beginNewDbaConnection(ctx context.Context, settingsQuery string) (*StatefulConnection, error) {
dbConn, err := connpool.NewConn(ctx, te.env.Config().DB.DbaWithDB(), nil, nil, te.env)
if err != nil {
return nil, err
}
+ // If we have a settings query that we need to apply, we do that before starting the transaction.
+ if settingsQuery != "" {
+ if _, err = dbConn.ExecOnce(ctx, settingsQuery, 1, false); err != nil {
+ return nil, err
+ }
+ }
+
sc := &StatefulConnection{
dbConn: &connpool.PooledConn{
Conn: dbConn,
@@ -720,7 +744,7 @@ func (te *TxEngine) beginNewDbaConnection(ctx context.Context) (*StatefulConnect
env: te.env,
}
- _, _, err = te.txPool.begin(ctx, nil, false, sc, nil)
+ _, _, err = te.txPool.begin(ctx, nil, false, sc)
return sc, err
}
diff --git a/go/vt/vttablet/tabletserver/tx_engine_test.go b/go/vt/vttablet/tabletserver/tx_engine_test.go
index a9958525587..cba7bf86e8f 100644
--- a/go/vt/vttablet/tabletserver/tx_engine_test.go
+++ b/go/vt/vttablet/tabletserver/tx_engine_test.go
@@ -62,11 +62,11 @@ func TestTxEngineClose(t *testing.T) {
// Normal close with timeout wait.
te.AcceptReadWrite()
- c, beginSQL, _, err := te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ c, beginSQL, _, err := te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
require.Equal(t, "begin", beginSQL)
c.Unlock()
- c, beginSQL, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ c, beginSQL, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
require.Equal(t, "begin", beginSQL)
c.Unlock()
@@ -78,7 +78,7 @@ func TestTxEngineClose(t *testing.T) {
// Immediate close.
te.AcceptReadOnly()
- c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
if err != nil {
t.Fatal(err)
}
@@ -90,7 +90,7 @@ func TestTxEngineClose(t *testing.T) {
// Normal close with short grace period.
te.shutdownGracePeriod = 25 * time.Millisecond
te.AcceptReadWrite()
- c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
c.Unlock()
start = time.Now()
@@ -101,7 +101,7 @@ func TestTxEngineClose(t *testing.T) {
// Normal close with short grace period, but pool gets empty early.
te.shutdownGracePeriod = 25 * time.Millisecond
te.AcceptReadWrite()
- c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
c.Unlock()
go func() {
@@ -117,7 +117,7 @@ func TestTxEngineClose(t *testing.T) {
// Immediate close, but connection is in use.
te.AcceptReadOnly()
- c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
go func() {
time.Sleep(100 * time.Millisecond)
@@ -138,7 +138,7 @@ func TestTxEngineClose(t *testing.T) {
te.AcceptReadWrite()
_, err = te.Reserve(ctx, &querypb.ExecuteOptions{}, 0, nil)
require.NoError(t, err)
- _, _, err = te.ReserveBegin(ctx, &querypb.ExecuteOptions{}, nil, nil)
+ _, _, err = te.ReserveBegin(ctx, &querypb.ExecuteOptions{}, nil)
require.NoError(t, err)
start = time.Now()
te.Close()
@@ -159,11 +159,11 @@ func TestTxEngineBegin(t *testing.T) {
for _, exec := range []func() (int64, string, error){
func() (int64, string, error) {
- tx, _, schemaStateChanges, err := te.Begin(ctx, nil, 0, nil, &querypb.ExecuteOptions{})
+ tx, _, schemaStateChanges, err := te.Begin(ctx, 0, nil, &querypb.ExecuteOptions{})
return tx, schemaStateChanges, err
},
func() (int64, string, error) {
- return te.ReserveBegin(ctx, &querypb.ExecuteOptions{}, nil, nil)
+ return te.ReserveBegin(ctx, &querypb.ExecuteOptions{}, nil)
},
} {
te.AcceptReadOnly()
@@ -204,7 +204,7 @@ func TestTxEngineRenewFails(t *testing.T) {
te := NewTxEngine(tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "TabletServerTest"), nil)
te.AcceptReadOnly()
options := &querypb.ExecuteOptions{}
- connID, _, err := te.ReserveBegin(ctx, options, nil, nil)
+ connID, _, err := te.ReserveBegin(ctx, options, nil)
require.NoError(t, err)
conn, err := te.txPool.GetAndLock(connID, "for test")
@@ -559,7 +559,7 @@ func startTx(te *TxEngine, writeTransaction bool) error {
} else {
options.TransactionIsolation = querypb.ExecuteOptions_CONSISTENT_SNAPSHOT_READ_ONLY
}
- _, _, _, err := te.Begin(context.Background(), nil, 0, nil, options)
+ _, _, _, err := te.Begin(context.Background(), 0, nil, options)
return err
}
@@ -577,7 +577,7 @@ func TestTxEngineFailReserve(t *testing.T) {
_, err := te.Reserve(ctx, options, 0, nil)
assert.EqualError(t, err, "tx engine can't accept new connections in state NotServing")
- _, _, err = te.ReserveBegin(ctx, options, nil, nil)
+ _, _, err = te.ReserveBegin(ctx, options, nil)
assert.EqualError(t, err, "tx engine can't accept new connections in state NotServing")
te.AcceptReadOnly()
@@ -586,14 +586,14 @@ func TestTxEngineFailReserve(t *testing.T) {
_, err = te.Reserve(ctx, options, 0, []string{"dummy_query"})
assert.EqualError(t, err, "unknown error: failed executing dummy_query (errno 1105) (sqlstate HY000) during query: dummy_query")
- _, _, err = te.ReserveBegin(ctx, options, []string{"dummy_query"}, nil)
+ _, _, err = te.ReserveBegin(ctx, options, []string{"dummy_query"})
assert.EqualError(t, err, "unknown error: failed executing dummy_query (errno 1105) (sqlstate HY000) during query: dummy_query")
nonExistingID := int64(42)
_, err = te.Reserve(ctx, options, nonExistingID, nil)
assert.EqualError(t, err, "transaction 42: not found (potential transaction timeout)")
- txID, _, _, err := te.Begin(ctx, nil, 0, nil, options)
+ txID, _, _, err := te.Begin(ctx, 0, nil, options)
require.NoError(t, err)
conn, err := te.txPool.GetAndLock(txID, "for test")
require.NoError(t, err)
@@ -613,8 +613,7 @@ func TestCheckReceivedError(t *testing.T) {
cfg := tabletenv.NewDefaultConfig()
cfg.DB = newDBConfigs(db)
env := tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "TabletServerTest")
- env.Config().TwoPCEnable = true
- env.Config().TwoPCAbandonAge = 5
+ env.Config().TwoPCAbandonAge = 5 * time.Second
te := NewTxEngine(env, nil)
te.AcceptReadWrite()
@@ -625,43 +624,43 @@ func TestCheckReceivedError(t *testing.T) {
}{{
receivedErr: vterrors.New(vtrpcpb.Code_DEADLINE_EXCEEDED, "deadline exceeded"),
retryable: true,
- expQuery: `update _vt.redo_state set state = 1, message = 'deadline exceeded' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 1, message = 'deadline exceeded' where dtid = _binary'aa'`,
}, {
receivedErr: vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "invalid argument"),
retryable: false,
- expQuery: `update _vt.redo_state set state = 0, message = 'invalid argument' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 0, message = 'invalid argument' where dtid = _binary'aa'`,
}, {
receivedErr: sqlerror.NewSQLError(sqlerror.ERLockDeadlock, sqlerror.SSLockDeadlock, "Deadlock found when trying to get lock; try restarting transaction"),
retryable: false,
- expQuery: `update _vt.redo_state set state = 0, message = 'Deadlock found when trying to get lock; try restarting transaction (errno 1213) (sqlstate 40001)' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 0, message = 'Deadlock found when trying to get lock; try restarting transaction (errno 1213) (sqlstate 40001)' where dtid = _binary'aa'`,
}, {
receivedErr: context.DeadlineExceeded,
retryable: true,
- expQuery: `update _vt.redo_state set state = 1, message = 'context deadline exceeded' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 1, message = 'context deadline exceeded' where dtid = _binary'aa'`,
}, {
receivedErr: context.Canceled,
retryable: true,
- expQuery: `update _vt.redo_state set state = 1, message = 'context canceled' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 1, message = 'context canceled' where dtid = _binary'aa'`,
}, {
receivedErr: sqlerror.NewSQLError(sqlerror.CRServerLost, sqlerror.SSUnknownSQLState, "Lost connection to MySQL server during query"),
retryable: true,
- expQuery: `update _vt.redo_state set state = 1, message = 'Lost connection to MySQL server during query (errno 2013) (sqlstate HY000)' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 1, message = 'Lost connection to MySQL server during query (errno 2013) (sqlstate HY000)' where dtid = _binary'aa'`,
}, {
receivedErr: sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "Malformed packet"),
retryable: false,
- expQuery: `update _vt.redo_state set state = 0, message = 'Malformed packet (errno 2027) (sqlstate HY000)' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 0, message = 'Malformed packet (errno 2027) (sqlstate HY000)' where dtid = _binary'aa'`,
}, {
receivedErr: sqlerror.NewSQLError(sqlerror.CRServerGone, sqlerror.SSUnknownSQLState, "Server has gone away"),
retryable: true,
- expQuery: `update _vt.redo_state set state = 1, message = 'Server has gone away (errno 2006) (sqlstate HY000)' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 1, message = 'Server has gone away (errno 2006) (sqlstate HY000)' where dtid = _binary'aa'`,
}, {
receivedErr: vterrors.New(vtrpcpb.Code_ABORTED, "Row count exceeded"),
retryable: false,
- expQuery: `update _vt.redo_state set state = 0, message = 'Row count exceeded' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 0, message = 'Row count exceeded' where dtid = _binary'aa'`,
}, {
receivedErr: errors.New("(errno 2013) (sqlstate HY000) lost connection"),
retryable: true,
- expQuery: `update _vt.redo_state set state = 1, message = '(errno 2013) (sqlstate HY000) lost connection' where dtid = 'aa'`,
+ expQuery: `update _vt.redo_state set state = 1, message = '(errno 2013) (sqlstate HY000) lost connection' where dtid = _binary'aa'`,
}}
for _, tc := range tcases {
@@ -721,3 +720,89 @@ func TestIsTwoPCAllowed(t *testing.T) {
})
}
}
+
+// TestPrepareTx tests prepareTx function in transaction engine.
+func TestPrepareTx(t *testing.T) {
+ testcases := []struct {
+ name string
+ preparedTx *tx.PreparedTx
+ requireFailure bool
+ errWanted string
+ queryLogWanted string
+ }{
+ {
+ name: "Success",
+ preparedTx: &tx.PreparedTx{
+ Queries: []string{
+ "insert into vitess_test (intval) values(40)",
+ "set @@time_zone='+10:30'",
+ "insert into vitess_test (intval) values(20)",
+ },
+ },
+ requireFailure: false,
+ errWanted: "",
+ queryLogWanted: "use `fakesqldb`;begin;insert into vitess_test (intval) values(40);set @@time_zone='+10:30';insert into vitess_test (intval) values(20)",
+ },
+ {
+ name: "Unretryable failure during query",
+ preparedTx: &tx.PreparedTx{
+ Queries: []string{
+ "insert into vitess_test (intval) values(40)",
+ "failing query",
+ "insert into vitess_test (intval) values(20)",
+ },
+ },
+ requireFailure: true,
+ errWanted: "(errno 1105) (sqlstate HY000)",
+ },
+ {
+ name: "Retryable failure during query",
+ preparedTx: &tx.PreparedTx{
+ Queries: []string{
+ "insert into vitess_test (intval) values(40)",
+ "retryable query",
+ "insert into vitess_test (intval) values(20)",
+ },
+ },
+ requireFailure: false,
+ errWanted: "Retryable error (errno 2002) (sqlstate HY000)",
+ },
+ {
+ name: "Success - Settings query in the beginning",
+ preparedTx: &tx.PreparedTx{
+ Queries: []string{
+ "set @@time_zone='+10:30'",
+ "insert into vitess_test (intval) values(40)",
+ "insert into vitess_test (intval) values(20)",
+ },
+ },
+ requireFailure: false,
+ errWanted: "",
+ queryLogWanted: "use `fakesqldb`;set @@time_zone='+10:30';begin;insert into vitess_test (intval) values(40);insert into vitess_test (intval) values(20)",
+ },
+ }
+ for _, tt := range testcases {
+ t.Run(tt.name, func(t *testing.T) {
+ db := setUpQueryExecutorTest(t)
+ defer db.Close()
+ db.AddQueryPattern(".*", &sqltypes.Result{})
+ db.AddRejectedQuery("failing query", assert.AnError)
+ db.AddRejectedQuery("retryable query", sqlerror.NewSQLError(sqlerror.CRConnectionError, "", "Retryable error"))
+ cfg := tabletenv.NewDefaultConfig()
+ cfg.DB = newDBConfigs(db)
+ cfg.TwoPCAbandonAge = 200 * time.Second
+ te := NewTxEngine(tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "TabletServerTest"), nil)
+ te.AcceptReadWrite()
+ db.ResetQueryLog()
+ failed, err := te.prepareTx(context.Background(), tt.preparedTx)
+ require.EqualValues(t, tt.requireFailure, failed)
+ if tt.errWanted != "" {
+ require.ErrorContains(t, err, tt.errWanted)
+ return
+ }
+ require.NoError(t, err)
+ require.EqualValues(t, 1, len(te.preparedPool.conns))
+ require.EqualValues(t, tt.queryLogWanted, db.QueryLog())
+ })
+ }
+}
diff --git a/go/vt/vttablet/tabletserver/tx_pool.go b/go/vt/vttablet/tabletserver/tx_pool.go
index 52f356e0cca..302a3d41050 100644
--- a/go/vt/vttablet/tabletserver/tx_pool.go
+++ b/go/vt/vttablet/tabletserver/tx_pool.go
@@ -40,9 +40,8 @@ import (
)
const (
- txLogInterval = 1 * time.Minute
- beginWithCSRO = "start transaction with consistent snapshot, read only"
- trackGtidQuery = "set session session_track_gtids = START_GTID"
+ txLogInterval = 1 * time.Minute
+ beginWithCSRO = "start transaction with consistent snapshot, read only"
)
var txIsolations = map[querypb.ExecuteOptions_TransactionIsolation]string{
@@ -230,7 +229,7 @@ func (tp *TxPool) Rollback(ctx context.Context, txConn *StatefulConnection) erro
// the statements (if any) executed to initiate the transaction. In autocommit
// mode the statement will be "".
// The connection returned is locked for the callee and its responsibility is to unlock the connection.
-func (tp *TxPool) Begin(ctx context.Context, options *querypb.ExecuteOptions, readOnly bool, reservedID int64, savepointQueries []string, setting *smartconnpool.Setting) (*StatefulConnection, string, string, error) {
+func (tp *TxPool) Begin(ctx context.Context, options *querypb.ExecuteOptions, readOnly bool, reservedID int64, setting *smartconnpool.Setting) (*StatefulConnection, string, string, error) {
span, ctx := trace.NewSpan(ctx, "TxPool.Begin")
defer span.Finish()
@@ -262,25 +261,28 @@ func (tp *TxPool) Begin(ctx context.Context, options *querypb.ExecuteOptions, re
if err != nil {
return nil, "", "", err
}
- sql, sessionStateChanges, err := tp.begin(ctx, options, readOnly, conn, savepointQueries)
+ sql, sessionStateChanges, err := tp.begin(ctx, options, readOnly, conn)
if err != nil {
conn.Close()
conn.Release(tx.ConnInitFail)
return nil, "", "", err
}
+ // If we have applied any settings on the connection, then we need to record the query
+ // in case we need to redo the transaction because of a failure.
+ if setting != nil {
+ conn.TxProperties().RecordQueryDetail(setting.ApplyQuery(), nil)
+ }
return conn, sql, sessionStateChanges, nil
}
-func (tp *TxPool) begin(ctx context.Context, options *querypb.ExecuteOptions, readOnly bool, conn *StatefulConnection, savepointQueries []string) (string, string, error) {
+func (tp *TxPool) begin(ctx context.Context, options *querypb.ExecuteOptions, readOnly bool, conn *StatefulConnection) (string, string, error) {
immediateCaller := callerid.ImmediateCallerIDFromContext(ctx)
effectiveCaller := callerid.EffectiveCallerIDFromContext(ctx)
- beginQueries, autocommit, sessionStateChanges, err := createTransaction(ctx, options, conn, readOnly, savepointQueries)
+ beginQueries, autocommit, sessionStateChanges, err := createTransaction(ctx, options, conn, readOnly)
if err != nil {
return "", "", err
}
-
conn.txProps = tp.NewTxProps(immediateCaller, effectiveCaller, autocommit)
-
return beginQueries, sessionStateChanges, nil
}
@@ -306,7 +308,6 @@ func createTransaction(
options *querypb.ExecuteOptions,
conn *StatefulConnection,
readOnly bool,
- savepointQueries []string,
) (beginQueries string, autocommitTransaction bool, sessionStateChanges string, err error) {
switch options.GetTransactionIsolation() {
case querypb.ExecuteOptions_CONSISTENT_SNAPSHOT_READ_ONLY:
@@ -344,12 +345,6 @@ func createTransaction(
default:
return "", false, "", vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] don't know how to open a transaction of this type: %v", options.GetTransactionIsolation())
}
-
- for _, savepoint := range savepointQueries {
- if _, err = conn.Exec(ctx, savepoint, 1, false); err != nil {
- return "", false, "", err
- }
- }
return
}
@@ -398,16 +393,6 @@ func createStartTxStmt(options *querypb.ExecuteOptions, readOnly bool) (string,
}
func handleConsistentSnapshotCase(ctx context.Context, conn *StatefulConnection) (beginSQL string, sessionStateChanges string, err error) {
- _, err = conn.execWithRetry(ctx, trackGtidQuery, 1, false)
- // We allow this to fail since this is a custom MySQL extension, but we return
- // then if this query was executed or not.
- //
- // Callers also can know because the sessionStateChanges will be empty for a snapshot
- // transaction and get GTID information in another (less efficient) way.
- if err == nil {
- beginSQL = trackGtidQuery + "; "
- }
-
isolationLevel := txIsolations[querypb.ExecuteOptions_CONSISTENT_SNAPSHOT_READ_ONLY]
execSQL, err := setIsolationLevel(ctx, conn, isolationLevel)
diff --git a/go/vt/vttablet/tabletserver/tx_pool_test.go b/go/vt/vttablet/tabletserver/tx_pool_test.go
index e80f1edb17f..22810d4c422 100644
--- a/go/vt/vttablet/tabletserver/tx_pool_test.go
+++ b/go/vt/vttablet/tabletserver/tx_pool_test.go
@@ -48,7 +48,7 @@ func TestTxPoolExecuteCommit(t *testing.T) {
sql := "select 'this is a query'"
// begin a transaction and then return the connection
- conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
id := conn.ReservedID()
@@ -83,7 +83,7 @@ func TestTxPoolExecuteRollback(t *testing.T) {
db, txPool, _, closer := setup(t)
defer closer()
- conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
defer conn.Release(tx.TxRollback)
@@ -104,7 +104,7 @@ func TestTxPoolExecuteRollbackOnClosedConn(t *testing.T) {
db, txPool, _, closer := setup(t)
defer closer()
- conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
defer conn.Release(tx.TxRollback)
@@ -125,9 +125,9 @@ func TestTxPoolRollbackNonBusy(t *testing.T) {
defer closer()
// start two transactions, and mark one of them as unused
- conn1, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn1, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
- conn2, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn2, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
conn2.Unlock() // this marks conn2 as NonBusy
@@ -154,7 +154,7 @@ func TestTxPoolTransactionIsolation(t *testing.T) {
db, txPool, _, closer := setup(t)
defer closer()
- c2, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{TransactionIsolation: querypb.ExecuteOptions_READ_COMMITTED}, false, 0, nil, nil)
+ c2, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{TransactionIsolation: querypb.ExecuteOptions_READ_COMMITTED}, false, 0, nil)
require.NoError(t, err)
c2.Release(tx.TxClose)
@@ -172,7 +172,7 @@ func TestTxPoolAutocommit(t *testing.T) {
// to mysql.
// This test is meaningful because if txPool.Begin were to send a BEGIN statement to the connection, it will fatal
// because is not in the list of expected queries (i.e db.AddQuery hasn't been called).
- conn1, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{TransactionIsolation: querypb.ExecuteOptions_AUTOCOMMIT}, false, 0, nil, nil)
+ conn1, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{TransactionIsolation: querypb.ExecuteOptions_AUTOCOMMIT}, false, 0, nil)
require.NoError(t, err)
// run a query to see it in the query log
@@ -204,7 +204,7 @@ func TestTxPoolBeginWithPoolConnectionError_Errno2006_Transient(t *testing.T) {
err := db.WaitForClose(2 * time.Second)
require.NoError(t, err)
- txConn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ txConn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err, "Begin should have succeeded after the retry in DBConn.Exec()")
txConn.Release(tx.TxCommit)
}
@@ -225,7 +225,7 @@ func primeTxPoolWithConnection(t *testing.T, ctx context.Context) (*fakesqldb.DB
// reused by subsequent transactions.
db.AddQuery("begin", &sqltypes.Result{})
db.AddQuery("rollback", &sqltypes.Result{})
- txConn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ txConn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
txConn.Release(tx.TxCommit)
@@ -248,7 +248,7 @@ func TestTxPoolBeginWithError(t *testing.T) {
}
ctxWithCallerID := callerid.NewContext(ctx, ef, im)
- _, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ _, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "error: rejected")
require.Equal(t, vtrpcpb.Code_UNKNOWN, vterrors.Code(err), "wrong error code for Begin error")
@@ -270,19 +270,6 @@ func TestTxPoolBeginWithError(t *testing.T) {
}, limiter.Actions())
}
-func TestTxPoolBeginWithPreQueryError(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- db, txPool, _, closer := setup(t)
- defer closer()
- db.AddRejectedQuery("pre_query", errRejected)
- _, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, []string{"pre_query"}, nil)
- require.Error(t, err)
- require.Contains(t, err.Error(), "error: rejected")
- require.Equal(t, vtrpcpb.Code_UNKNOWN, vterrors.Code(err), "wrong error code for Begin error")
-}
-
func TestTxPoolCancelledContextError(t *testing.T) {
// given
db, txPool, _, closer := setup(t)
@@ -291,7 +278,7 @@ func TestTxPoolCancelledContextError(t *testing.T) {
cancel()
// when
- _, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ _, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
// then
require.Error(t, err)
@@ -312,12 +299,12 @@ func TestTxPoolWaitTimeoutError(t *testing.T) {
defer closer()
// lock the only connection in the pool.
- conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
defer conn.Unlock()
// try locking one more connection.
- _, _, _, err = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ _, _, _, err = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
// then
require.Error(t, err)
@@ -337,7 +324,7 @@ func TestTxPoolRollbackFailIsPassedThrough(t *testing.T) {
defer closer()
db.AddRejectedQuery("rollback", errRejected)
- conn1, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn1, _, _, err := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
_, err = conn1.Exec(ctx, sql, 1, true)
@@ -357,7 +344,7 @@ func TestTxPoolGetConnRecentlyRemovedTransaction(t *testing.T) {
db, txPool, _, _ := setup(t)
defer db.Close()
- conn1, _, _, _ := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn1, _, _, _ := txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
id := conn1.ReservedID()
conn1.Unlock()
txPool.Close()
@@ -380,7 +367,7 @@ func TestTxPoolGetConnRecentlyRemovedTransaction(t *testing.T) {
params := dbconfigs.New(db.ConnParams())
txPool.Open(params, params, params)
- conn1, _, _, _ = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn1, _, _, _ = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
id = conn1.ReservedID()
_, err := txPool.Commit(ctx, conn1)
require.NoError(t, err)
@@ -396,7 +383,7 @@ func TestTxPoolGetConnRecentlyRemovedTransaction(t *testing.T) {
txPool.Open(params, params, params)
defer txPool.Close()
- conn1, _, _, err = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn1, _, _, err = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err, "unable to start transaction: %v", err)
conn1.Unlock()
id = conn1.ReservedID()
@@ -412,7 +399,7 @@ func TestTxPoolCloseKillsStrayTransactions(t *testing.T) {
startingStray := txPool.env.Stats().InternalErrors.Counts()["StrayTransactions"]
// Start stray transaction.
- conn, _, _, err := txPool.Begin(context.Background(), &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(context.Background(), &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
conn.Unlock()
@@ -443,7 +430,7 @@ func TestTxTimeoutKillsTransactions(t *testing.T) {
ctxWithCallerID := callerid.NewContext(ctx, ef, im)
// Start transaction.
- conn, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
conn.Unlock()
@@ -491,7 +478,7 @@ func TestTxTimeoutDoesNotKillShortLivedTransactions(t *testing.T) {
ctxWithCallerID := callerid.NewContext(ctx, ef, im)
// Start transaction.
- conn, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
conn.Unlock()
@@ -526,7 +513,7 @@ func TestTxTimeoutKillsOlapTransactions(t *testing.T) {
// Start transaction.
conn, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{
Workload: querypb.ExecuteOptions_OLAP,
- }, false, 0, nil, nil)
+ }, false, 0, nil)
require.NoError(t, err)
conn.Unlock()
@@ -561,11 +548,11 @@ func TestTxTimeoutNotEnforcedForZeroLengthTimeouts(t *testing.T) {
ctxWithCallerID := callerid.NewContext(ctx, ef, im)
// Start transactions.
- conn0, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil, nil)
+ conn0, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, 0, nil)
require.NoError(t, err)
conn1, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{
Workload: querypb.ExecuteOptions_OLAP,
- }, false, 0, nil, nil)
+ }, false, 0, nil)
require.NoError(t, err)
conn0.Unlock()
conn1.Unlock()
@@ -606,7 +593,7 @@ func TestTxTimeoutReservedConn(t *testing.T) {
// Start OLAP transaction and return it to pool right away.
conn0, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{
Workload: querypb.ExecuteOptions_OLAP,
- }, false, 0, nil, nil)
+ }, false, 0, nil)
require.NoError(t, err)
// Taint the connection.
conn0.Taint(ctxWithCallerID, nil)
@@ -648,14 +635,14 @@ func TestTxTimeoutReusedReservedConn(t *testing.T) {
// Start OLAP transaction and return it to pool right away.
conn0, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{
Workload: querypb.ExecuteOptions_OLAP,
- }, false, 0, nil, nil)
+ }, false, 0, nil)
require.NoError(t, err)
// Taint the connection.
conn0.Taint(ctxWithCallerID, nil)
conn0.Unlock()
// Reuse underlying connection in an OLTP transaction.
- conn1, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, conn0.ReservedID(), nil, nil)
+ conn1, _, _, err := txPool.Begin(ctxWithCallerID, &querypb.ExecuteOptions{}, false, conn0.ReservedID(), nil)
require.NoError(t, err)
require.Equal(t, conn1.ReservedID(), conn0.ReservedID())
conn1.Unlock()
@@ -714,11 +701,11 @@ func TestTxPoolBeginStatements(t *testing.T) {
expBeginSQL: "set transaction isolation level serializable; start transaction read only",
}, {
txIsolationLevel: querypb.ExecuteOptions_CONSISTENT_SNAPSHOT_READ_ONLY,
- expBeginSQL: "set session session_track_gtids = START_GTID; set transaction isolation level repeatable read; start transaction with consistent snapshot, read only",
+ expBeginSQL: "set transaction isolation level repeatable read; start transaction with consistent snapshot, read only",
}, {
txIsolationLevel: querypb.ExecuteOptions_CONSISTENT_SNAPSHOT_READ_ONLY,
readOnly: true,
- expBeginSQL: "set session session_track_gtids = START_GTID; set transaction isolation level repeatable read; start transaction with consistent snapshot, read only",
+ expBeginSQL: "set transaction isolation level repeatable read; start transaction with consistent snapshot, read only",
}, {
txIsolationLevel: querypb.ExecuteOptions_AUTOCOMMIT,
expBeginSQL: "",
@@ -786,7 +773,7 @@ func TestTxPoolBeginStatements(t *testing.T) {
TransactionIsolation: tc.txIsolationLevel,
TransactionAccessMode: tc.txAccessModes,
}
- conn, beginSQL, _, err := txPool.Begin(ctx, options, tc.readOnly, 0, nil, nil)
+ conn, beginSQL, _, err := txPool.Begin(ctx, options, tc.readOnly, 0, nil)
if tc.expErr != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tc.expErr)
diff --git a/go/vt/vttablet/tabletserver/vstreamer/helper_event_test.go b/go/vt/vttablet/tabletserver/vstreamer/helper_event_test.go
index c52e81564c0..675677bf820 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/helper_event_test.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/helper_event_test.go
@@ -142,6 +142,11 @@ type TestQuery struct {
type TestRowChange struct {
before []string
after []string
+
+ // If you need to customize the image you can use the raw types.
+ beforeRaw *query.Row
+ afterRaw *query.Row
+ dataColumnsRaw *binlogdatapb.RowChange_Bitmap
}
// TestRowEventSpec is used for defining a custom row event.
@@ -161,7 +166,12 @@ func (s *TestRowEventSpec) String() string {
if len(s.changes) > 0 {
for _, c := range s.changes {
rowChange := binlogdatapb.RowChange{}
- if len(c.before) > 0 {
+ if c.dataColumnsRaw != nil {
+ rowChange.DataColumns = c.dataColumnsRaw
+ }
+ if c.beforeRaw != nil {
+ rowChange.Before = c.beforeRaw
+ } else if len(c.before) > 0 {
rowChange.Before = &query.Row{}
for _, val := range c.before {
if val == sqltypes.NullStr {
@@ -171,7 +181,9 @@ func (s *TestRowEventSpec) String() string {
rowChange.Before.Values = append(rowChange.Before.Values, []byte(val)...)
}
}
- if len(c.after) > 0 {
+ if c.afterRaw != nil {
+ rowChange.After = c.afterRaw
+ } else if len(c.after) > 0 {
rowChange.After = &query.Row{}
for i, val := range c.after {
if val == sqltypes.NullStr {
@@ -354,6 +366,7 @@ func (ts *TestSpec) getBindVarsForUpdate(stmt sqlparser.Statement) (string, map[
require.True(ts.t, ok, "field event for table %s not found", table)
index := int64(0)
state := ts.getCurrentState(table)
+ require.NotNil(ts.t, state)
for i, col := range fe.cols {
bv[col.name] = string(state.Values[index : index+state.Lengths[i]])
index += state.Lengths[i]
diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go
index 9bbc98ca2bd..e5115afe6d3 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go
@@ -89,6 +89,8 @@ const (
NotEqual
// IsNotNull is used to filter a column if it is NULL
IsNotNull
+ // In is used to filter a comparable column if equals any of the values from a specific tuple
+ In
)
// Filter contains opcodes for filtering.
@@ -97,6 +99,9 @@ type Filter struct {
ColNum int
Value sqltypes.Value
+ // Values will be used to store tuple/list values.
+ Values []sqltypes.Value
+
// Parameters for VindexMatch.
// Vindex, VindexColumns and KeyRange, if set, will be used
// to filter the row.
@@ -166,6 +171,8 @@ func getOpcode(comparison *sqlparser.ComparisonExpr) (Opcode, error) {
opcode = GreaterThanEqual
case sqlparser.NotEqualOp:
opcode = NotEqual
+ case sqlparser.InOp:
+ opcode = In
default:
return -1, fmt.Errorf("comparison operator %s not supported", comparison.Operator.ToString())
}
@@ -238,6 +245,24 @@ func (plan *Plan) filter(values, result []sqltypes.Value, charsets []collations.
if values[filter.ColNum].IsNull() {
return false, nil
}
+ case In:
+ if filter.Values == nil {
+ return false, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected empty filter values when performing IN operator")
+ }
+ found := false
+ for _, filterValue := range filter.Values {
+ match, err := compare(Equal, values[filter.ColNum], filterValue, plan.env.CollationEnv(), charsets[filter.ColNum])
+ if err != nil {
+ return false, err
+ }
+ if match {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return false, nil
+ }
default:
match, err := compare(filter.Opcode, values[filter.ColNum], filter.Value, plan.env.CollationEnv(), charsets[filter.ColNum])
if err != nil {
@@ -514,6 +539,27 @@ func (plan *Plan) getColumnFuncExpr(columnName string) *sqlparser.FuncExpr {
return nil
}
+func (plan *Plan) appendTupleFilter(values sqlparser.ValTuple, opcode Opcode, colnum int) error {
+ pv, err := evalengine.Translate(values, &evalengine.Config{
+ Collation: plan.env.CollationEnv().DefaultConnectionCharset(),
+ Environment: plan.env,
+ })
+ if err != nil {
+ return err
+ }
+ env := evalengine.EmptyExpressionEnv(plan.env)
+ resolved, err := env.Evaluate(pv)
+ if err != nil {
+ return err
+ }
+ plan.Filters = append(plan.Filters, Filter{
+ Opcode: opcode,
+ ColNum: colnum,
+ Values: resolved.TupleValues(),
+ })
+ return nil
+}
+
func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) error {
if where == nil {
return nil
@@ -537,6 +583,20 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er
if err != nil {
return err
}
+ // The Right Expr is typically expected to be a Literal value,
+ // except for the IN operator, where a Tuple value is expected.
+ // Handle the IN operator case first.
+ if opcode == In {
+ values, ok := expr.Right.(sqlparser.ValTuple)
+ if !ok {
+ return fmt.Errorf("unexpected: %v", sqlparser.String(expr))
+ }
+ err := plan.appendTupleFilter(values, opcode, colnum)
+ if err != nil {
+ return err
+ }
+ continue
+ }
val, ok := expr.Right.(*sqlparser.Literal)
if !ok {
return fmt.Errorf("unexpected: %v", sqlparser.String(expr))
diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go
index ba345b2a00b..aba74368802 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go
@@ -710,9 +710,15 @@ func TestPlanBuilderFilterComparison(t *testing.T) {
outFilters: []Filter{{Opcode: LessThan, ColNum: 0, Value: sqltypes.NewInt64(2)},
{Opcode: LessThanEqual, ColNum: 1, Value: sqltypes.NewVarChar("xyz")},
},
+ }, {
+ name: "in-operator",
+ inFilter: "select * from t1 where id in (1, 2)",
+ outFilters: []Filter{
+ {Opcode: In, ColNum: 0, Values: []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}},
+ },
}, {
name: "vindex-and-operators",
- inFilter: "select * from t1 where in_keyrange(id, 'hash', '-80') and id = 2 and val <> 'xyz'",
+ inFilter: "select * from t1 where in_keyrange(id, 'hash', '-80') and id = 2 and val <> 'xyz' and id in (100, 30)",
outFilters: []Filter{
{
Opcode: VindexMatch,
@@ -727,6 +733,7 @@ func TestPlanBuilderFilterComparison(t *testing.T) {
},
{Opcode: Equal, ColNum: 0, Value: sqltypes.NewInt64(2)},
{Opcode: NotEqual, ColNum: 1, Value: sqltypes.NewVarChar("xyz")},
+ {Opcode: In, ColNum: 0, Values: []sqltypes.Value{sqltypes.NewInt64(100), sqltypes.NewInt64(30)}},
},
}}
diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go
index 2c75c4a73b0..47db5fb839a 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go
@@ -202,7 +202,7 @@ func TestStreamRowsScan(t *testing.T) {
`fields:{name:"id" type:INT32 table:"t3" org_table:"t3" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t3" org_table:"t3" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63} pkfields:{name:"val" type:VARBINARY charset:63}`,
`rows:{lengths:1 lengths:3 values:"2bbb"} lastpk:{lengths:1 lengths:3 values:"2bbb"}`,
}
- wantQuery = "select /*+ MAX_EXECUTION_TIME(3600000) */ id, val from t3 where (id = 1 and val > 'aaa') or (id > 1) order by id, val"
+ wantQuery = "select /*+ MAX_EXECUTION_TIME(3600000) */ id, val from t3 where (id = 1 and val > _binary'aaa') or (id > 1) order by id, val"
checkStream(t, "select * from t3", []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewVarBinary("aaa")}, wantQuery, wantStream)
// t4: all rows
diff --git a/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go b/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go
index ec326cc4159..ee141ce9859 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go
@@ -132,29 +132,6 @@ func (conn *snapshotConn) startSnapshot(ctx context.Context, table string) (gtid
return replication.EncodePosition(mpos), nil
}
-// startSnapshotWithConsistentGTID performs the snapshotting without locking tables. This assumes
-// session_track_gtids = START_GTID, which is a contribution to MySQL and is not in vanilla MySQL at the
-// time of this writing.
-func (conn *snapshotConn) startSnapshotWithConsistentGTID(ctx context.Context) (gtid string, err error) {
- if _, err := conn.ExecuteFetch("set transaction isolation level repeatable read", 1, false); err != nil {
- return "", err
- }
- result, err := conn.ExecuteFetch("start transaction with consistent snapshot, read only", 1, false)
- if err != nil {
- return "", err
- }
- // The "session_track_gtids = START_GTID" patch is only applicable to MySQL56 GTID, which is
- // why we hardcode the position as mysql.Mysql56FlavorID
- mpos, err := replication.ParsePosition(replication.Mysql56FlavorID, result.SessionStateChanges)
- if err != nil {
- return "", err
- }
- if _, err := conn.ExecuteFetch("set @@session.time_zone = '+00:00'", 1, false); err != nil {
- return "", err
- }
- return replication.EncodePosition(mpos), nil
-}
-
// Close rolls back any open transactions and closes the connection.
func (conn *snapshotConn) Close() {
_, _ = conn.ExecuteFetch("rollback", 1, false)
diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go
index e7453b2b70f..59db723ff2b 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go
@@ -37,6 +37,7 @@ import (
"vitess.io/vitess/go/vt/logutil"
vtschema "vitess.io/vitess/go/vt/schema"
"vitess.io/vitess/go/vt/sqlparser"
+ "vitess.io/vitess/go/vt/vtenv"
"vitess.io/vitess/go/vt/vterrors"
vttablet "vitess.io/vitess/go/vt/vttablet/common"
"vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
@@ -374,7 +375,7 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog
}
return fmt.Errorf("unexpected server EOF")
}
- vevents, err := vs.parseEvent(ev)
+ vevents, err := vs.parseEvent(ev, bufferAndTransmit)
if err != nil {
vs.vse.errorCounts.Add("ParseEvent", 1)
return err
@@ -415,7 +416,11 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog
}
// parseEvent parses an event from the binlog and converts it to a list of VEvents.
-func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, error) {
+// The bufferAndTransmit function must be passed if the event is a TransactionPayloadEvent
+// as for larger payloads (> ZstdInMemoryDecompressorMaxSize) the internal events need
+// to be streamed directly here in order to avoid holding the entire payload's contents,
+// which can be 10s or even 100s of GiBs, all in memory.
+func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent, bufferAndTransmit func(vevent *binlogdatapb.VEvent) error) ([]*binlogdatapb.VEvent, error) {
if !ev.IsValid() {
return nil, fmt.Errorf("can't parse binlog event: invalid data: %#v", ev)
}
@@ -671,11 +676,31 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e
}
return nil, err
}
- tpvevents, err := vs.parseEvent(tpevent)
+ tpvevents, err := vs.parseEvent(tpevent, nil) // Parse the internal event
if err != nil {
return nil, vterrors.Wrap(err, "failed to parse transaction payload's internal event")
}
- vevents = append(vevents, tpvevents...)
+ if tp.StreamingContents {
+ // Transmit each internal event individually to avoid buffering
+ // the large transaction's entire payload of events in memory, as
+ // the uncompressed size can be 10s or even 100s of GiBs in size.
+ if bufferAndTransmit == nil {
+ return nil, vterrors.New(vtrpcpb.Code_INTERNAL, "[bug] cannot stream compressed transaction payload's internal events as no bufferAndTransmit function was provided")
+ }
+ for _, tpvevent := range tpvevents {
+ tpvevent.Timestamp = int64(ev.Timestamp())
+ tpvevent.CurrentTime = time.Now().UnixNano()
+ if err := bufferAndTransmit(tpvevent); err != nil {
+ if err == io.EOF {
+ return nil, nil
+ }
+ vs.vse.errorCounts.Add("TransactionPayloadBufferAndTransmit", 1)
+ return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "error sending compressed transaction payload's internal event: %v", err)
+ }
+ }
+ } else { // Process the payload's internal events all at once
+ vevents = append(vevents, tpvevents...)
+ }
}
vs.vse.vstreamerCompressedTransactionsDecoded.Add(1)
}
@@ -783,7 +808,7 @@ func (vs *vstreamer) buildTablePlan(id uint64, tm *mysql.TableMap) (*binlogdatap
vs.plans[id] = nil
return nil, nil
}
- if err := addEnumAndSetMappingstoPlan(plan, cols, tm.Metadata); err != nil {
+ if err := addEnumAndSetMappingstoPlan(vs.se.Environment(), plan, cols, tm.Metadata); err != nil {
return nil, vterrors.Wrapf(err, "failed to build ENUM and SET column integer to string mappings")
}
vs.plans[id] = &streamerPlan{
@@ -1097,13 +1122,13 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo
// Convert the integer values in the binlog event for any SET and ENUM fields into their
// string representations.
if plan.Table.Fields[colNum].Type == querypb.Type_ENUM || mysqlType == mysqlbinlog.TypeEnum {
- value, err = buildEnumStringValue(plan, colNum, value)
+ value, err = buildEnumStringValue(vs.se.Environment(), plan, colNum, value)
if err != nil {
return false, nil, false, vterrors.Wrapf(err, "failed to perform ENUM column integer to string value mapping")
}
}
if plan.Table.Fields[colNum].Type == querypb.Type_SET || mysqlType == mysqlbinlog.TypeSet {
- value, err = buildSetStringValue(plan, colNum, value)
+ value, err = buildSetStringValue(vs.se.Environment(), plan, colNum, value)
if err != nil {
return false, nil, false, vterrors.Wrapf(err, "failed to perform SET column integer to string value mapping")
}
@@ -1120,7 +1145,7 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo
}
// addEnumAndSetMappingstoPlan sets up any necessary ENUM and SET integer to string mappings.
-func addEnumAndSetMappingstoPlan(plan *Plan, cols []*querypb.Field, metadata []uint16) error {
+func addEnumAndSetMappingstoPlan(env *vtenv.Environment, plan *Plan, cols []*querypb.Field, metadata []uint16) error {
plan.EnumSetValuesMap = make(map[int]map[int]string)
for i, col := range cols {
// If the column is a CHAR based type with a binary collation (e.g. utf8mb4_bin) then
@@ -1139,21 +1164,25 @@ func addEnumAndSetMappingstoPlan(plan *Plan, cols []*querypb.Field, metadata []u
return fmt.Errorf("enum or set column %s does not have valid string values: %s",
col.Name, col.ColumnType)
}
- plan.EnumSetValuesMap[i] = vtschema.ParseEnumOrSetTokensMap(col.ColumnType[begin+1 : end])
+ var err error
+ plan.EnumSetValuesMap[i], err = vtschema.ParseEnumOrSetTokensMap(env, col.ColumnType[begin+1:end])
+ if err != nil {
+ return err
+ }
}
}
return nil
}
// buildEnumStringValue takes the integer value of an ENUM column and returns the string value.
-func buildEnumStringValue(plan *streamerPlan, colNum int, value sqltypes.Value) (sqltypes.Value, error) {
+func buildEnumStringValue(env *vtenv.Environment, plan *streamerPlan, colNum int, value sqltypes.Value) (sqltypes.Value, error) {
if value.IsNull() { // No work is needed
return value, nil
}
// Add the mappings just-in-time in case we haven't properly received and processed a
// table map event to initialize it.
if plan.EnumSetValuesMap == nil {
- if err := addEnumAndSetMappingstoPlan(plan.Plan, plan.Table.Fields, plan.TableMap.Metadata); err != nil {
+ if err := addEnumAndSetMappingstoPlan(env, plan.Plan, plan.Table.Fields, plan.TableMap.Metadata); err != nil {
return sqltypes.Value{}, err
}
}
@@ -1181,14 +1210,14 @@ func buildEnumStringValue(plan *streamerPlan, colNum int, value sqltypes.Value)
}
// buildSetStringValue takes the integer value of a SET column and returns the string value.
-func buildSetStringValue(plan *streamerPlan, colNum int, value sqltypes.Value) (sqltypes.Value, error) {
+func buildSetStringValue(env *vtenv.Environment, plan *streamerPlan, colNum int, value sqltypes.Value) (sqltypes.Value, error) {
if value.IsNull() { // No work is needed
return value, nil
}
// Add the mappings just-in-time in case we haven't properly received and processed a
// table map event to initialize it.
if plan.EnumSetValuesMap == nil {
- if err := addEnumAndSetMappingstoPlan(plan.Plan, plan.Table.Fields, plan.TableMap.Metadata); err != nil {
+ if err := addEnumAndSetMappingstoPlan(env, plan.Plan, plan.Table.Fields, plan.TableMap.Metadata); err != nil {
return sqltypes.Value{}, err
}
}
diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go
index c265fb45b85..5282b5f372d 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go
@@ -41,6 +41,7 @@ import (
"vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/testenv"
binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
+ querypb "vitess.io/vitess/go/vt/proto/query"
)
type testcase struct {
@@ -87,6 +88,9 @@ func TestNoBlob(t *testing.T) {
"create table t2(id int, txt text, val varchar(4), unique key(id, val))",
// t3 has a text column and a primary key. The text column will not be in update row events.
"create table t3(id int, txt text, val varchar(4), primary key(id))",
+ // t4 has a blob column and a primary key, along with a generated virtual column. The blob
+ // column will not be in update row events.
+ "create table t4(id int, cOl varbinary(8) generated always as (concat(val, 'tsty')) virtual, blb blob, val varbinary(4), primary key(id))",
},
options: &TestSpecOptions{
noblob: true,
@@ -94,6 +98,18 @@ func TestNoBlob(t *testing.T) {
}
defer ts.Close()
ts.Init()
+
+ insertGeneratedFE := &TestFieldEvent{
+ table: "t4",
+ db: testenv.DBName,
+ cols: []*TestColumn{
+ {name: "id", dataType: "INT32", colType: "int(11)", len: 11, collationID: 63},
+ {name: "cOl", dataType: "VARBINARY", colType: "varbinary(8)", len: 8, collationID: 63},
+ {name: "blb", dataType: "BLOB", colType: "blob", len: 65535, collationID: 63},
+ {name: "val", dataType: "VARBINARY", colType: "varbinary(4)", len: 4, collationID: 63},
+ },
+ }
+
ts.tests = [][]*TestQuery{{
{"begin", nil},
{"insert into t1 values (1, 'blob1', 'aaa')", nil},
@@ -107,6 +123,29 @@ func TestNoBlob(t *testing.T) {
{"insert into t3 values (1, 'text1', 'aaa')", nil},
{"update t3 set val = 'bbb'", nil},
{"commit", nil},
+ }, {{"begin", nil},
+ {"insert into t4 (id, blb, val) values (1, 'text1', 'aaa')", []TestRowEvent{
+ {event: insertGeneratedFE.String()},
+ {spec: &TestRowEventSpec{table: "t4", changes: []TestRowChange{{after: []string{"1", "aaatsty", "text1", "aaa"}}}}},
+ }},
+ {"update t4 set val = 'bbb'", []TestRowEvent{
+ // The blob column is not in the update row event's before or after image.
+ {spec: &TestRowEventSpec{table: "t4", changes: []TestRowChange{{
+ beforeRaw: &querypb.Row{
+ Lengths: []int64{1, 7, -1, 3}, // -1 for the 3rd column / blob field, as it's not present
+ Values: []byte("1aaatstyaaa"),
+ },
+ afterRaw: &querypb.Row{
+ Lengths: []int64{1, 7, -1, 3}, // -1 for the 3rd column / blob field, as it's not present
+ Values: []byte("1bbbtstybbb"),
+ },
+ dataColumnsRaw: &binlogdatapb.RowChange_Bitmap{
+ Count: 4,
+ Cols: []byte{0x0b}, // Columns bitmap of 00001011 as the third column/bit position representing the blob column has no data
+ },
+ }}}},
+ }},
+ {"commit", nil},
}}
ts.Run()
}
@@ -1927,7 +1966,7 @@ func TestFilteredMultipleWhere(t *testing.T) {
filter: &binlogdatapb.Filter{
Rules: []*binlogdatapb.Rule{{
Match: "t1",
- Filter: "select id1, val from t1 where in_keyrange('-80') and id2 = 200 and id3 = 1000 and val = 'newton'",
+ Filter: "select id1, val from t1 where in_keyrange('-80') and id2 = 200 and id3 = 1000 and val = 'newton' and id1 in (1, 2, 129)",
}},
},
customFieldEvents: true,
@@ -1949,9 +1988,7 @@ func TestFilteredMultipleWhere(t *testing.T) {
{spec: &TestRowEventSpec{table: "t1", changes: []TestRowChange{{after: []string{"2", "newton"}}}}},
}},
{"insert into t1 values (3, 100, 2000, 'kepler')", noEvents},
- {"insert into t1 values (128, 200, 1000, 'newton')", []TestRowEvent{
- {spec: &TestRowEventSpec{table: "t1", changes: []TestRowChange{{after: []string{"128", "newton"}}}}},
- }},
+ {"insert into t1 values (128, 200, 1000, 'newton')", noEvents},
{"insert into t1 values (5, 200, 2000, 'kepler')", noEvents},
{"insert into t1 values (129, 200, 1000, 'kepler')", noEvents},
{"commit", nil},
@@ -2041,3 +2078,33 @@ func TestGeneratedInvisiblePrimaryKey(t *testing.T) {
}}
ts.Run()
}
+
+func TestFilteredInOperator(t *testing.T) {
+ ts := &TestSpec{
+ t: t,
+ ddls: []string{
+ "create table t1(id1 int, id2 int, val varbinary(128), primary key(id1))",
+ },
+ options: &TestSpecOptions{
+ filter: &binlogdatapb.Filter{
+ Rules: []*binlogdatapb.Rule{{
+ Match: "t1",
+ Filter: "select id1, val from t1 where val in ('eee', 'bbb', 'ddd') and id1 in (4, 5)",
+ }},
+ },
+ },
+ }
+ defer ts.Close()
+ ts.Init()
+ ts.fieldEvents["t1"].cols[1].skip = true
+ ts.tests = [][]*TestQuery{{
+ {"begin", nil},
+ {"insert into t1 values (1, 100, 'aaa')", noEvents},
+ {"insert into t1 values (2, 200, 'bbb')", noEvents},
+ {"insert into t1 values (3, 100, 'ccc')", noEvents},
+ {"insert into t1 values (4, 200, 'ddd')", nil},
+ {"insert into t1 values (5, 200, 'eee')", nil},
+ {"commit", nil},
+ }}
+ ts.Run()
+}
diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go
index 9e97c033776..9d570b8f6c7 100644
--- a/go/vt/vttablet/tabletservermock/controller.go
+++ b/go/vt/vttablet/tabletservermock/controller.go
@@ -24,6 +24,7 @@ import (
"vitess.io/vitess/go/vt/dbconfigs"
"vitess.io/vitess/go/vt/mysqlctl"
+ "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
"vitess.io/vitess/go/vt/servenv"
"vitess.io/vitess/go/vt/topo"
"vitess.io/vitess/go/vt/vttablet/queryservice"
@@ -249,6 +250,12 @@ func (tqsc *Controller) ReadTransaction(ctx context.Context, target *querypb.Tar
return nil, nil
}
+// GetTransactionInfo is part of the tabletserver.Controller interface
+func (tqsc *Controller) GetTransactionInfo(ctx context.Context, target *querypb.Target, dtid string) (*tabletmanagerdata.GetTransactionInfoResponse, error) {
+ tqsc.MethodCalled["GetTransactionInfo"] = true
+ return nil, nil
+}
+
// ConcludeTransaction is part of the tabletserver.Controller interface
func (tqsc *Controller) ConcludeTransaction(context.Context, *querypb.Target, string) error {
tqsc.MethodCalled["ConcludeTransaction"] = true
diff --git a/go/vt/vttablet/tmclient/rpc_client_api.go b/go/vt/vttablet/tmclient/rpc_client_api.go
index 2a22a9a90a7..2b5cc967e9f 100644
--- a/go/vt/vttablet/tmclient/rpc_client_api.go
+++ b/go/vt/vttablet/tmclient/rpc_client_api.go
@@ -91,6 +91,9 @@ type TabletManagerClient interface {
// SetReadWrite makes the mysql instance read-write
SetReadWrite(ctx context.Context, tablet *topodatapb.Tablet) error
+ // ChangeTags asks the remote tablet to change its tags
+ ChangeTags(ctx context.Context, tablet *topodatapb.Tablet, tabletTags map[string]string, replace bool) (*tabletmanagerdatapb.ChangeTagsResponse, error)
+
// ChangeType asks the remote tablet to change its type
ChangeType(ctx context.Context, tablet *topodatapb.Tablet, dbType topodatapb.TabletType, semiSync bool) error
@@ -151,11 +154,13 @@ type TabletManagerClient interface {
//
// GetUnresolvedTransactions returns the list of unresolved transactions for the tablet.
- GetUnresolvedTransactions(ctx context.Context, tablet *topodatapb.Tablet) ([]*querypb.TransactionMetadata, error)
+ GetUnresolvedTransactions(ctx context.Context, tablet *topodatapb.Tablet, abandonAge int64) ([]*querypb.TransactionMetadata, error)
// ReadTransaction returns the metadata for the specified distributed transaction ID.
ReadTransaction(ctx context.Context, tablet *topodatapb.Tablet, dtid string) (*querypb.TransactionMetadata, error)
+ GetTransactionInfo(ctx context.Context, tablet *topodatapb.Tablet, dtid string) (*tabletmanagerdatapb.GetTransactionInfoResponse, error)
+
// ConcludeTransaction conclude the transaction on the tablet.
ConcludeTransaction(ctx context.Context, tablet *topodatapb.Tablet, dtid string, mm bool) error
@@ -202,6 +207,7 @@ type TabletManagerClient interface {
//
CreateVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.CreateVReplicationWorkflowRequest) (*tabletmanagerdatapb.CreateVReplicationWorkflowResponse, error)
+ DeleteTableData(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.DeleteTableDataRequest) (*tabletmanagerdatapb.DeleteTableDataResponse, error)
DeleteVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (*tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, error)
HasVReplicationWorkflows(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.HasVReplicationWorkflowsRequest) (*tabletmanagerdatapb.HasVReplicationWorkflowsResponse, error)
ReadVReplicationWorkflows(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.ReadVReplicationWorkflowsRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowsResponse, error)
@@ -232,6 +238,9 @@ type TabletManagerClient interface {
// its reparent_journal table.
PopulateReparentJournal(ctx context.Context, tablet *topodatapb.Tablet, timeCreatedNS int64, actionName string, tabletAlias *topodatapb.TabletAlias, pos string) error
+ // ReadReparentJournalInfo reads the information from reparent journal
+ ReadReparentJournalInfo(ctx context.Context, tablet *topodatapb.Tablet) (int, error)
+
// InitReplica tells a tablet to start replicating from the
// passed in primary tablet alias, and wait for the row in the
// reparent_journal table.
diff --git a/go/vt/vttablet/tmrpctest/test_tm_rpc.go b/go/vt/vttablet/tmrpctest/test_tm_rpc.go
index 4b1785565fb..a106b43bf2c 100644
--- a/go/vt/vttablet/tmrpctest/test_tm_rpc.go
+++ b/go/vt/vttablet/tmrpctest/test_tm_rpc.go
@@ -61,6 +61,11 @@ func (fra *fakeRPCTM) CreateVReplicationWorkflow(ctx context.Context, req *table
panic("implement me")
}
+func (fra *fakeRPCTM) DeleteTableData(ctx context.Context, req *tabletmanagerdatapb.DeleteTableDataRequest) (*tabletmanagerdatapb.DeleteTableDataResponse, error) {
+ // TODO implement me
+ panic("implement me")
+}
+
func (fra *fakeRPCTM) DeleteVReplicationWorkflow(ctx context.Context, req *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (*tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, error) {
// TODO implement me
panic("implement me")
@@ -416,12 +421,12 @@ func tmRPCTestGetGlobalStatusVarsPanic(ctx context.Context, t *testing.T, client
}
func tmRPCTestGetUnresolvedTransactions(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
- _, err := client.GetUnresolvedTransactions(ctx, tablet)
+ _, err := client.GetUnresolvedTransactions(ctx, tablet, 0)
require.NoError(t, err)
}
func tmRPCTestGetUnresolvedTransactionsPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
- _, err := client.GetUnresolvedTransactions(ctx, tablet)
+ _, err := client.GetUnresolvedTransactions(ctx, tablet, 0)
expectHandleRPCPanic(t, "GetUnresolvedTransactions", false /*verbose*/, err)
}
@@ -435,6 +440,16 @@ func tmRPCTestReadTransactionPanic(ctx context.Context, t *testing.T, client tmc
expectHandleRPCPanic(t, "ReadTransaction", false /*verbose*/, err)
}
+func tmRPCTestGetTransactionInfo(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
+ _, err := client.GetTransactionInfo(ctx, tablet, "aa")
+ require.NoError(t, err)
+}
+
+func tmRPCTestGetTransactionInfoPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
+ _, err := client.GetTransactionInfo(ctx, tablet, "aa")
+ expectHandleRPCPanic(t, "GetTransactionInfo", false /*verbose*/, err)
+}
+
//
// Various read-write methods
//
@@ -471,6 +486,30 @@ func tmRPCTestSetReadOnlyPanic(ctx context.Context, t *testing.T, client tmclien
expectHandleRPCPanic(t, "SetReadWrite", true /*verbose*/, err)
}
+var testChangeTagsValue = map[string]string{
+ "test": "12345",
+}
+
+func (fra *fakeRPCTM) ChangeTags(ctx context.Context, tabletTags map[string]string, replace bool) (map[string]string, error) {
+ if fra.panics {
+ panic(fmt.Errorf("test-triggered panic"))
+ }
+ compare(fra.t, "ChangeTags tabletType", tabletTags, testChangeTagsValue)
+ return tabletTags, nil
+}
+
+func tmRPCTestChangeTags(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
+ _, err := client.ChangeTags(ctx, tablet, testChangeTagsValue, false)
+ if err != nil {
+ t.Errorf("ChangeTags failed: %v", err)
+ }
+}
+
+func tmRPCTestChangeTagsPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
+ _, err := client.ChangeTags(ctx, tablet, testChangeTagsValue, false)
+ expectHandleRPCPanic(t, "ChangeTags", true /*verbose*/, err)
+}
+
var testChangeTypeValue = topodatapb.TabletType_REPLICA
func (fra *fakeRPCTM) ChangeType(ctx context.Context, tabletType topodatapb.TabletType, semiSync bool) error {
@@ -771,6 +810,13 @@ func (fra *fakeRPCTM) ReadTransaction(ctx context.Context, req *tabletmanagerdat
return nil, nil
}
+func (fra *fakeRPCTM) GetTransactionInfo(ctx context.Context, req *tabletmanagerdatapb.GetTransactionInfoRequest) (*tabletmanagerdatapb.GetTransactionInfoResponse, error) {
+ if fra.panics {
+ panic(fmt.Errorf("test-triggered panic"))
+ }
+ return nil, nil
+}
+
func (fra *fakeRPCTM) ConcludeTransaction(ctx context.Context, req *tabletmanagerdatapb.ConcludeTransactionRequest) error {
if fra.panics {
panic(fmt.Errorf("test-triggered panic"))
@@ -1133,6 +1179,15 @@ func (fra *fakeRPCTM) PopulateReparentJournal(ctx context.Context, timeCreatedNS
return nil
}
+var testReparentJournalLen = 10
+
+func (fra *fakeRPCTM) ReadReparentJournalInfo(context.Context) (int, error) {
+ if fra.panics {
+ panic(fmt.Errorf("test-triggered panic"))
+ }
+ return testReparentJournalLen, nil
+}
+
func tmRPCTestPopulateReparentJournal(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.PopulateReparentJournal(ctx, tablet, testTimeCreatedNS, testActionName, testPrimaryAlias, testReplicationPosition)
compareError(t, "PopulateReparentJournal", err, true, testPopulateReparentJournalCalled)
@@ -1143,6 +1198,16 @@ func tmRPCTestPopulateReparentJournalPanic(ctx context.Context, t *testing.T, cl
expectHandleRPCPanic(t, "PopulateReparentJournal", true /*verbose*/, err)
}
+func tmRPCTestReadReparentJournalInfo(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
+ length, err := client.ReadReparentJournalInfo(ctx, tablet)
+ compareError(t, "ReadReparentJournalInfo", err, length, testReparentJournalLen)
+}
+
+func tmRPCTestReadReparentJournalInfoPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
+ _, err := client.ReadReparentJournalInfo(ctx, tablet)
+ expectHandleRPCPanic(t, "ReadReparentJournalInfo", true /*verbose*/, err)
+}
+
func tmRPCTestWaitForPositionPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.WaitForPosition(ctx, tablet, testReplicationPosition)
expectHandleRPCPanic(t, "WaitForPosition", true /*verbose*/, err)
@@ -1363,6 +1428,10 @@ func (fra *fakeRPCTM) Backup(ctx context.Context, logger logutil.Logger, request
return nil
}
+func (fra *fakeRPCTM) IsBackupRunning() bool {
+ return false
+}
+
func tmRPCTestBackup(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
req := &tabletmanagerdatapb.BackupRequest{Concurrency: testBackupConcurrency, AllowPrimary: testBackupAllowPrimary}
stream, err := client.Backup(ctx, tablet, req)
@@ -1478,6 +1547,7 @@ func Run(t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.T
tmRPCTestGetGlobalStatusVars(ctx, t, client, tablet)
tmRPCTestGetUnresolvedTransactions(ctx, t, client, tablet)
tmRPCTestReadTransaction(ctx, t, client, tablet)
+ tmRPCTestGetTransactionInfo(ctx, t, client, tablet)
// Various read-write methods
tmRPCTestSetReadOnly(ctx, t, client, tablet)
@@ -1511,6 +1581,7 @@ func Run(t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.T
tmRPCTestResetReplication(ctx, t, client, tablet)
tmRPCTestInitPrimary(ctx, t, client, tablet)
tmRPCTestPopulateReparentJournal(ctx, t, client, tablet)
+ tmRPCTestReadReparentJournalInfo(ctx, t, client, tablet)
tmRPCTestDemotePrimary(ctx, t, client, tablet)
tmRPCTestUndoDemotePrimary(ctx, t, client, tablet)
tmRPCTestSetReplicationSource(ctx, t, client, tablet)
@@ -1541,6 +1612,7 @@ func Run(t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.T
tmRPCTestGetGlobalStatusVarsPanic(ctx, t, client, tablet)
tmRPCTestGetUnresolvedTransactionsPanic(ctx, t, client, tablet)
tmRPCTestReadTransactionPanic(ctx, t, client, tablet)
+ tmRPCTestGetTransactionInfoPanic(ctx, t, client, tablet)
// Various read-write methods
tmRPCTestSetReadOnlyPanic(ctx, t, client, tablet)
@@ -1570,6 +1642,7 @@ func Run(t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.T
tmRPCTestResetReplicationPanic(ctx, t, client, tablet)
tmRPCTestInitPrimaryPanic(ctx, t, client, tablet)
tmRPCTestPopulateReparentJournalPanic(ctx, t, client, tablet)
+ tmRPCTestReadReparentJournalInfoPanic(ctx, t, client, tablet)
tmRPCTestWaitForPositionPanic(ctx, t, client, tablet)
tmRPCTestDemotePrimaryPanic(ctx, t, client, tablet)
tmRPCTestUndoDemotePrimaryPanic(ctx, t, client, tablet)
diff --git a/go/vt/vttest/local_cluster.go b/go/vt/vttest/local_cluster.go
index 576a78bb761..fa7e40feb12 100644
--- a/go/vt/vttest/local_cluster.go
+++ b/go/vt/vttest/local_cluster.go
@@ -660,6 +660,7 @@ func (db *LocalCluster) JSONConfig() any {
config := map[string]any{
"bind_address": db.vt.BindAddress,
"port": db.vt.Port,
+ "grpc_bind_address": db.vt.BindAddressGprc,
"socket": db.mysql.UnixSocket(),
"vtcombo_mysql_port": db.Env.PortForProtocol("vtcombo_mysql_port", ""),
"mysql": db.Env.PortForProtocol("mysql", ""),
diff --git a/go/vt/vttest/vtprocess.go b/go/vt/vttest/vtprocess.go
index 3f34994bb75..6371811a60e 100644
--- a/go/vt/vttest/vtprocess.go
+++ b/go/vt/vttest/vtprocess.go
@@ -44,16 +44,17 @@ type HealthChecker func(addr string) bool
// It can be spawned manually or through one of the available
// helper methods.
type VtProcess struct {
- Name string
- Directory string
- LogDirectory string
- Binary string
- ExtraArgs []string
- Env []string
- BindAddress string
- Port int
- PortGrpc int
- HealthCheck HealthChecker
+ Name string
+ Directory string
+ LogDirectory string
+ Binary string
+ ExtraArgs []string
+ Env []string
+ BindAddress string
+ BindAddressGprc string
+ Port int
+ PortGrpc int
+ HealthCheck HealthChecker
proc *exec.Cmd
exit chan error
@@ -139,6 +140,11 @@ func (vtp *VtProcess) WaitStart() (err error) {
vtp.proc.Args = append(vtp.proc.Args, fmt.Sprintf("%d", vtp.PortGrpc))
}
+ if vtp.BindAddressGprc != "" {
+ vtp.proc.Args = append(vtp.proc.Args, "--grpc_bind_address")
+ vtp.proc.Args = append(vtp.proc.Args, vtp.BindAddressGprc)
+ }
+
vtp.proc.Args = append(vtp.proc.Args, vtp.ExtraArgs...)
vtp.proc.Env = append(vtp.proc.Env, os.Environ()...)
vtp.proc.Env = append(vtp.proc.Env, vtp.Env...)
@@ -199,16 +205,22 @@ func VtcomboProcess(environment Environment, args *Config, mysql MySQLManager) (
if args.VtComboBindAddress != "" {
vtcomboBindAddress = args.VtComboBindAddress
}
+ grpcBindAddress := "127.0.0.1"
+ if servenv.GRPCBindAddress() != "" {
+ grpcBindAddress = servenv.GRPCBindAddress()
+ }
+
vt := &VtProcess{
- Name: "vtcombo",
- Directory: environment.Directory(),
- LogDirectory: environment.LogDirectory(),
- Binary: environment.BinaryPath("vtcombo"),
- BindAddress: vtcomboBindAddress,
- Port: environment.PortForProtocol("vtcombo", ""),
- PortGrpc: environment.PortForProtocol("vtcombo", "grpc"),
- HealthCheck: environment.ProcessHealthCheck("vtcombo"),
- Env: environment.EnvVars(),
+ Name: "vtcombo",
+ Directory: environment.Directory(),
+ LogDirectory: environment.LogDirectory(),
+ Binary: environment.BinaryPath("vtcombo"),
+ BindAddress: vtcomboBindAddress,
+ BindAddressGprc: grpcBindAddress,
+ Port: environment.PortForProtocol("vtcombo", ""),
+ PortGrpc: environment.PortForProtocol("vtcombo", "grpc"),
+ HealthCheck: environment.ProcessHealthCheck("vtcombo"),
+ Env: environment.EnvVars(),
}
user, pass := mysql.Auth()
diff --git a/go/vt/wrangler/fake_dbclient_test.go b/go/vt/wrangler/fake_dbclient_test.go
index 14ef0913383..02ee79210d7 100644
--- a/go/vt/wrangler/fake_dbclient_test.go
+++ b/go/vt/wrangler/fake_dbclient_test.go
@@ -153,6 +153,10 @@ func (dc *fakeDBClient) Rollback() error {
func (dc *fakeDBClient) Close() {
}
+func (dc *fakeDBClient) IsClosed() bool {
+ return false
+}
+
// ExecuteFetch is part of the DBClient interface
func (dc *fakeDBClient) ExecuteFetch(query string, maxrows int) (*sqltypes.Result, error) {
dc.mu.Lock()
diff --git a/go/vt/wrangler/materializer.go b/go/vt/wrangler/materializer.go
index 7e24945cde7..bd7ae553130 100644
--- a/go/vt/wrangler/materializer.go
+++ b/go/vt/wrangler/materializer.go
@@ -551,11 +551,8 @@ func (wr *Wrangler) prepareCreateLookup(ctx context.Context, keyspace string, sp
if len(vindexFromCols) != 1 {
return nil, nil, nil, fmt.Errorf("unique vindex 'from' should have only one column: %v", vindex)
}
- } else {
- if len(vindexFromCols) < 2 {
- return nil, nil, nil, fmt.Errorf("non-unique vindex 'from' should have more than one column: %v", vindex)
- }
}
+
vindexToCol = vindex.Params["to"]
// Make the vindex write_only. If one exists already in the vschema,
// it will need to match this vindex exactly, including the write_only setting.
diff --git a/go/vt/wrangler/materializer_test.go b/go/vt/wrangler/materializer_test.go
index 1728ba6efc2..1871d778c6b 100644
--- a/go/vt/wrangler/materializer_test.go
+++ b/go/vt/wrangler/materializer_test.go
@@ -1599,7 +1599,7 @@ func TestCreateLookupVindexFailures(t *testing.T) {
},
err: "unique vindex 'from' should have only one column",
}, {
- description: "non-unique lookup should have more than one column",
+ description: "non-unique lookup can have only one column",
input: &vschemapb.Keyspace{
Vindexes: map[string]*vschemapb.Vindex{
"v": {
@@ -1612,7 +1612,7 @@ func TestCreateLookupVindexFailures(t *testing.T) {
},
},
},
- err: "non-unique vindex 'from' should have more than one column",
+ err: "",
}, {
description: "vindex not found",
input: &vschemapb.Keyspace{
diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go
index f1977df3f16..b540fc9f8f0 100644
--- a/go/vt/wrangler/testlib/backup_test.go
+++ b/go/vt/wrangler/testlib/backup_test.go
@@ -28,7 +28,6 @@ import (
"github.com/stretchr/testify/require"
"vitess.io/vitess/go/mysql"
- "vitess.io/vitess/go/mysql/capabilities"
"vitess.io/vitess/go/mysql/fakesqldb"
"vitess.io/vitess/go/mysql/replication"
"vitess.io/vitess/go/sqltypes"
@@ -36,6 +35,7 @@ import (
"vitess.io/vitess/go/vt/logutil"
"vitess.io/vitess/go/vt/mysqlctl"
"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
+ "vitess.io/vitess/go/vt/mysqlctl/blackbox"
"vitess.io/vitess/go/vt/mysqlctl/filebackupstorage"
"vitess.io/vitess/go/vt/topo/memorytopo"
"vitess.io/vitess/go/vt/topo/topoproto"
@@ -132,7 +132,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
require.NoError(t, os.MkdirAll(s, os.ModePerm))
}
- needIt, err := needInnoDBRedoLogSubdir()
+ needIt, err := blackbox.NeedInnoDBRedoLogSubdir()
require.NoError(t, err)
if needIt {
newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
@@ -149,7 +149,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -157,7 +157,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
Sequence: 457,
},
},
- }
+ })
// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
@@ -169,7 +169,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)}
- sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -177,7 +177,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
Sequence: 457,
},
},
- }
+ })
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP REPLICA",
@@ -220,7 +220,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
- destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -228,7 +228,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
Sequence: 457,
},
},
- }
+ })
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP REPLICA",
@@ -247,7 +247,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
destTablet.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SHOW DATABASES": {},
}
- destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition
+ destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))
destTablet.StartActionLoop(t, wr)
@@ -300,7 +300,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
"START REPLICA",
}
- primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.CurrentPrimaryPosition
+ primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.GetPrimaryPositionLocked()
// restore primary from latest backup
require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "" /* restoreToPos */, []string{} /* ignoreBackupEngines */, mysqlShutdownTimeout),
@@ -371,7 +371,7 @@ func TestBackupRestoreLagged(t *testing.T) {
}
require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
- needIt, err := needInnoDBRedoLogSubdir()
+ needIt, err := blackbox.NeedInnoDBRedoLogSubdir()
require.NoError(t, err)
if needIt {
newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
@@ -387,7 +387,7 @@ func TestBackupRestoreLagged(t *testing.T) {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -395,7 +395,7 @@ func TestBackupRestoreLagged(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
@@ -406,7 +406,7 @@ func TestBackupRestoreLagged(t *testing.T) {
sourceTablet := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
- sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -414,7 +414,7 @@ func TestBackupRestoreLagged(t *testing.T) {
Sequence: 456,
},
},
- }
+ })
sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)}
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
@@ -448,7 +448,7 @@ func TestBackupRestoreLagged(t *testing.T) {
timer := time.NewTicker(1 * time.Second)
<-timer.C
- sourceTablet.FakeMysqlDaemon.CurrentPrimaryPositionLocked(replication.Position{
+ sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -467,7 +467,7 @@ func TestBackupRestoreLagged(t *testing.T) {
require.NoError(t, sourceTablet.FakeMysqlDaemon.CheckSuperQueryList())
assert.True(t, sourceTablet.FakeMysqlDaemon.Replicating)
assert.True(t, sourceTablet.FakeMysqlDaemon.Running)
- assert.Equal(t, primary.FakeMysqlDaemon.CurrentPrimaryPosition, sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition)
+ assert.Equal(t, primary.FakeMysqlDaemon.GetPrimaryPositionLocked(), sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked())
case <-timer2.C:
require.FailNow(t, "Backup timed out")
}
@@ -476,7 +476,7 @@ func TestBackupRestoreLagged(t *testing.T) {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
- destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -484,7 +484,7 @@ func TestBackupRestoreLagged(t *testing.T) {
Sequence: 456,
},
},
- }
+ })
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP REPLICA",
@@ -503,7 +503,7 @@ func TestBackupRestoreLagged(t *testing.T) {
destTablet.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SHOW DATABASES": {},
}
- destTablet.FakeMysqlDaemon.SetReplicationPositionPos = destTablet.FakeMysqlDaemon.CurrentPrimaryPosition
+ destTablet.FakeMysqlDaemon.SetReplicationPositionPos = destTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))
destTablet.StartActionLoop(t, wr)
@@ -526,7 +526,7 @@ func TestBackupRestoreLagged(t *testing.T) {
timer = time.NewTicker(1 * time.Second)
<-timer.C
- destTablet.FakeMysqlDaemon.CurrentPrimaryPositionLocked(replication.Position{
+ destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -544,7 +544,7 @@ func TestBackupRestoreLagged(t *testing.T) {
require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed")
assert.True(t, destTablet.FakeMysqlDaemon.Replicating)
assert.True(t, destTablet.FakeMysqlDaemon.Running)
- assert.Equal(t, primary.FakeMysqlDaemon.CurrentPrimaryPosition, destTablet.FakeMysqlDaemon.CurrentPrimaryPosition)
+ assert.Equal(t, primary.FakeMysqlDaemon.GetPrimaryPositionLocked(), destTablet.FakeMysqlDaemon.GetPrimaryPositionLocked())
case <-timer2.C:
require.FailNow(t, "Restore timed out")
}
@@ -591,7 +591,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
}
require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
- needIt, err := needInnoDBRedoLogSubdir()
+ needIt, err := blackbox.NeedInnoDBRedoLogSubdir()
require.NoError(t, err)
if needIt {
newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
@@ -607,7 +607,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -615,7 +615,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
@@ -625,7 +625,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
sourceTablet := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
- sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -633,7 +633,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)}
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
@@ -667,7 +667,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
- destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -675,7 +675,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP REPLICA",
@@ -691,7 +691,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
destTablet.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SHOW DATABASES": {},
}
- destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition
+ destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))
destTablet.StartActionLoop(t, wr)
@@ -767,7 +767,7 @@ func TestDisableActiveReparents(t *testing.T) {
}
require.NoError(t, os.WriteFile(path.Join(sourceInnodbDataDir, "innodb_data_1"), []byte("innodb data 1 contents"), os.ModePerm))
- needIt, err := needInnoDBRedoLogSubdir()
+ needIt, err := blackbox.NeedInnoDBRedoLogSubdir()
require.NoError(t, err)
if needIt {
newPath := path.Join(sourceInnodbLogDir, mysql.DynamicRedoLogSubdir)
@@ -783,7 +783,7 @@ func TestDisableActiveReparents(t *testing.T) {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -791,7 +791,7 @@ func TestDisableActiveReparents(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
@@ -802,7 +802,7 @@ func TestDisableActiveReparents(t *testing.T) {
sourceTablet := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
- sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -810,7 +810,7 @@ func TestDisableActiveReparents(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP REPLICA",
}
@@ -835,7 +835,7 @@ func TestDisableActiveReparents(t *testing.T) {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
- destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -843,7 +843,7 @@ func TestDisableActiveReparents(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE RESET BINARY LOGS AND GTIDS",
"FAKE SET GLOBAL gtid_purged",
@@ -855,7 +855,7 @@ func TestDisableActiveReparents(t *testing.T) {
destTablet.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{
"SHOW DATABASES": {},
}
- destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition
+ destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))
destTablet.StartActionLoop(t, wr)
@@ -877,25 +877,3 @@ func TestDisableActiveReparents(t *testing.T) {
assert.False(t, destTablet.FakeMysqlDaemon.Replicating)
assert.True(t, destTablet.FakeMysqlDaemon.Running)
}
-
-// needInnoDBRedoLogSubdir indicates whether we need to create a redo log subdirectory.
-// Starting with MySQL 8.0.30, the InnoDB redo logs are stored in a subdirectory of the
-// (/. by default) called "#innodb_redo". See:
-//
-// https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-modifying-redo-log-capacity
-func needInnoDBRedoLogSubdir() (needIt bool, err error) {
- mysqldVersionStr, err := mysqlctl.GetVersionString()
- if err != nil {
- return needIt, err
- }
- _, sv, err := mysqlctl.ParseVersionString(mysqldVersionStr)
- if err != nil {
- return needIt, err
- }
- versionStr := fmt.Sprintf("%d.%d.%d", sv.Major, sv.Minor, sv.Patch)
- capableOf := mysql.ServerVersionCapableOf(versionStr)
- if capableOf == nil {
- return needIt, fmt.Errorf("cannot determine database flavor details for version %s", versionStr)
- }
- return capableOf(capabilities.DynamicRedoLogCapacityFlavorCapability)
-}
diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go
index 96f9df74405..3167be5e512 100644
--- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go
+++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go
@@ -63,7 +63,7 @@ func TestEmergencyReparentShard(t *testing.T) {
reparenttestutil.SetKeyspaceDurability(context.Background(), t, ts, "test_keyspace", "semi_sync")
oldPrimary.FakeMysqlDaemon.Replicating = false
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -71,7 +71,7 @@ func TestEmergencyReparentShard(t *testing.T) {
Sequence: 456,
},
},
- }
+ })
currentPrimaryFilePosition, _ := replication.ParseFilePosGTIDSet("mariadb-bin.000010:456")
oldPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{
GTIDSet: currentPrimaryFilePosition,
@@ -80,7 +80,7 @@ func TestEmergencyReparentShard(t *testing.T) {
// new primary
newPrimary.FakeMysqlDaemon.ReadOnly = true
newPrimary.FakeMysqlDaemon.Replicating = true
- newPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ newPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -88,7 +88,7 @@ func TestEmergencyReparentShard(t *testing.T) {
Sequence: 456,
},
},
- }
+ })
newPrimaryRelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:456")
newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{
GTIDSet: newPrimaryRelayLogPos,
@@ -123,7 +123,7 @@ func TestEmergencyReparentShard(t *testing.T) {
// good replica 1 is replicating
goodReplica1.FakeMysqlDaemon.ReadOnly = true
goodReplica1.FakeMysqlDaemon.Replicating = true
- goodReplica1.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ goodReplica1.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -131,7 +131,7 @@ func TestEmergencyReparentShard(t *testing.T) {
Sequence: 455,
},
},
- }
+ })
goodReplica1RelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:455")
goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{
GTIDSet: goodReplica1RelayLogPos,
@@ -154,7 +154,7 @@ func TestEmergencyReparentShard(t *testing.T) {
// good replica 2 is not replicating
goodReplica2.FakeMysqlDaemon.ReadOnly = true
goodReplica2.FakeMysqlDaemon.Replicating = false
- goodReplica2.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ goodReplica2.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -162,7 +162,7 @@ func TestEmergencyReparentShard(t *testing.T) {
Sequence: 454,
},
},
- }
+ })
goodReplica2RelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:454")
goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{
GTIDSet: goodReplica2RelayLogPos,
@@ -217,7 +217,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) {
newPrimary.FakeMysqlDaemon.Replicating = true
// It has transactions in its relay log, but not as many as
// moreAdvancedReplica
- newPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ newPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -225,7 +225,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) {
Sequence: 456,
},
},
- }
+ })
newPrimaryRelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:456")
newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{
GTIDSet: newPrimaryRelayLogPos,
@@ -250,7 +250,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) {
// more advanced replica
moreAdvancedReplica.FakeMysqlDaemon.Replicating = true
// relay log position is more advanced than desired new primary
- moreAdvancedReplica.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ moreAdvancedReplica.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
@@ -258,14 +258,14 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) {
Sequence: 457,
},
},
- }
+ })
moreAdvancedReplicaLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:457")
moreAdvancedReplica.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{
GTIDSet: moreAdvancedReplicaLogPos,
}
moreAdvancedReplica.FakeMysqlDaemon.SetReplicationSourceInputs = append(moreAdvancedReplica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet))
moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPositions = append(moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.CurrentSourceFilePosition)
- newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.CurrentPrimaryPosition)
+ newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.GetPrimaryPositionLocked())
moreAdvancedReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP REPLICA",
diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go
index 28ffd34b756..1894c6bb4eb 100644
--- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go
+++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go
@@ -96,7 +96,7 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) {
oldPrimary.FakeMysqlDaemon.ReadOnly = false
oldPrimary.FakeMysqlDaemon.Replicating = false
oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0]
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0])
oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet))
oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE SET SOURCE",
@@ -213,7 +213,7 @@ func TestPlannedReparentShardNoError(t *testing.T) {
oldPrimary.FakeMysqlDaemon.ReadOnly = false
oldPrimary.FakeMysqlDaemon.Replicating = false
oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0]
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0])
oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet))
oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE SET SOURCE",
@@ -434,7 +434,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) {
oldPrimary.FakeMysqlDaemon.ReadOnly = false
oldPrimary.FakeMysqlDaemon.Replicating = false
// set to incorrect value to make promote fail on WaitForReplicationPos
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.PromoteResult
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(newPrimary.FakeMysqlDaemon.PromoteResult)
oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet))
oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE SET SOURCE",
@@ -542,7 +542,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) {
// old primary
oldPrimary.FakeMysqlDaemon.ReadOnly = false
oldPrimary.FakeMysqlDaemon.Replicating = false
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0]
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0])
oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet))
oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE SET SOURCE",
@@ -616,7 +616,7 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) {
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
primary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
7: replication.MariadbGTID{
Domain: 7,
@@ -624,7 +624,7 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) {
Sequence: 990,
},
},
- }
+ })
primary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, primary_alias, replication_position) VALUES",
}
@@ -697,7 +697,7 @@ func TestPlannedReparentShardRelayLogErrorStartReplication(t *testing.T) {
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
primary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
7: replication.MariadbGTID{
Domain: 7,
@@ -705,7 +705,7 @@ func TestPlannedReparentShardRelayLogErrorStartReplication(t *testing.T) {
Sequence: 990,
},
},
- }
+ })
primary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, primary_alias, replication_position) VALUES",
}
@@ -815,7 +815,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) {
oldPrimary.FakeMysqlDaemon.ReadOnly = false
oldPrimary.FakeMysqlDaemon.Replicating = false
oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0]
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0])
oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet))
oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE SET SOURCE",
@@ -823,7 +823,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) {
// We call a SetReplicationSource explicitly
"FAKE SET SOURCE",
"START REPLICA",
- // extra SetReplicationSource call due to retry
+ // extra SetReplicationSource call due to retry)
"FAKE SET SOURCE",
"START REPLICA",
}
@@ -885,7 +885,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) {
// retrying should work
newPrimary.FakeMysqlDaemon.PromoteError = nil
- newPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0]
+ newPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0])
// run PlannedReparentShard
err = vp.Run([]string{"PlannedReparentShard", "--wait_replicas_timeout", "10s", "--keyspace_shard", newPrimary.Tablet.Keyspace + "/" + newPrimary.Tablet.Shard, "--new_primary", topoproto.TabletAliasString(newPrimary.Tablet.Alias)})
@@ -922,7 +922,7 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) {
oldPrimary.FakeMysqlDaemon.ReadOnly = true
oldPrimary.FakeMysqlDaemon.Replicating = false
oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica
- oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
7: replication.MariadbGTID{
Domain: 7,
@@ -930,7 +930,7 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) {
Sequence: 990,
},
},
- }
+ })
oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, primary_alias, replication_position) VALUES",
}
diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go
index ea2e34b66bd..b199a64340a 100644
--- a/go/vt/wrangler/testlib/reparent_utils_test.go
+++ b/go/vt/wrangler/testlib/reparent_utils_test.go
@@ -67,7 +67,7 @@ func TestShardReplicationStatuses(t *testing.T) {
}
// primary action loop (to initialize host and port)
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
5: replication.MariadbGTID{
Domain: 5,
@@ -75,12 +75,12 @@ func TestShardReplicationStatuses(t *testing.T) {
Sequence: 892,
},
},
- }
+ })
primary.StartActionLoop(t, wr)
defer primary.StopActionLoop(t)
// replica loop
- replica.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ replica.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
5: replication.MariadbGTID{
Domain: 5,
@@ -88,7 +88,7 @@ func TestShardReplicationStatuses(t *testing.T) {
Sequence: 890,
},
},
- }
+ })
replica.FakeMysqlDaemon.CurrentSourceHost = primary.Tablet.MysqlHostname
replica.FakeMysqlDaemon.CurrentSourcePort = primary.Tablet.MysqlPort
replica.FakeMysqlDaemon.SetReplicationSourceInputs = append(replica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))
@@ -207,7 +207,8 @@ func TestSetReplicationSource(t *testing.T) {
require.NoError(t, err, "UpdateShardFields failed")
pos, err := replication.DecodePosition("MySQL56/8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-8")
require.NoError(t, err)
- primary.FakeMysqlDaemon.CurrentPrimaryPositionLocked(pos)
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(pos)
+ primary.FakeMysqlDaemon.ServerUUID = "8bc65c84-3fe4-11ed-a912-257f0fcdd6c9"
// primary action loop (to initialize host and port)
primary.StartActionLoop(t, wr)
diff --git a/go/vt/wrangler/traffic_switcher_env_test.go b/go/vt/wrangler/traffic_switcher_env_test.go
index 4e58024785d..a99e6ba2c43 100644
--- a/go/vt/wrangler/traffic_switcher_env_test.go
+++ b/go/vt/wrangler/traffic_switcher_env_test.go
@@ -776,7 +776,7 @@ func (tme *testMigraterEnv) createDBClients(ctx context.Context, t *testing.T) {
func (tme *testMigraterEnv) setPrimaryPositions() {
for _, primary := range tme.sourcePrimaries {
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
5: replication.MariadbGTID{
Domain: 5,
@@ -784,10 +784,10 @@ func (tme *testMigraterEnv) setPrimaryPositions() {
Sequence: 892,
},
},
- }
+ })
}
for _, primary := range tme.targetPrimaries {
- primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
+ primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
5: replication.MariadbGTID{
Domain: 5,
@@ -795,7 +795,7 @@ func (tme *testMigraterEnv) setPrimaryPositions() {
Sequence: 893,
},
},
- }
+ })
}
}
diff --git a/java/client/pom.xml b/java/client/pom.xml
index fc78d17b564..1cb1ecb5862 100644
--- a/java/client/pom.xml
+++ b/java/client/pom.xml
@@ -5,7 +5,7 @@
io.vitessvitess-parent
- 21.0.0-SNAPSHOT
+ 22.0.0-SNAPSHOTvitess-client
diff --git a/java/client/src/main/java/io/vitess/client/VTGateBlockingConnection.java b/java/client/src/main/java/io/vitess/client/VTGateBlockingConnection.java
index 4c24c12bfcf..8a19659ef65 100644
--- a/java/client/src/main/java/io/vitess/client/VTGateBlockingConnection.java
+++ b/java/client/src/main/java/io/vitess/client/VTGateBlockingConnection.java
@@ -111,9 +111,6 @@ public Cursor streamExecute(Context ctx,
return vtGateConnection.streamExecute(ctx, query, bindVars, vtSession);
}
- /**
- * @inheritDoc
- */
@Override
public void close() throws IOException {
vtGateConnection.close();
diff --git a/java/client/src/main/java/io/vitess/client/VTGateConnection.java b/java/client/src/main/java/io/vitess/client/VTGateConnection.java
index ee2fb270a9a..683f4bf5f15 100644
--- a/java/client/src/main/java/io/vitess/client/VTGateConnection.java
+++ b/java/client/src/main/java/io/vitess/client/VTGateConnection.java
@@ -205,9 +205,6 @@ StreamIterator getVStream(Context ctx, VStreamRequest vstreamRe
return client.getVStream(ctx, request);
}
- /**
- * @inheritDoc
- */
@Override
public void close() throws IOException {
client.close();
diff --git a/java/client/src/main/java/io/vitess/client/VTSession.java b/java/client/src/main/java/io/vitess/client/VTSession.java
index 9974e5f034a..3b52b4d3c2e 100644
--- a/java/client/src/main/java/io/vitess/client/VTSession.java
+++ b/java/client/src/main/java/io/vitess/client/VTSession.java
@@ -103,7 +103,7 @@ public Query.ExecuteOptions.TransactionIsolation getTransactionIsolation() {
/**
* Sets this session's transaction isolation level.
*
- * @param Transaction Isolation Level of the Session
+ * @param isolation Isolation Level of the Session
*/
public void setTransactionIsolation(Query.ExecuteOptions.TransactionIsolation isolation) {
this.session = this.session.toBuilder()
diff --git a/java/client/src/main/java/io/vitess/client/grpc/tls/TlsOptions.java b/java/client/src/main/java/io/vitess/client/grpc/tls/TlsOptions.java
index ffd570d413b..4defac70378 100644
--- a/java/client/src/main/java/io/vitess/client/grpc/tls/TlsOptions.java
+++ b/java/client/src/main/java/io/vitess/client/grpc/tls/TlsOptions.java
@@ -17,11 +17,10 @@
package io.vitess.client.grpc.tls;
import java.io.File;
-import java.net.InetSocketAddress;
/**
*
A wrapper type holding TLS-related fields for the
- * {@link io.vitess.client.RpcClientFactory#createTls(InetSocketAddress, TlsOptions)} method, so
+ * createTls(InetSocketAddress, TlsOptions) method, so
* that this method won't have an unwieldy number of direct parameters.
*
*
This path uses a builder pattern style:
diff --git a/java/example/pom.xml b/java/example/pom.xml
index c2b226b6806..fa3220f51bd 100644
--- a/java/example/pom.xml
+++ b/java/example/pom.xml
@@ -5,7 +5,7 @@
io.vitessvitess-parent
- 21.0.0-SNAPSHOT
+ 22.0.0-SNAPSHOTvitess-example
diff --git a/java/grpc-client/pom.xml b/java/grpc-client/pom.xml
index 41bd4de8291..55c406b0e8f 100644
--- a/java/grpc-client/pom.xml
+++ b/java/grpc-client/pom.xml
@@ -5,7 +5,7 @@
io.vitessvitess-parent
- 21.0.0-SNAPSHOT
+ 22.0.0-SNAPSHOTvitess-grpc-client
@@ -51,6 +51,10 @@
io.grpcgrpc-api
+
+ io.grpc
+ grpc-inprocess
+ io.nettynetty-handler
diff --git a/java/grpc-client/src/main/java/io/vitess/client/grpc/RetryingInterceptor.java b/java/grpc-client/src/main/java/io/vitess/client/grpc/RetryingInterceptor.java
index 9323a1b00d4..301ddaed282 100644
--- a/java/grpc-client/src/main/java/io/vitess/client/grpc/RetryingInterceptor.java
+++ b/java/grpc-client/src/main/java/io/vitess/client/grpc/RetryingInterceptor.java
@@ -39,8 +39,8 @@
/**
* RetryingInterceptor is used for retrying certain classes of failed requests in the underlying
- * gRPC connection. At this time it handles {@link MethodDescriptor.MethodType.UNARY} requests with
- * status {@link Status.Code.UNAVAILABLE}, which is according to the spec meant to be a transient
+ * gRPC connection. At this time it handles unary requests with
+ * status Unavailable, which is according to the spec meant to be a transient
* error. This class can be configured with {@link RetryingInterceptorConfig} to determine what
* level of exponential backoff to apply to the handled types of failing requests.
*
diff --git a/java/grpc-client/src/test/java/io/vitess/client/grpc/RetryingInterceptorTest.java b/java/grpc-client/src/test/java/io/vitess/client/grpc/RetryingInterceptorTest.java
index 376a920ad31..9b3644139fd 100644
--- a/java/grpc-client/src/test/java/io/vitess/client/grpc/RetryingInterceptorTest.java
+++ b/java/grpc-client/src/test/java/io/vitess/client/grpc/RetryingInterceptorTest.java
@@ -55,6 +55,8 @@ public void testNoopConfigPassesThrough() throws ExecutionException, Interrupted
Assert.fail("Should have failed after 1 attempt");
} catch (Exception e) {
Assert.assertEquals(1, forceRetryNTimesInterceptor.getNumRetryableFailures());
+ }finally {
+ channel.shutdownNow();
}
}
@@ -75,6 +77,8 @@ public void testRetryAfterBackoff() throws ExecutionException, InterruptedExcept
} catch (Exception e) {
e.printStackTrace();
Assert.assertEquals(3, forceRetryNTimesInterceptor.getNumRetryableFailures());
+ } finally {
+ channel.shutdownNow();
}
}
@@ -95,6 +99,8 @@ public void testRetryDeadlineExceeded() throws ExecutionException, InterruptedEx
Assert.fail("Should have failed");
} catch (Exception e) {
Assert.assertEquals(1, forceRetryNTimesInterceptor.getNumRetryableFailures());
+ }finally {
+ channel.shutdownNow();
}
}
diff --git a/java/jdbc/pom.xml b/java/jdbc/pom.xml
index 49a5d61779b..dd554e64501 100644
--- a/java/jdbc/pom.xml
+++ b/java/jdbc/pom.xml
@@ -5,7 +5,7 @@
io.vitessvitess-parent
- 21.0.0-SNAPSHOT
+ 22.0.0-SNAPSHOTvitess-jdbc
diff --git a/java/jdbc/src/main/java/io/vitess/jdbc/VitessConnection.java b/java/jdbc/src/main/java/io/vitess/jdbc/VitessConnection.java
index 5e51ff908dc..f4bc43ed65e 100644
--- a/java/jdbc/src/main/java/io/vitess/jdbc/VitessConnection.java
+++ b/java/jdbc/src/main/java/io/vitess/jdbc/VitessConnection.java
@@ -105,7 +105,7 @@ public Statement createStatement() throws SQLException {
}
/**
- * Create PreparedStatement for the given connection & sql
+ * Create PreparedStatement for the given connection and sql
*
* @param sql - Sql Statement
* @return PreparedStatement Object
@@ -338,7 +338,6 @@ public void setTransactionIsolation(int level) throws SQLException {
/**
* Return Warnings
- *
* TODO: Not implementing as Error is Thrown when occurred
*
* @return SQLWarning or null
diff --git a/java/jdbc/src/main/java/io/vitess/jdbc/VitessDriver.java b/java/jdbc/src/main/java/io/vitess/jdbc/VitessDriver.java
index 5b6a889103f..a77e1ec0860 100644
--- a/java/jdbc/src/main/java/io/vitess/jdbc/VitessDriver.java
+++ b/java/jdbc/src/main/java/io/vitess/jdbc/VitessDriver.java
@@ -60,12 +60,10 @@ public Connection connect(String url, Properties info) throws SQLException {
/**
* Checks whether a given url is in a valid format.
- *
* The current uri format is: jdbc:vitess://[host]:[port]
*
* @param url the URL of the database
* @return true, if this driver understands the given URL; false, otherwise
- *
* TODO: Write a better regex
*/
@Override
diff --git a/java/jdbc/src/main/java/io/vitess/jdbc/VitessJDBCUrl.java b/java/jdbc/src/main/java/io/vitess/jdbc/VitessJDBCUrl.java
index 6234d037807..e6a1b0187dd 100644
--- a/java/jdbc/src/main/java/io/vitess/jdbc/VitessJDBCUrl.java
+++ b/java/jdbc/src/main/java/io/vitess/jdbc/VitessJDBCUrl.java
@@ -93,7 +93,6 @@ public String toString() {
*
trustAlias=alias_under_which_certificate_chain_is_stored (if not set,
* then the first valid X509Certificate found in the trustStore will be used)
*
- *
*
*
If useSSL=true, and any of these additional properties are not set on the JDBC
* URL, then the driver will look to see if these corresponding property was set at JVM startup
@@ -109,7 +108,6 @@ public String toString() {
*
diff --git a/java/jdbc/src/main/java/io/vitess/jdbc/VitessStatement.java b/java/jdbc/src/main/java/io/vitess/jdbc/VitessStatement.java
index ddaf813af2d..90a83b56485 100644
--- a/java/jdbc/src/main/java/io/vitess/jdbc/VitessStatement.java
+++ b/java/jdbc/src/main/java/io/vitess/jdbc/VitessStatement.java
@@ -241,7 +241,6 @@ public void setQueryTimeout(int seconds) throws SQLException {
/**
* Return Warnings
- *
* Not implementing as Error is Thrown when occurred
*
* @return SQLWarning or null
diff --git a/java/jdbc/src/main/java/io/vitess/util/charset/CharsetMapping.java b/java/jdbc/src/main/java/io/vitess/util/charset/CharsetMapping.java
index c1388bfb7bb..343ae0e90eb 100644
--- a/java/jdbc/src/main/java/io/vitess/util/charset/CharsetMapping.java
+++ b/java/jdbc/src/main/java/io/vitess/util/charset/CharsetMapping.java
@@ -541,6 +541,7 @@ public class CharsetMapping {
/**
* MySQL charset could map to several Java encodings. So here we choose the one according to next
* rules:
+ *
*
if there is no static mapping for this charset then return javaEncoding value as is
* because this could be a custom charset for example
*
if static mapping exists and javaEncoding equals to one of Java encoding canonical names
@@ -550,6 +551,7 @@ public class CharsetMapping {
*
if static mapping exists and javaEncoding doesn't match any Java encoding canonical
* names or aliases available for this mapping then return default Java encoding (the first in
* mapping list)
+ *
*/
public static String getJavaEncodingForCollationIndex(Integer collationIndex,
String javaEncoding) {
diff --git a/java/pom.xml b/java/pom.xml
index 8d8f6ab9275..6742258a6b8 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -11,7 +11,7 @@
io.vitessvitess-parent
- 21.0.0-SNAPSHOT
+ 22.0.0-SNAPSHOTpomVitess Java Client libraries [Parent]
@@ -66,16 +66,16 @@
UTF-8
- 1.57.1
+ 1.67.1
- 4.1.94.Final
- 2.0.61.Final
+ 4.1.110.Final
+ 2.0.65.Final
- 3.25.5
+ 4.28.33.24.33.0.0
- 2.17.1
+ 2.24.1
@@ -89,12 +89,12 @@
com.google.code.gsongson
- 2.8.9
+ 2.11.0com.google.guavaguava
- 32.0.0-jre
+ 33.3.1-jrecom.google.protobuf
@@ -109,7 +109,7 @@
commons-iocommons-io
- 2.7
+ 2.17.0
@@ -122,6 +122,11 @@
grpc-netty${grpc.version}
+
+ io.grpc
+ grpc-inprocess
+ ${grpc.version}
+ io.grpcgrpc-stub
@@ -180,7 +185,7 @@
joda-timejoda-time
- 2.10
+ 2.13.0
@@ -227,7 +232,7 @@
org.apache.maven.pluginsmaven-compiler-plugin
- 3.8.1
+ 3.13.01.81.8
@@ -288,7 +293,7 @@
org.sonatype.pluginsnexus-staging-maven-plugin
- 1.6.13
+ 1.7.0trueossrh
@@ -299,7 +304,7 @@
org.apache.maven.pluginsmaven-source-plugin
- 2.4
+ 3.3.1attach-sources
@@ -312,7 +317,7 @@
org.apache.maven.pluginsmaven-javadoc-plugin
- 2.10.4
+ 3.3.1attach-javadocs
@@ -325,7 +330,7 @@
org.apache.maven.pluginsmaven-gpg-plugin
- 1.6
+ 3.2.7sign-artifacts
diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto
index 595760dcd52..e1df792776b 100644
--- a/proto/binlogdata.proto
+++ b/proto/binlogdata.proto
@@ -353,6 +353,10 @@ message FieldEvent {
repeated query.Field fields = 2;
string keyspace = 3;
string shard = 4;
+
+ // Field numbers in the gap between shard (4) and enum_set_string_values
+ // (25) are NOT reserved and can be used.
+
// Are ENUM and SET field values already mapped to strings in the ROW
// events? This allows us to transition VTGate VStream consumers from
// the pre v20 behavior of having to do this mapping themselves to the
@@ -362,6 +366,9 @@ message FieldEvent {
// vstreams managed by the vstreamManager.
bool enum_set_string_values = 25;
bool is_internal_table = 26; // set for sidecardb tables
+
+ // Add new members in the field number gap between shard (4) and
+ // enum_set_string_values (25).
}
// ShardGtid contains the GTID position for one shard.
diff --git a/proto/query.proto b/proto/query.proto
index 917883e5ad8..dd690788004 100644
--- a/proto/query.proto
+++ b/proto/query.proto
@@ -610,8 +610,16 @@ message StartCommitRequest {
string dtid = 5;
}
+enum StartCommitState {
+ Unknown = 0;
+ Fail = 1;
+ Success = 2;
+}
+
// StartCommitResponse is the returned value from StartCommit
-message StartCommitResponse {}
+message StartCommitResponse {
+ StartCommitState state = 1;
+}
// SetRollbackRequest is the payload to SetRollback
message SetRollbackRequest {
diff --git a/proto/replicationdata.proto b/proto/replicationdata.proto
index 1a8b608f984..e788fc64bc3 100644
--- a/proto/replicationdata.proto
+++ b/proto/replicationdata.proto
@@ -50,6 +50,7 @@ message Status {
bool has_replication_filters = 22;
bool ssl_allowed = 23;
bool replication_lag_unknown = 24;
+ bool backup_running = 25;
}
// Configuration holds replication configuration information gathered from performance_schema and global variables.
@@ -77,6 +78,7 @@ enum StopReplicationMode {
message PrimaryStatus {
string position = 1;
string file_position = 2;
+ string server_uuid = 3;
}
// FullStatus contains the full status of MySQL including the replication information, semi-sync information, GTID information among others
diff --git a/proto/tabletmanagerdata.proto b/proto/tabletmanagerdata.proto
index 772501d8c57..bb20e712e7f 100644
--- a/proto/tabletmanagerdata.proto
+++ b/proto/tabletmanagerdata.proto
@@ -326,6 +326,18 @@ message ReadTransactionResponse {
query.TransactionMetadata transaction = 1;
}
+message GetTransactionInfoRequest {
+ string dtid = 1;
+}
+
+message GetTransactionInfoResponse {
+ string state = 1;
+ string message = 2;
+ int64 time_created = 3;
+ repeated string statements = 4;
+}
+
+
message ConcludeTransactionRequest {
string dtid = 1;
bool mm = 2;
@@ -448,6 +460,13 @@ message PopulateReparentJournalRequest {
message PopulateReparentJournalResponse {
}
+message ReadReparentJournalInfoRequest {
+}
+
+message ReadReparentJournalInfoResponse {
+ int32 length = 1;
+}
+
message InitReplicaRequest {
topodata.TabletAlias parent = 1;
string replication_position = 2;
@@ -605,6 +624,18 @@ message CreateVReplicationWorkflowResponse {
query.QueryResult result = 1;
}
+message DeleteTableDataRequest {
+ // The key is the table that we want to delete data from.
+ // The value is the filter or WHERE clause to use when deleting
+ // data in the table.
+ map table_filters = 1;
+ // BatchSize is the number of rows to delete in a single batch.
+ int64 batch_size = 2;
+}
+
+message DeleteTableDataResponse {
+}
+
message DeleteVReplicationWorkflowRequest {
string workflow = 1;
}
@@ -908,3 +939,12 @@ message GetThrottlerStatusResponse {
// RecentApps is a map of app names to their recent check status
map recent_apps = 18;
}
+
+message ChangeTagsRequest {
+ map tags = 1;
+ bool replace = 2;
+}
+
+message ChangeTagsResponse {
+ map tags = 1;
+}
diff --git a/proto/tabletmanagerservice.proto b/proto/tabletmanagerservice.proto
index 0fe337c75c0..3dac1aa1ee8 100644
--- a/proto/tabletmanagerservice.proto
+++ b/proto/tabletmanagerservice.proto
@@ -57,6 +57,9 @@ service TabletManager {
rpc SetReadWrite(tabletmanagerdata.SetReadWriteRequest) returns (tabletmanagerdata.SetReadWriteResponse) {};
+ // ChangeTags asks the remote tablet to change its tags
+ rpc ChangeTags(tabletmanagerdata.ChangeTagsRequest) returns (tabletmanagerdata.ChangeTagsResponse) {};
+
// ChangeType asks the remote tablet to change its type
rpc ChangeType(tabletmanagerdata.ChangeTypeRequest) returns (tabletmanagerdata.ChangeTypeResponse) {};
@@ -94,6 +97,8 @@ service TabletManager {
rpc ReadTransaction(tabletmanagerdata.ReadTransactionRequest) returns (tabletmanagerdata.ReadTransactionResponse) {};
+ rpc GetTransactionInfo(tabletmanagerdata.GetTransactionInfoRequest) returns (tabletmanagerdata.GetTransactionInfoResponse) {};
+
rpc ConcludeTransaction(tabletmanagerdata.ConcludeTransactionRequest) returns (tabletmanagerdata.ConcludeTransactionResponse) {};
rpc MysqlHostMetrics(tabletmanagerdata.MysqlHostMetricsRequest) returns (tabletmanagerdata.MysqlHostMetricsResponse) {};
@@ -133,6 +138,7 @@ service TabletManager {
// VReplication API
rpc CreateVReplicationWorkflow(tabletmanagerdata.CreateVReplicationWorkflowRequest) returns (tabletmanagerdata.CreateVReplicationWorkflowResponse) {};
+ rpc DeleteTableData(tabletmanagerdata.DeleteTableDataRequest) returns(tabletmanagerdata.DeleteTableDataResponse) {}
rpc DeleteVReplicationWorkflow(tabletmanagerdata.DeleteVReplicationWorkflowRequest) returns(tabletmanagerdata.DeleteVReplicationWorkflowResponse) {};
rpc HasVReplicationWorkflows(tabletmanagerdata.HasVReplicationWorkflowsRequest) returns(tabletmanagerdata.HasVReplicationWorkflowsResponse) {};
rpc ReadVReplicationWorkflow(tabletmanagerdata.ReadVReplicationWorkflowRequest) returns(tabletmanagerdata.ReadVReplicationWorkflowResponse) {};
@@ -160,6 +166,9 @@ service TabletManager {
// reparent journal
rpc PopulateReparentJournal(tabletmanagerdata.PopulateReparentJournalRequest) returns (tabletmanagerdata.PopulateReparentJournalResponse) {};
+ // ReadReparentJournalInfo reads the information from reparent journal
+ rpc ReadReparentJournalInfo(tabletmanagerdata.ReadReparentJournalInfoRequest) returns (tabletmanagerdata.ReadReparentJournalInfoResponse) {};
+
// InitReplica tells the tablet to reparent to the primary unconditionally
rpc InitReplica(tabletmanagerdata.InitReplicaRequest) returns (tabletmanagerdata.InitReplicaResponse) {};
diff --git a/proto/vtadmin.proto b/proto/vtadmin.proto
index a8e6775de98..963d1fa5779 100644
--- a/proto/vtadmin.proto
+++ b/proto/vtadmin.proto
@@ -44,6 +44,8 @@ service VTAdmin {
// CompleteSchemaMigration completes one or all migrations in the given
// cluster executed with --postpone-completion.
rpc CompleteSchemaMigration(CompleteSchemaMigrationRequest) returns (vtctldata.CompleteSchemaMigrationResponse) {};
+ // ConcludeTransaction concludes a distributed transaction identified by the provided dtid.
+ rpc ConcludeTransaction(ConcludeTransactionRequest) returns (vtctldata.ConcludeTransactionResponse) {};
// CreateKeyspace creates a new keyspace in the given cluster.
rpc CreateKeyspace(CreateKeyspaceRequest) returns (CreateKeyspaceResponse) {};
// CreateShard creates a new shard in the given cluster and keyspace.
@@ -114,6 +116,8 @@ service VTAdmin {
rpc GetTablets(GetTabletsRequest) returns (GetTabletsResponse) {};
// GetTopologyPath returns the cell located at the specified path in the topology server.
rpc GetTopologyPath(GetTopologyPathRequest) returns (vtctldata.GetTopologyPathResponse){};
+ // GetTransactionInfo returns the information about a single transaction.
+ rpc GetTransactionInfo(GetTransactionInfoRequest) returns (vtctldata.GetTransactionInfoResponse){};
// GetUnresolvedTransactions returns the unresolved transactions for the request.
rpc GetUnresolvedTransactions(GetUnresolvedTransactionsRequest) returns (vtctldata.GetUnresolvedTransactionsResponse){};
// GetVSchema returns a VSchema for the specified keyspace in the specified
@@ -137,9 +141,15 @@ service VTAdmin {
// LaunchSchemaMigration launches one or all migrations in the given
// cluster executed with --postpone-launch.
rpc LaunchSchemaMigration(LaunchSchemaMigrationRequest) returns (vtctldata.LaunchSchemaMigrationResponse) {};
+ // MoveTablesComplete completes the move and cleans up the workflow and
+ // its related artifacts.
+ rpc MoveTablesComplete(MoveTablesCompleteRequest) returns (vtctldata.MoveTablesCompleteResponse) {};
// MoveTablesCreate creates a workflow which moves one or more tables from a
// source keyspace to a target keyspace.
rpc MoveTablesCreate(MoveTablesCreateRequest) returns (vtctldata.WorkflowStatusResponse) {};
+ // MaterializeCreate creates a workflow to materialize one or more tables
+ // from a source keyspace to a target keyspace using a provided expressions.
+ rpc MaterializeCreate(MaterializeCreateRequest) returns (vtctldata.MaterializeCreateResponse) {};
// PingTablet checks that the specified tablet is awake and responding to
// RPCs. This command can be blocked by other in-flight operations.
rpc PingTablet(PingTabletRequest) returns (PingTabletResponse) {};
@@ -171,6 +181,8 @@ service VTAdmin {
rpc RetrySchemaMigration(RetrySchemaMigrationRequest) returns (vtctldata.RetrySchemaMigrationResponse) {};
// RunHealthCheck runs a healthcheck on the tablet.
rpc RunHealthCheck(RunHealthCheckRequest) returns (RunHealthCheckResponse) {};
+ // ReshardCreate creates a workflow to reshard a keyspace.
+ rpc ReshardCreate(ReshardCreateRequest) returns (vtctldata.WorkflowStatusResponse) {};
// SetReadOnly sets the tablet to read-only mode.
rpc SetReadOnly(SetReadOnlyRequest) returns (SetReadOnlyResponse) {};
// SetReadWrite sets the tablet to read-write mode.
@@ -208,9 +220,15 @@ service VTAdmin {
rpc ValidateVersionKeyspace(ValidateVersionKeyspaceRequest) returns (vtctldata.ValidateVersionKeyspaceResponse) {};
// ValidateVersionShard validates that the version on the primary matches all of the replicas.
rpc ValidateVersionShard(ValidateVersionShardRequest) returns (vtctldata.ValidateVersionShardResponse) {};
+ rpc VDiffCreate(VDiffCreateRequest) returns (vtctldata.VDiffCreateResponse) {};
+ rpc VDiffShow(VDiffShowRequest) returns (VDiffShowResponse) {};
// VTExplain provides information on how Vitess plans to execute a
// particular query.
rpc VTExplain(VTExplainRequest) returns (VTExplainResponse) {};
+ // WorkflowDelete deletes a vreplication workflow.
+ rpc WorkflowDelete(WorkflowDeleteRequest) returns (vtctldata.WorkflowDeleteResponse) {};
+ // WorkflowSwitchTraffic switches traffic for a VReplication workflow.
+ rpc WorkflowSwitchTraffic(WorkflowSwitchTrafficRequest) returns (vtctldata.WorkflowSwitchTrafficResponse) {};
}
/* Data types */
@@ -356,11 +374,25 @@ message Workflow {
vtctldata.Workflow workflow = 3;
}
+message WorkflowDeleteRequest {
+ string cluster_id = 1;
+ vtctldata.WorkflowDeleteRequest request = 2;
+}
+
+message WorkflowSwitchTrafficRequest {
+ string cluster_id = 1;
+ vtctldata.WorkflowSwitchTrafficRequest request = 2;
+}
+
/* Request/Response types */
message ApplySchemaRequest {
string cluster_id = 1;
- vtctldata.ApplySchemaRequest request = 2;
+ // Request.Sql will be overriden by this Sql field.
+ string sql = 2;
+ // Request.CallerId will be overriden by this CallerId field.
+ string caller_id = 3;
+ vtctldata.ApplySchemaRequest request = 4;
}
message CancelSchemaMigrationRequest {
@@ -378,6 +410,11 @@ message CompleteSchemaMigrationRequest {
vtctldata.CompleteSchemaMigrationRequest request = 2;
}
+message ConcludeTransactionRequest {
+ string cluster_id = 1;
+ string dtid = 2;
+}
+
message CreateKeyspaceRequest {
string cluster_id = 1;
vtctldata.CreateKeyspaceRequest options = 2;
@@ -624,9 +661,15 @@ message GetTopologyPathRequest {
string path = 2;
}
+message GetTransactionInfoRequest {
+ string cluster_id = 1;
+ vtctldata.GetTransactionInfoRequest request = 2;
+}
+
message GetUnresolvedTransactionsRequest {
string cluster_id = 1;
string keyspace = 2;
+ int64 abandon_age = 3;
}
message GetVSchemaRequest {
@@ -706,6 +749,19 @@ message LaunchSchemaMigrationRequest {
vtctldata.LaunchSchemaMigrationRequest request = 2;
}
+message MaterializeCreateRequest {
+ string cluster_id = 1;
+ // TableSettings is a JSON string defining what tables to materialize using
+ // what select statements.
+ string table_settings = 2;
+ vtctldata.MaterializeCreateRequest request = 3;
+}
+
+message MoveTablesCompleteRequest {
+ string cluster_id = 1;
+ vtctldata.MoveTablesCompleteRequest request = 2;
+}
+
message MoveTablesCreateRequest {
string cluster_id = 1;
vtctldata.MoveTablesCreateRequest request = 2;
@@ -909,6 +965,11 @@ message RunHealthCheckResponse {
Cluster cluster = 2;
}
+message ReshardCreateRequest {
+ string cluster_id = 1;
+ vtctldata.ReshardCreateRequest request = 2;
+}
+
message SetReadOnlyRequest {
topodata.TabletAlias alias = 1;
repeated string cluster_ids = 2;
@@ -999,6 +1060,34 @@ message ValidateVersionShardRequest {
string shard = 3;
}
+message VDiffCreateRequest {
+ string cluster_id = 1;
+ vtctldata.VDiffCreateRequest request = 2;
+}
+
+message VDiffShowRequest {
+ string cluster_id = 1;
+ vtctldata.VDiffShowRequest request = 2;
+}
+
+message VDiffProgress {
+ double percentage = 1;
+ string eta = 2;
+}
+
+message VDiffShardReport {
+ string state = 1;
+ int64 rows_compared = 2;
+ bool has_mismatch = 3;
+ string started_at = 4;
+ string completed_at = 5;
+ VDiffProgress progress = 6;
+}
+
+message VDiffShowResponse {
+ map shard_report = 1;
+}
+
message VTExplainRequest {
string cluster = 1;
string keyspace = 2;
diff --git a/proto/vtctldata.proto b/proto/vtctldata.proto
index 49b541ea2c5..b1e5fb215bc 100644
--- a/proto/vtctldata.proto
+++ b/proto/vtctldata.proto
@@ -209,16 +209,25 @@ message Shard {
topodata.Shard shard = 3;
}
+enum ShardedAutoIncrementHandling {
+ LEAVE = 0;
+ REMOVE = 1;
+ REPLACE = 2;
+}
+
message WorkflowOptions {
string tenant_id = 1;
// Remove auto_increment clauses on tables when moving them to a sharded
- // keyspace.
- bool strip_sharded_auto_increment = 2;
+ // keyspace and optionally replace them with vschema AutoIncrement
+ // definitions.
+ ShardedAutoIncrementHandling sharded_auto_increment_handling = 2;
// Shards on which vreplication streams in the target keyspace are created for this workflow and to which the data
// from the source will be vreplicated.
repeated string shards = 3;
map config = 4;
-
+ // Where to create any related schema and vschema objects such as
+ // sequence tables.
+ string global_keyspace = 5;
}
// TODO: comment the hell out of this.
@@ -488,6 +497,17 @@ message CancelSchemaMigrationResponse {
map rows_affected_by_shard = 1;
}
+message ChangeTabletTagsRequest {
+ topodata.TabletAlias tablet_alias = 1;
+ map tags = 2;
+ bool replace = 3;
+}
+
+message ChangeTabletTagsResponse {
+ map before_tags = 1;
+ map after_tags = 2;
+}
+
message ChangeTabletTypeRequest {
topodata.TabletAlias tablet_alias = 1;
topodata.TabletType db_type = 2;
@@ -687,6 +707,9 @@ message EmergencyReparentShardRequest {
// WaitForAllTablets makes ERS wait for a response from all the tablets before proceeding.
// Useful when all the tablets are up and reachable.
bool wait_for_all_tablets = 7;
+ // ExpectedPrimary is the optional alias we expect to be the current primary in order for
+ // the reparent operation to succeed.
+ topodata.TabletAlias expected_primary = 8;
}
message EmergencyReparentShardResponse {
@@ -1117,6 +1140,24 @@ message GetUnresolvedTransactionsResponse {
repeated query.TransactionMetadata transactions = 1;
}
+message GetTransactionInfoRequest {
+ string dtid = 1;
+}
+
+message ShardTransactionState {
+ string shard = 1;
+ string state = 2;
+ string message = 3;
+ int64 time_created = 4;
+ repeated string statements = 5;
+}
+
+message GetTransactionInfoResponse {
+ query.TransactionMetadata metadata = 1;
+ repeated ShardTransactionState shard_states = 2;
+}
+
+
message ConcludeTransactionRequest {
string dtid = 1;
repeated query.Target participants = 2;
@@ -1383,6 +1424,9 @@ message PlannedReparentShardRequest {
vttime.Duration tolerable_replication_lag = 6;
// AllowCrossCellPromotion allows cross cell promotion,
bool allow_cross_cell_promotion = 7;
+ // ExpectedPrimary is the optional alias we expect to be the current primary in order for
+ // the reparent operation to succeed.
+ topodata.TabletAlias expected_primary = 8;
}
message PlannedReparentShardResponse {
@@ -1870,27 +1914,85 @@ message ValidateVSchemaResponse {
}
message VDiffCreateRequest {
+ // The name of the workflow that we're diffing tables for.
string workflow = 1;
+ // The keyspace where the vreplication workflow is running.
string target_keyspace = 2;
+ // A unique identifier for the vdiff.
+ // If empty, a new UUID will be generated.
string uuid = 3;
+ // The cells to look for source tablets in.
+ // If empty, all cells are used.
repeated string source_cells = 4;
+ // The cells to look for target tablets in.
+ // If empty, all cells are used.
repeated string target_cells = 5;
+ // The tablet types to use when searching for tablets to use when streaming
+ // results.
+ // A default value of "replica,rdonly,primary" is used by the tablet picker.
repeated topodata.TabletType tablet_types = 6;
+ // When performing source tablet selection, look for candidates in the type
+ // order as they are listed in the tablet_types value (or the default of
+ // "replica,rdonly,primary" that the tablet picker uses).
+ // The default is ANY (0) and you can use INORDER (1) to e.g. ensure that a
+ // primary tablet is only used if there are no available replica or rdonly
+ // tablets.
tabletmanagerdata.TabletSelectionPreference tablet_selection_preference = 7;
+ // The tables to compare. If empty, all tables in the workflow are compared.
repeated string tables = 8;
+ // The maximum number of rows to compare for each table on each shard.
+ // The default is a max int64 value: 2^63 - 1 or 9,223,372,036,854,775,807.
int64 limit = 9;
+ // How long to wait for the relevant vreplication stream(s) to catch up when
+ // attempting to setup the table snapshots on the source and target to use for
+ // the diff on each shard.
+ // The default is 30s.
vttime.Duration filtered_replication_wait_time = 10;
+ // Include the MySQL query used for the diff in the report that is stored on
+ // each shard primary tablet in the _vt.vdiff_table records.
bool debug_query = 11;
+ // Only show the Primary Key columns in any row diff output. You would
+ // typically want to use this if you set the max_sample_rows very high.
bool only_p_ks = 12;
+ // Update the table statistics, using ANALYZE TABLE, on each table involved
+ // in the vdiff during initialization on each target shard. This will ensure
+ // that progress estimates are as accurate as possible -- but it does involve
+ // locks and can potentially impact query processing on the target keyspace.
bool update_table_stats = 13;
+ // If there are collation differences between the source and target, you can
+ // have rows that are identical but simply returned in a different order from
+ // MySQL. We will do a second pass to compare the rows for any actual
+ // differences in this case and this flag allows you to control the resources
+ // used for this operation.
+ // The default is 0, comparing no extra rows.
int64 max_extra_rows_to_compare = 14;
+ // Wait for the vdiff to complete before returning (making the call synchronous
+ // vs asynchronous by default).
bool wait = 15;
+ // When wait is true, this is how frequently the vdiff progress will be shown.
vttime.Duration wait_update_interval = 16;
+ // Automatically retry the vdiff if we encounter an error. This should almost
+ // always be set to true (default is false).
bool auto_retry = 17;
+ // Include additional information in the vdiff report that is produced and
+ // stored on each target shard primary's _vt sidecar database.
bool verbose = 18;
+ // The maximum number of rows to include in the row diff report (when
+ // differences are found) for each table on each shard.
+ // The default is 0, which will include no sample rows that differed.
int64 max_report_sample_rows = 19;
+ // The maximum time that a diff of a single table can run on each target shard
+ // before it is stopped and then later resumed from where we left off. This
+ // can be helpful in limiting the impact of holding open that large transaction
+ // where we scan up to every row in the table.
+ // The default is 0 or no limit.
vttime.Duration max_diff_duration = 20;
+ // At what length should we truncate the column values in the row diff report
+ // generated for each table on each shard when differences are detected.
+ // The default is 0, meaning do not truncate.
int64 row_diff_column_truncate_at = 21;
+ // Auto start the vdiff after creating it.
+ // The default is true if no value is specified.
optional bool auto_start = 22;
}
@@ -1948,6 +2050,10 @@ message WorkflowDeleteRequest {
bool keep_data = 3;
bool keep_routing_rules = 4;
repeated string shards = 5;
+ // The max records to delete from the moved tables when cleaning
+ // up the migrated data. This is only used with multi-tenant
+ // MoveTables migrations.
+ int64 delete_batch_size = 6;
}
message WorkflowDeleteResponse {
diff --git a/proto/vtctlservice.proto b/proto/vtctlservice.proto
index 23f00c2a376..3133d4fb4e1 100644
--- a/proto/vtctlservice.proto
+++ b/proto/vtctlservice.proto
@@ -58,6 +58,8 @@ service Vtctld {
rpc BackupShard(vtctldata.BackupShardRequest) returns (stream vtctldata.BackupResponse) {};
// CancelSchemaMigration cancels one or all migrations, terminating any running ones as needed.
rpc CancelSchemaMigration(vtctldata.CancelSchemaMigrationRequest) returns (vtctldata.CancelSchemaMigrationResponse) {};
+ // ChangeTabletTags changes the tags of the specified tablet, if possible.
+ rpc ChangeTabletTags(vtctldata.ChangeTabletTagsRequest) returns (vtctldata.ChangeTabletTagsResponse) {};
// ChangeTabletType changes the db type for the specified tablet, if possible.
// This is used primarily to arrange replicas, and it will not convert a
// primary. For that, use InitShardPrimary.
@@ -171,6 +173,8 @@ service Vtctld {
rpc GetThrottlerStatus(vtctldata.GetThrottlerStatusRequest) returns (vtctldata.GetThrottlerStatusResponse) {};
// GetTopologyPath returns the topology cell at a given path.
rpc GetTopologyPath(vtctldata.GetTopologyPathRequest) returns (vtctldata.GetTopologyPathResponse) {};
+ // GetTransactionInfo reads a given transactions information.
+ rpc GetTransactionInfo(vtctldata.GetTransactionInfoRequest) returns (vtctldata.GetTransactionInfoResponse) {};
// GetTransactions returns the unresolved transactions for the request.
rpc GetUnresolvedTransactions(vtctldata.GetUnresolvedTransactionsRequest) returns (vtctldata.GetUnresolvedTransactionsResponse) {};
// GetVersion returns the version of a tablet from its debug vars.
diff --git a/proto/vtgate.proto b/proto/vtgate.proto
index 5b080178218..aadf211f0a2 100644
--- a/proto/vtgate.proto
+++ b/proto/vtgate.proto
@@ -76,6 +76,8 @@ message Session {
// reserved connection if a dedicated connection is needed
int64 reserved_id = 4;
bool vindex_only = 5;
+ // rows_affected tracks if any query has modified the rows.
+ bool rows_affected = 6;
}
// shard_sessions keep track of per-shard transaction info.
repeated ShardSession shard_sessions = 2;
diff --git a/test.go b/test.go
index 52b1d06115d..95d62af892f 100755
--- a/test.go
+++ b/test.go
@@ -77,7 +77,7 @@ For example:
// Flags
var (
flavor = flag.String("flavor", "mysql80", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors)
- bootstrapVersion = flag.String("bootstrap-version", "36", "the version identifier to use for the docker images")
+ bootstrapVersion = flag.String("bootstrap-version", "39", "the version identifier to use for the docker images")
runCount = flag.Int("runs", 1, "run each test this many times")
retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests")
logPass = flag.Bool("log-pass", false, "log test output even if it passes")
diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go
index af8b0a84d47..b24db1154fb 100644
--- a/test/ci_workflow_gen.go
+++ b/test/ci_workflow_gen.go
@@ -103,6 +103,7 @@ var (
"vtgate_vindex_heavy",
"vtgate_vschema",
"vtgate_queries",
+ "vtgate_plantests",
"vtgate_schema_tracker",
"vtgate_foreignkey_stress",
"vtorc",
@@ -116,7 +117,8 @@ var (
"vreplication_v2",
"vreplication_partial_movetables_and_materialize",
"vreplication_foreign_key_stress",
- "vreplication_migrate_vdiff2_convert_tz",
+ "vreplication_migrate",
+ "vreplication_vtctldclient_vdiff2_movetables_tz",
"vreplication_multi_tenant",
"schemadiff_vrepl",
"topo_connection_cache",
@@ -153,7 +155,11 @@ var (
"onlineddl_vrepl_stress_suite",
"onlineddl_vrepl_suite",
"vreplication_basic",
- "vreplication_migrate_vdiff2_convert_tz",
+ "vreplication_migrate",
+ "vreplication_vtctldclient_vdiff2_movetables_tz",
+ }
+ clusterRequiringMinio = []string{
+ "21",
}
)
@@ -172,6 +178,7 @@ type clusterTest struct {
EnableBinlogTransactionCompression bool
PartialKeyspace bool
Cores16 bool
+ NeedsMinio bool
}
type vitessTesterTest struct {
@@ -284,6 +291,13 @@ func generateClusterWorkflows(list []string, tpl string) {
break
}
}
+ minioClusters := canonnizeList(clusterRequiringMinio)
+ for _, minioCluster := range minioClusters {
+ if minioCluster == cluster {
+ test.NeedsMinio = true
+ break
+ }
+ }
if mysqlVersion == mysql57 {
test.Platform = string(mysql57)
}
diff --git a/test/config.json b/test/config.json
index cfc89de84ce..da0026f0125 100644
--- a/test/config.json
+++ b/test/config.json
@@ -136,6 +136,15 @@
"RetryMax": 1,
"Tags": []
},
+ "backup_s3": {
+ "File": "unused.go",
+ "Args": ["vitess.io/vitess/go/test/endtoend/backup/s3", "-timeout", "30m"],
+ "Command": [],
+ "Manual": false,
+ "Shard": "21",
+ "RetryMax": 1,
+ "Tags": []
+ },
"backup_only": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/backup/vtbackup", "-timeout", "20m"],
@@ -340,17 +349,6 @@
"RetryMax": 1,
"Tags": []
},
- "pitr": {
- "File": "unused.go",
- "Args": ["vitess.io/vitess/go/test/endtoend/recovery/pitr"],
- "Command": [],
- "Manual": false,
- "Shard": "10",
- "RetryMax": 1,
- "Tags": [
- "site_test"
- ]
- },
"recovery": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/recovery/unshardedrecovery"],
@@ -851,6 +849,15 @@
"RetryMax": 1,
"Tags": []
},
+ "vtgate_transaction_twopc_metric": {
+ "File": "unused.go",
+ "Args": ["vitess.io/vitess/go/test/endtoend/transaction/twopc/metric"],
+ "Command": [],
+ "Manual": false,
+ "Shard": "vtgate_transaction",
+ "RetryMax": 1,
+ "Tags": []
+ },
"vtgate_transaction_twopc_stress": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/transaction/twopc/stress"],
@@ -878,6 +885,15 @@
"RetryMax": 1,
"Tags": []
},
+ "vtgate_plantests": {
+ "File": "unused.go",
+ "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/plan_tests"],
+ "Command": [],
+ "Manual": false,
+ "Shard": "vtgate_plantests",
+ "RetryMax": 1,
+ "Tags": []
+ },
"vtgate_unsharded": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/vtgate/unsharded"],
@@ -1229,6 +1245,17 @@
"RetryMax": 1,
"Tags": []
},
+ "vtop_example": {
+ "File": "",
+ "Args": [],
+ "Command": [
+ "test/vtop_example.sh"
+ ],
+ "Manual": false,
+ "Shard": "",
+ "RetryMax": 1,
+ "Tags": []
+ },
"vtorc_primary_failure": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/vtorc/primaryfailure"],
@@ -1319,21 +1346,12 @@
"RetryMax": 1,
"Tags": []
},
- "vreplication_vtctldclient_cli": {
- "File": "unused.go",
- "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVtctldclientCLI", "-timeout", "20m"],
- "Command": [],
- "Manual": false,
- "Shard": "vreplication_migrate_vdiff2_convert_tz",
- "RetryMax": 1,
- "Tags": []
- },
"vreplication_vtctl_migrate": {
"File": "unused.go",
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVtctlMigrate", "-timeout", "30m"],
"Command": [],
"Manual": false,
- "Shard": "vreplication_migrate_vdiff2_convert_tz",
+ "Shard": "vreplication_migrate",
"RetryMax": 1,
"Tags": []
},
@@ -1342,7 +1360,7 @@
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVtctldMigrate", "-timeout", "30m"],
"Command": [],
"Manual": false,
- "Shard": "vreplication_migrate_vdiff2_convert_tz",
+ "Shard": "vreplication_migrate",
"RetryMax": 1,
"Tags": []
},
@@ -1351,7 +1369,16 @@
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVDiff2", "-timeout", "30m"],
"Command": [],
"Manual": false,
- "Shard": "vreplication_migrate_vdiff2_convert_tz",
+ "Shard": "vreplication_vtctldclient_vdiff2_movetables_tz",
+ "RetryMax": 1,
+ "Tags": []
+ },
+ "vreplication_vtctldclient_cli": {
+ "File": "unused.go",
+ "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestVtctldclientCLI", "-timeout", "20m"],
+ "Command": [],
+ "Manual": false,
+ "Shard": "vreplication_vtctldclient_vdiff2_movetables_tz",
"RetryMax": 1,
"Tags": []
},
@@ -1360,7 +1387,25 @@
"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestMoveTablesTZ"],
"Command": [],
"Manual": false,
- "Shard": "vreplication_migrate_vdiff2_convert_tz",
+ "Shard": "vreplication_vtctldclient_vdiff2_movetables_tz",
+ "RetryMax": 1,
+ "Tags": []
+ },
+ "loopkup_index": {
+ "File": "unused.go",
+ "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestLookupIndex"],
+ "Command": [],
+ "Manual": false,
+ "Shard": "vreplication_vtctldclient_vdiff2_movetables_tz",
+ "RetryMax": 1,
+ "Tags": []
+ },
+ "vtadmin": {
+ "File": "unused.go",
+ "Args": ["vitess.io/vitess/go/test/endtoend/vtadmin"],
+ "Command": [],
+ "Manual": false,
+ "Shard": "15",
"RetryMax": 1,
"Tags": []
},
diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl
index 1b035ff0807..8d0a2f650b5 100644
--- a/test/templates/cluster_endtoend_test.tpl
+++ b/test/templates/cluster_endtoend_test.tpl
@@ -14,7 +14,7 @@ env:
jobs:
build:
name: Run endtoend tests on {{.Name}}
- runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1{{else}}ubuntu-latest{{end}}
+ runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}}
steps:
- name: Skip CI
@@ -117,19 +117,28 @@ jobs:
sudo apt-get -qq update
# Install everything else we need, and configure
- sudo apt-get -qq install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils libncurses5
+ sudo apt-get -qq install -y percona-server-server percona-server-client make unzip g++ etcd-client etcd-server git wget eatmydata xz-utils libncurses6
{{else}}
# Get key to latest MySQL repo
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
# Setup MySQL 8.0
- wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb
+ wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
sudo apt-get -qq update
+
+ # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also:
+ # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
+ curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb
+ sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb
+ # libtinfo5 is also needed for older MySQL 5.7 builds.
+ curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
+ sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
+
# Install everything else we need, and configure
- sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5
+ sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6
{{end}}
@@ -148,6 +157,15 @@ jobs:
{{end}}
+ {{if .NeedsMinio }}
+ - name: Install Minio
+ if: steps.skip-workflow.outputs.skip-workflow == 'false'
+ run: |
+ wget https://dl.min.io/server/minio/release/linux-amd64/minio
+ chmod +x minio
+ mv minio /usr/local/bin
+ {{end}}
+
{{if .MakeTools}}
- name: Installing zookeeper and consul
diff --git a/test/templates/cluster_endtoend_test_docker.tpl b/test/templates/cluster_endtoend_test_docker.tpl
index f68e4223c75..f7e8aa2c1d8 100644
--- a/test/templates/cluster_endtoend_test_docker.tpl
+++ b/test/templates/cluster_endtoend_test_docker.tpl
@@ -6,7 +6,7 @@ permissions: read-all
jobs:
build:
name: Run endtoend tests on {{.Name}}
- runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1{{else}}ubuntu-latest{{end}}
+ runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}}
steps:
- name: Skip CI
diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl
index 6e2b98cb7e4..f4152c939b0 100644
--- a/test/templates/cluster_endtoend_test_mysql57.tpl
+++ b/test/templates/cluster_endtoend_test_mysql57.tpl
@@ -19,7 +19,7 @@ env:
jobs:
build:
name: Run endtoend tests on {{.Name}}
- runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1{{else}}ubuntu-latest{{end}}
+ runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}}
steps:
- name: Skip CI
@@ -126,16 +126,20 @@ jobs:
# Get key to latest MySQL repo
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
- wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb
+ wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb
# Bionic packages are still compatible for Jammy since there's no MySQL 5.7
# packages for Jammy.
echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections
echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
sudo apt-get update
- sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses5
+ # We have to install this old version of libaio1. See also:
+ # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
+ curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb
+ sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb
+ sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6
- sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata
+ sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata
sudo service mysql stop
sudo service etcd stop
diff --git a/test/templates/cluster_vitess_tester.tpl b/test/templates/cluster_vitess_tester.tpl
index db492dad21b..b8d77754ba6 100644
--- a/test/templates/cluster_vitess_tester.tpl
+++ b/test/templates/cluster_vitess_tester.tpl
@@ -14,7 +14,7 @@ env:
jobs:
build:
name: Run endtoend tests on {{.Name}}
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
steps:
- name: Skip CI
@@ -93,12 +93,12 @@ jobs:
# Get key to latest MySQL repo
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
# Setup MySQL 8.0
- wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb
+ wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
sudo apt-get -qq update
# Install everything else we need, and configure
- sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5
+ sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6
sudo service mysql stop
sudo service etcd stop
@@ -110,7 +110,7 @@ jobs:
go install github.com/vitessio/go-junit-report@HEAD
# install vitess tester
- go install github.com/vitessio/vitess-tester/go/vt@374fd9f495c1afd3b6bea9d4ec7728119714055
+ go install github.com/vitessio/vt/go/vt@e43009309f599378504905d4b804460f47822ac5
- name: Setup launchable dependencies
if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
@@ -140,7 +140,7 @@ jobs:
i=1
for dir in {{.Path}}/*/; do
# We go over all the directories in the given path.
- # If there is a vschema file there, we use it, otherwise we let vitess-tester autogenerate it.
+ # If there is a vschema file there, we use it, otherwise we let vt tester autogenerate it.
if [ -f $dir/vschema.json ]; then
vt tester --xunit --vschema "$dir"vschema.json $dir/*.test
else
diff --git a/test/templates/dockerfile.tpl b/test/templates/dockerfile.tpl
index 3a6d664753c..af4376d3ca9 100644
--- a/test/templates/dockerfile.tpl
+++ b/test/templates/dockerfile.tpl
@@ -1,4 +1,4 @@
-ARG bootstrap_version=36
+ARG bootstrap_version=39
ARG image="vitess/bootstrap:${bootstrap_version}-{{.Platform}}"
FROM "${image}"
diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl
index 3f5556ee314..3704aebac4e 100644
--- a/test/templates/unit_test.tpl
+++ b/test/templates/unit_test.tpl
@@ -14,7 +14,7 @@ env:
jobs:
test:
name: {{.Name}}
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
steps:
- name: Skip CI
@@ -87,20 +87,20 @@ jobs:
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
run: |
export DEBIAN_FRONTEND="noninteractive"
- sudo apt-get -qq update
+ sudo apt-get update
# Uninstall any previously installed MySQL first
sudo systemctl stop apparmor
- sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common
- sudo apt-get -qq -y autoremove
- sudo apt-get -qq -y autoclean
+ sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
+ sudo apt-get -y autoremove
+ sudo apt-get -y autoclean
sudo deluser mysql
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
# Get key to latest MySQL repo
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
- wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb
+ wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb
{{if (eq .Platform "mysql57")}}
# Bionic packages are still compatible for Jammy since there's no MySQL 5.7
@@ -108,25 +108,32 @@ jobs:
echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections
echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
- sudo apt-get -qq update
- sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses5
+ sudo apt-get update
+ # We have to install this old version of libaio1. See also:
+ # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
+ curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb
+ sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb
+ # libtinfo5 is also needed for older MySQL 5.7 builds.
+ curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
+ sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
+ sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6
{{end}}
{{if (eq .Platform "mysql80")}}
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
- sudo apt-get -qq update
- sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client
+ sudo apt-get update
+ sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
{{end}}
{{if (eq .Platform "mysql84")}}
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
- sudo apt-get -qq update
- sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client
+ sudo apt-get update
+ sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
{{end}}
- sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata
+ sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata
sudo service mysql stop
sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
diff --git a/test/vtop_example.sh b/test/vtop_example.sh
index 5ff90a2be7e..c537c0f844c 100755
--- a/test/vtop_example.sh
+++ b/test/vtop_example.sh
@@ -482,11 +482,12 @@ EOF
waitForKeyspaceToBeServing customer 80- 1
}
+kind delete cluster --name kind || true
# Build the docker image for vitess/lite using the local code
docker build -f docker/lite/Dockerfile -t vitess/lite:pr .
# Build the docker image for vitess/vtadmin using the local code
-docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr .
+docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr ./docker/binaries/vtadmin
# Print the docker images available
docker image ls
diff --git a/tools/get_kubectl_kind.sh b/tools/get_kubectl_kind.sh
index 57df414fdd8..169b120aaa0 100755
--- a/tools/get_kubectl_kind.sh
+++ b/tools/get_kubectl_kind.sh
@@ -12,7 +12,7 @@ source build.env
mkdir -p "$VTROOT/bin"
cd "$VTROOT/bin"
-KUBE_VERSION="${KUBE_VERSION:-v1.21.1}"
+KUBE_VERSION="${KUBE_VERSION:-v1.31.0}"
KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}"
# Download kubectl if needed.
@@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl
if ! command -v kind &> /dev/null
then
echo "Downloading kind..."
- curl -L https://kind.sigs.k8s.io/dl/v0.12.0/kind-linux-amd64 > "kind"
+ curl -L https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64 > "kind"
chmod +x "kind"
echo "Installed kind"
else
diff --git a/tools/map-shard-for-value/Makefile b/tools/map-shard-for-value/Makefile
new file mode 100644
index 00000000000..61bc88ac0ed
--- /dev/null
+++ b/tools/map-shard-for-value/Makefile
@@ -0,0 +1,22 @@
+# Copyright 2024 The Vitess Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build:
+ go build map-shard-for-value.go
+
+test:
+ echo "1\n-1\n99" | go run map-shard-for-value.go --total_shards=4 --vindex=xxhash
+
+clean:
+ rm -f map-shard-for-value
diff --git a/tools/map-shard-for-value/map-shard-for-value.go b/tools/map-shard-for-value/map-shard-for-value.go
new file mode 100755
index 00000000000..18a092d1371
--- /dev/null
+++ b/tools/map-shard-for-value/map-shard-for-value.go
@@ -0,0 +1,207 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "bufio"
+ "context"
+ "encoding/hex"
+ "fmt"
+ "log"
+ "os"
+ "strconv"
+ "strings"
+
+ flag "github.com/spf13/pflag"
+
+ "vitess.io/vitess/go/vt/topo"
+
+ "vitess.io/vitess/go/sqltypes"
+ "vitess.io/vitess/go/vt/key"
+ "vitess.io/vitess/go/vt/proto/topodata"
+ "vitess.io/vitess/go/vt/vtgate/vindexes"
+)
+
+/*
+ * This tool reads a list of values from stdin and prints the
+ * corresponding keyspace ID and shard for each value. It uses the given vindex
+ * and shard ranges to determine the shard. The vindex is expected to be a
+ * single-column vindex. The shard ranges are specified as a comma-separated
+ * list of key ranges, example "-80,80-".
+ * If you have uniformly distributed shards, you can specify the total number
+ * of shards using the -total_shards flag, and the tool will generate the shard ranges
+ * using the same logic as the Vitess operator does (using the key.GenerateShardRanges() function).
+ *
+ * Example usage:
+ * echo "1\n2\n3" | go run shard-from-id.go -vindex=hash -shards=-80,80-
+ *
+ * Currently tested only for integer values and hash/xxhash vindexes.
+ */
+
+func mapShard(allShards []*topodata.ShardReference, ksid key.DestinationKeyspaceID) (string, error) {
+ foundShard := ""
+ addShard := func(shard string) error {
+ foundShard = shard
+ return nil
+ }
+ if err := ksid.Resolve(allShards, addShard); err != nil {
+ return "", fmt.Errorf("failed to resolve keyspace ID: %v:: %s", ksid.String(), err)
+ }
+
+ if foundShard == "" {
+ return "", fmt.Errorf("no shard found for keyspace ID: %v", ksid)
+ }
+ return foundShard, nil
+}
+
+func selectShard(vindex vindexes.Vindex, value sqltypes.Value, allShards []*topodata.ShardReference) (string, key.DestinationKeyspaceID, error) {
+ ctx := context.Background()
+
+ destinations, err := vindexes.Map(ctx, vindex, nil, [][]sqltypes.Value{{value}})
+ if err != nil {
+ return "", nil, fmt.Errorf("failed to map value to keyspace ID: %w", err)
+ }
+
+ if len(destinations) != 1 {
+ return "", nil, fmt.Errorf("unexpected number of destinations: %d", len(destinations))
+ }
+
+ ksid, ok := destinations[0].(key.DestinationKeyspaceID)
+ if !ok {
+ return "", nil, fmt.Errorf("unexpected destination type: %T", destinations[0])
+ }
+
+ foundShard, err := mapShard(allShards, ksid)
+ if err != nil {
+ return "", nil, fmt.Errorf("failed to map shard, original value %v, keyspace id %s: %w", value, ksid, err)
+ }
+ return foundShard, ksid, nil
+}
+
+func getValue(valueStr, valueType string) (sqltypes.Value, error) {
+ var value sqltypes.Value
+
+ switch valueType {
+ case "int":
+ valueInt, err := strconv.ParseInt(valueStr, 10, 64)
+ if err != nil {
+ return value, fmt.Errorf("failed to parse int value: %w", err)
+ }
+ value = sqltypes.NewInt64(valueInt)
+ case "uint":
+ valueUint, err := strconv.ParseUint(valueStr, 10, 64)
+ if err != nil {
+ return value, fmt.Errorf("failed to parse uint value: %w", err)
+ }
+ value = sqltypes.NewUint64(valueUint)
+ case "string":
+ value = sqltypes.NewVarChar(valueStr)
+ default:
+ return value, fmt.Errorf("unsupported value type: %s", valueType)
+ }
+
+ return value, nil
+}
+
+func getShardMap(shardsCSV *string) []*topodata.ShardReference {
+ var allShards []*topodata.ShardReference
+
+ for _, shard := range strings.Split(*shardsCSV, ",") {
+ _, keyRange, err := topo.ValidateShardName(shard)
+ if err != nil {
+ log.Fatalf("invalid shard range: %s", shard)
+ }
+ allShards = append(allShards, &topodata.ShardReference{
+ Name: shard,
+ KeyRange: keyRange,
+ })
+ }
+ return allShards
+}
+
+type output struct {
+ Value string
+ KeyspaceID string
+ Shard string
+}
+
+func processValues(scanner *bufio.Scanner, shardsCSV *string, vindexName string, valueType string) ([]output, error) {
+ allShards := getShardMap(shardsCSV)
+
+ vindex, err := vindexes.CreateVindex(vindexName, vindexName, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create vindex: %v", err)
+ }
+ var outputs []output
+ for scanner.Scan() {
+ valueStr := scanner.Text()
+ if valueStr == "" {
+ continue
+ }
+ value, err := getValue(valueStr, valueType)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get value for: %v, value_type %s:: %v", valueStr, valueType, err)
+ }
+ shard, ksid, err := selectShard(vindex, value, allShards)
+ if err != nil {
+ // ignore errors so that we can go ahead with the computation for other values
+ continue
+ }
+ outputs = append(outputs, output{Value: valueStr, KeyspaceID: hex.EncodeToString(ksid), Shard: shard})
+ }
+ return outputs, nil
+}
+
+func printOutput(outputs []output) {
+ fmt.Println("value,keyspaceID,shard")
+ for _, output := range outputs {
+ fmt.Printf("%s,%s,%s\n", output.Value, output.KeyspaceID, output.Shard)
+ }
+}
+
+func main() {
+ // Explicitly configuring the logger since it was flaky in displaying logs locally without this.
+ log.SetOutput(os.Stderr)
+ log.SetFlags(log.LstdFlags)
+ log.SetPrefix("LOG: ")
+
+ vindexName := flag.String("vindex", "xxhash", "name of the vindex")
+ shardsCSV := flag.String("shards", "", "comma-separated list of shard ranges")
+ totalShards := flag.Int("total_shards", 0, "total number of uniformly distributed shards")
+ valueType := flag.String("value_type", "int", "type of the value (int, uint, or string)")
+ flag.Parse()
+
+ if *totalShards > 0 {
+ if *shardsCSV != "" {
+ log.Fatalf("cannot specify both total_shards and shards")
+ }
+ shardArr, err := key.GenerateShardRanges(*totalShards)
+ if err != nil {
+ log.Fatalf("failed to generate shard ranges: %v", err)
+ }
+ *shardsCSV = strings.Join(shardArr, ",")
+ }
+ if *shardsCSV == "" {
+ log.Fatal("shards or total_shards must be specified")
+ }
+ scanner := bufio.NewScanner(os.Stdin)
+ outputs, err := processValues(scanner, shardsCSV, *vindexName, *valueType)
+ if err != nil {
+ log.Fatalf("failed to process values: %v", err)
+ }
+ printOutput(outputs)
+}
diff --git a/tools/map-shard-for-value/map-shard-for-value.md b/tools/map-shard-for-value/map-shard-for-value.md
new file mode 100644
index 00000000000..17daf7f5fe5
--- /dev/null
+++ b/tools/map-shard-for-value/map-shard-for-value.md
@@ -0,0 +1,47 @@
+## Map Shard for Value Tool
+
+### Overview
+
+The `map-shard-for-value` tool maps a given value to a specific shard. This tool helps in determining
+which shard a particular value belongs to, based on the vindex algorithm and shard ranges.
+
+### Features
+-
+
+- Allows specifying the vindex type (e.g., `hash`, `xxhash`).
+- Allows specifying the shard list of (for uniformly distributed shard ranges) the total number of shards to generate.
+- Designed as a _filter_: Reads input values from `stdin` and outputs the corresponding shard information, so it can be
+ used to map values from a file or another program.
+
+### Usage
+
+```sh
+make build
+```
+
+```sh
+echo "1\n-1\n99" | ./map-shard-for-value --total_shards=4 --vindex=xxhash
+value,keyspaceID,shard
+1,d46405367612b4b7,c0-
+-1,d8e2a6a7c8c7623d,c0-
+99,200533312244abca,-40
+
+echo "1\n-1\n99" | ./map-shard-for-value --vindex=hash --shards="-80,80-"
+value,keyspaceID,shard
+1,166b40b44aba4bd6,-80
+-1,355550b2150e2451,-80
+99,2c40ad56f4593c47,-80
+```
+
+#### Flags
+
+- `--vindex`: Specifies the name of the vindex to use (e.g., `hash`, `xxhash`) (default `xxhash`)
+
+One (and only one) of these is required:
+
+- `--shards`: Comma-separated list of shard ranges
+- `--total_shards`: Total number of shards, only if shards are uniformly distributed
+
+Optional:
+- `--value_type`: Type of the value to map, one of int, uint, string (default `int`)
+
diff --git a/tools/map-shard-for-value/map-shard-for-value_test.go b/tools/map-shard-for-value/map-shard-for-value_test.go
new file mode 100644
index 00000000000..ca014818bb9
--- /dev/null
+++ b/tools/map-shard-for-value/map-shard-for-value_test.go
@@ -0,0 +1,90 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "bufio"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestProcess(t *testing.T) {
+ type testCase struct {
+ name string
+ shardsCSV string
+ vindexType string
+ values []int
+ valueType string
+ expected []output
+ }
+ testCases := []testCase{
+ {
+ name: "hash,2 shards",
+ shardsCSV: "-80,80-",
+ vindexType: "hash",
+ values: []int{1, 99},
+ valueType: "int",
+ expected: []output{
+ {
+ Value: "1",
+ KeyspaceID: "166b40b44aba4bd6",
+ Shard: "-80",
+ },
+ {
+ Value: "99",
+ KeyspaceID: "2c40ad56f4593c47",
+ Shard: "-80",
+ },
+ },
+ },
+ {
+ name: "xxhash,4 shards",
+ shardsCSV: "-40,40-80,80-c0,c0-",
+ vindexType: "xxhash",
+ values: []int{1, 99},
+ valueType: "int",
+ expected: []output{
+ {
+ Value: "1",
+ KeyspaceID: "d46405367612b4b7",
+ Shard: "c0-",
+ },
+ {
+ Value: "99",
+ KeyspaceID: "200533312244abca",
+ Shard: "-40",
+ },
+ },
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ var input strings.Builder
+ for _, num := range tc.values {
+ fmt.Fprintf(&input, "%d\n", num)
+ }
+ reader := strings.NewReader(input.String())
+ scanner := bufio.NewScanner(reader)
+ got, err := processValues(scanner, &tc.shardsCSV, tc.vindexType, tc.valueType)
+ require.NoError(t, err)
+ require.EqualValues(t, tc.expected, got)
+ })
+ }
+}
diff --git a/web/vtadmin/.prettiercc b/web/vtadmin/.prettiercc
index bed1b4f32cd..6fdc0af39dd 100644
--- a/web/vtadmin/.prettiercc
+++ b/web/vtadmin/.prettiercc
@@ -7,7 +7,6 @@
"bracketSpacing": true,
"endOfLine": "auto",
"jsxSingleQuote": false,
- "jsxBracketSameLine": false,
"quoteProps": "as-needed",
"htmlWhitespaceSensitivity": "css",
"printWidth": 120
diff --git a/web/vtadmin/package-lock.json b/web/vtadmin/package-lock.json
index 2dc201fbf88..8ad7c67a5b4 100644
--- a/web/vtadmin/package-lock.json
+++ b/web/vtadmin/package-lock.json
@@ -52,7 +52,7 @@
"eslint-config-react-app": "^7.0.1",
"i": "^0.3.7",
"jsdom": "^21.1.1",
- "msw": "^0.36.8",
+ "msw": "^2.5.2",
"npm": "^10.8.0",
"postcss": "^8.4.31",
"prettier": "^2.2.1",
@@ -2278,6 +2278,37 @@
"resolved": "https://registry.npmjs.org/@bugsnag/safe-json-stringify/-/safe-json-stringify-6.0.0.tgz",
"integrity": "sha512-htzFO1Zc57S8kgdRK9mLcPVTW1BY2ijfH7Dk2CeZmspTWKdKqSo1iwmqrq2WtRjFlo8aRZYgLX0wFrDXF/9DLA=="
},
+ "node_modules/@bundled-es-modules/cookie": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.1.tgz",
+ "integrity": "sha512-8o+5fRPLNbjbdGRRmJj3h6Hh1AQJf2dk3qQ/5ZFb+PXkRNiSoMGGUKlsgLfrxneb72axVJyIYji64E2+nNfYyw==",
+ "dev": true,
+ "license": "ISC",
+ "dependencies": {
+ "cookie": "^0.7.2"
+ }
+ },
+ "node_modules/@bundled-es-modules/statuses": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz",
+ "integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==",
+ "dev": true,
+ "license": "ISC",
+ "dependencies": {
+ "statuses": "^2.0.1"
+ }
+ },
+ "node_modules/@bundled-es-modules/tough-cookie": {
+ "version": "0.1.6",
+ "resolved": "https://registry.npmjs.org/@bundled-es-modules/tough-cookie/-/tough-cookie-0.1.6.tgz",
+ "integrity": "sha512-dvMHbL464C0zI+Yqxbz6kZ5TOEp7GLW+pry/RWndAR8MJQAXZ2rPmIs8tziTZjeIyhSNZgZbCePtfSbdWqStJw==",
+ "dev": true,
+ "license": "ISC",
+ "dependencies": {
+ "@types/tough-cookie": "^4.0.5",
+ "tough-cookie": "^4.1.4"
+ }
+ },
"node_modules/@csstools/cascade-layer-name-parser": {
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/@csstools/cascade-layer-name-parser/-/cascade-layer-name-parser-1.0.9.tgz",
@@ -3475,6 +3506,74 @@
"integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==",
"dev": true
},
+ "node_modules/@inquirer/core": {
+ "version": "10.0.1",
+ "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.0.1.tgz",
+ "integrity": "sha512-KKTgjViBQUi3AAssqjUFMnMO3CM3qwCHvePV9EW+zTKGKafFGFF01sc1yOIYjLJ7QU52G/FbzKc+c01WLzXmVQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@inquirer/figures": "^1.0.7",
+ "@inquirer/type": "^3.0.0",
+ "ansi-escapes": "^4.3.2",
+ "cli-width": "^4.1.0",
+ "mute-stream": "^2.0.0",
+ "signal-exit": "^4.1.0",
+ "strip-ansi": "^6.0.1",
+ "wrap-ansi": "^6.2.0",
+ "yoctocolors-cjs": "^2.1.2"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@inquirer/core/node_modules/@inquirer/type": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.0.tgz",
+ "integrity": "sha512-YYykfbw/lefC7yKj7nanzQXILM7r3suIvyFlCcMskc99axmsSewXWkAfXKwMbgxL76iAFVmRwmYdwNZNc8gjog==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@types/node": ">=18"
+ }
+ },
+ "node_modules/@inquirer/core/node_modules/@types/node": {
+ "version": "22.9.0",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.0.tgz",
+ "integrity": "sha512-vuyHg81vvWA1Z1ELfvLko2c8f34gyA0zaic0+Rllc5lbCnbSyuvb2Oxpm6TAUAC/2xZN3QGqxBNggD1nNR2AfQ==",
+ "dev": true,
+ "license": "MIT",
+ "peer": true,
+ "dependencies": {
+ "undici-types": "~6.19.8"
+ }
+ },
+ "node_modules/@inquirer/core/node_modules/signal-exit": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
+ "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+ "dev": true,
+ "license": "ISC",
+ "engines": {
+ "node": ">=14"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/isaacs"
+ }
+ },
+ "node_modules/@inquirer/figures": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.7.tgz",
+ "integrity": "sha512-m+Trk77mp54Zma6xLkLuY+mvanPxlE4A7yNKs2HBiyZ4UkVs28Mv5c/pgWrHeInx+USHeX/WEPzjrWrcJiQgjw==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ }
+ },
"node_modules/@isaacs/cliui": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -3663,28 +3762,22 @@
"node": ">=v12.0.0"
}
},
- "node_modules/@mswjs/cookies": {
- "version": "0.1.7",
- "resolved": "https://registry.npmjs.org/@mswjs/cookies/-/cookies-0.1.7.tgz",
- "integrity": "sha512-bDg1ReMBx+PYDB4Pk7y1Q07Zz1iKIEUWQpkEXiA2lEWg9gvOZ8UBmGXilCEUvyYoRFlmr/9iXTRR69TrgSwX/Q==",
- "dev": true,
- "dependencies": {
- "@types/set-cookie-parser": "^2.4.0",
- "set-cookie-parser": "^2.4.6"
- }
- },
"node_modules/@mswjs/interceptors": {
- "version": "0.12.7",
- "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.12.7.tgz",
- "integrity": "sha512-eGjZ3JRAt0Fzi5FgXiV/P3bJGj0NqsN7vBS0J0FO2AQRQ0jCKQS4lEFm4wvlSgKQNfeuc/Vz6d81VtU3Gkx/zg==",
+ "version": "0.36.9",
+ "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.36.9.tgz",
+ "integrity": "sha512-mMRDUBwSNeCgjSMEWfjoh4Rm9fbyZ7xQ9SBq8eGHiiyRn1ieTip3pNEt0wxWVPPxR4i1Rv9bTkeEbkX7M4c15A==",
"dev": true,
+ "license": "MIT",
"dependencies": {
- "@open-draft/until": "^1.0.3",
- "@xmldom/xmldom": "^0.7.2",
- "debug": "^4.3.2",
- "headers-utils": "^3.0.2",
- "outvariant": "^1.2.0",
- "strict-event-emitter": "^0.2.0"
+ "@open-draft/deferred-promise": "^2.2.0",
+ "@open-draft/logger": "^0.3.0",
+ "@open-draft/until": "^2.0.0",
+ "is-node-process": "^1.2.0",
+ "outvariant": "^1.4.3",
+ "strict-event-emitter": "^0.5.1"
+ },
+ "engines": {
+ "node": ">=18"
}
},
"node_modules/@nicolo-ribaudo/eslint-scope-5-internals": {
@@ -3753,11 +3846,30 @@
"node": ">= 8"
}
},
+ "node_modules/@open-draft/deferred-promise": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz",
+ "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/@open-draft/logger": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz",
+ "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "is-node-process": "^1.2.0",
+ "outvariant": "^1.4.0"
+ }
+ },
"node_modules/@open-draft/until": {
- "version": "1.0.3",
- "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-1.0.3.tgz",
- "integrity": "sha512-Aq58f5HiWdyDlFffbbSjAlv596h/cOnt2DO1w3DOC7OJ5EHs0hd/nycJfiu9RJbT6Yk6F1knnRRXNSpxoIVZ9Q==",
- "dev": true
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz",
+ "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==",
+ "dev": true,
+ "license": "MIT"
},
"node_modules/@pkgjs/parseargs": {
"version": "0.11.0",
@@ -4381,10 +4493,11 @@
}
},
"node_modules/@types/cookie": {
- "version": "0.4.1",
- "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz",
- "integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==",
- "dev": true
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz",
+ "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==",
+ "dev": true,
+ "license": "MIT"
},
"node_modules/@types/d3": {
"version": "7.4.3",
@@ -4635,16 +4748,6 @@
"resolved": "https://registry.npmjs.org/@types/history/-/history-4.7.11.tgz",
"integrity": "sha512-qjDJRrmvBMiTx+jyLxvLfJU7UznFuokDv4f3WRuriHKERccVpFU+8XMQUAbDzoiJCsmexxRExQeMwwCdamSKDA=="
},
- "node_modules/@types/inquirer": {
- "version": "8.2.10",
- "resolved": "https://registry.npmjs.org/@types/inquirer/-/inquirer-8.2.10.tgz",
- "integrity": "sha512-IdD5NmHyVjWM8SHWo/kPBgtzXatwPkfwzyP3fN1jF2g9BWt5WO+8hL2F4o2GKIYsU40PpqeevuUWvkS/roXJkA==",
- "dev": true,
- "dependencies": {
- "@types/through": "*",
- "rxjs": "^7.2.0"
- }
- },
"node_modules/@types/istanbul-lib-coverage": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz",
@@ -4704,12 +4807,6 @@
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
"integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="
},
- "node_modules/@types/js-levenshtein": {
- "version": "1.1.3",
- "resolved": "https://registry.npmjs.org/@types/js-levenshtein/-/js-levenshtein-1.1.3.tgz",
- "integrity": "sha512-jd+Q+sD20Qfu9e2aEXogiO3vpOC1PYJOUdyN9gvs4Qrvkg4wF43L5OhqrPeokdv8TL0/mXoYfpkcoGZMNN2pkQ==",
- "dev": true
- },
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -4844,20 +4941,18 @@
"integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==",
"dev": true
},
- "node_modules/@types/set-cookie-parser": {
- "version": "2.4.7",
- "resolved": "https://registry.npmjs.org/@types/set-cookie-parser/-/set-cookie-parser-2.4.7.tgz",
- "integrity": "sha512-+ge/loa0oTozxip6zmhRIk8Z/boU51wl9Q6QdLZcokIGMzY5lFXYy/x7Htj2HTC6/KZP1hUbZ1ekx8DYXICvWg==",
- "dev": true,
- "dependencies": {
- "@types/node": "*"
- }
- },
"node_modules/@types/stack-utils": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz",
"integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw=="
},
+ "node_modules/@types/statuses": {
+ "version": "2.0.5",
+ "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.5.tgz",
+ "integrity": "sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/@types/testing-library__jest-dom": {
"version": "5.14.9",
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.9.tgz",
@@ -4867,14 +4962,12 @@
"@types/jest": "*"
}
},
- "node_modules/@types/through": {
- "version": "0.0.33",
- "resolved": "https://registry.npmjs.org/@types/through/-/through-0.0.33.tgz",
- "integrity": "sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==",
+ "node_modules/@types/tough-cookie": {
+ "version": "4.0.5",
+ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+ "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
"dev": true,
- "dependencies": {
- "@types/node": "*"
- }
+ "license": "MIT"
},
"node_modules/@types/yargs": {
"version": "17.0.32",
@@ -5312,15 +5405,6 @@
"pretty-format": "^27.5.1"
}
},
- "node_modules/@xmldom/xmldom": {
- "version": "0.7.13",
- "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.7.13.tgz",
- "integrity": "sha512-lm2GW5PkosIzccsaZIz7tp8cPADSIlIHWDFTR1N0SzfinhhYgeIQjFMz4rYzanCScr3DqQLeomUDArp6MWKm+g==",
- "dev": true,
- "engines": {
- "node": ">=10.0.0"
- }
- },
"node_modules/@zeit/schemas": {
"version": "2.36.0",
"resolved": "https://registry.npmjs.org/@zeit/schemas/-/schemas-2.36.0.tgz",
@@ -5429,6 +5513,7 @@
"resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
"integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==",
"dev": true,
+ "license": "MIT",
"dependencies": {
"type-fest": "^0.21.3"
},
@@ -5444,6 +5529,7 @@
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
"dev": true,
+ "license": "(MIT OR CC0-1.0)",
"engines": {
"node": ">=10"
},
@@ -5898,26 +5984,6 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
- "node_modules/base64-js": {
- "version": "1.5.1",
- "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
- "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
- "dev": true,
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/feross"
- },
- {
- "type": "patreon",
- "url": "https://www.patreon.com/feross"
- },
- {
- "type": "consulting",
- "url": "https://feross.org/support"
- }
- ]
- },
"node_modules/big-integer": {
"version": "1.6.52",
"resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz",
@@ -5937,17 +6003,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
- "node_modules/bl": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
- "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
- "dev": true,
- "dependencies": {
- "buffer": "^5.5.0",
- "inherits": "^2.0.4",
- "readable-stream": "^3.4.0"
- }
- },
"node_modules/bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
@@ -6145,30 +6200,6 @@
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
}
},
- "node_modules/buffer": {
- "version": "5.7.1",
- "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
- "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
- "dev": true,
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/feross"
- },
- {
- "type": "patreon",
- "url": "https://www.patreon.com/feross"
- },
- {
- "type": "consulting",
- "url": "https://feross.org/support"
- }
- ],
- "dependencies": {
- "base64-js": "^1.3.1",
- "ieee754": "^1.1.13"
- }
- },
"node_modules/byline": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
@@ -6349,12 +6380,6 @@
"url": "https://github.com/chalk/chalk-template?sponsor=1"
}
},
- "node_modules/chardet": {
- "version": "0.7.0",
- "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
- "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
- "dev": true
- },
"node_modules/check-error": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz",
@@ -6437,30 +6462,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
- "node_modules/cli-cursor": {
- "version": "3.1.0",
- "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz",
- "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==",
- "dev": true,
- "dependencies": {
- "restore-cursor": "^3.1.0"
- },
- "engines": {
- "node": ">=8"
- }
- },
- "node_modules/cli-spinners": {
- "version": "2.9.2",
- "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz",
- "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==",
- "dev": true,
- "engines": {
- "node": ">=6"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/cli-truncate": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-3.1.0.tgz",
@@ -6562,12 +6563,13 @@
}
},
"node_modules/cli-width": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz",
- "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==",
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz",
+ "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==",
"dev": true,
+ "license": "ISC",
"engines": {
- "node": ">= 10"
+ "node": ">= 12"
}
},
"node_modules/client-only": {
@@ -6623,15 +6625,6 @@
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
}
},
- "node_modules/clone": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
- "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==",
- "dev": true,
- "engines": {
- "node": ">=0.8"
- }
- },
"node_modules/clsx": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz",
@@ -6771,10 +6764,11 @@
"dev": true
},
"node_modules/cookie": {
- "version": "0.4.2",
- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz",
- "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==",
+ "version": "0.7.2",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
+ "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"dev": true,
+ "license": "MIT",
"engines": {
"node": ">= 0.6"
}
@@ -7570,18 +7564,6 @@
"integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
"dev": true
},
- "node_modules/defaults": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
- "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==",
- "dev": true,
- "dependencies": {
- "clone": "^1.0.2"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/define-data-property": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
@@ -8549,15 +8531,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/events": {
- "version": "3.3.0",
- "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
- "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
- "dev": true,
- "engines": {
- "node": ">=0.8.x"
- }
- },
"node_modules/execa": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
@@ -8596,20 +8569,6 @@
"node": "^14.15.0 || ^16.10.0 || >=18.0.0"
}
},
- "node_modules/external-editor": {
- "version": "3.1.0",
- "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz",
- "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==",
- "dev": true,
- "dependencies": {
- "chardet": "^0.7.0",
- "iconv-lite": "^0.4.24",
- "tmp": "^0.0.33"
- },
- "engines": {
- "node": ">=4"
- }
- },
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -8683,30 +8642,6 @@
"reusify": "^1.0.4"
}
},
- "node_modules/figures": {
- "version": "3.2.0",
- "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
- "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==",
- "dev": true,
- "dependencies": {
- "escape-string-regexp": "^1.0.5"
- },
- "engines": {
- "node": ">=8"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
- "node_modules/figures/node_modules/escape-string-regexp": {
- "version": "1.0.5",
- "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
- "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
- "dev": true,
- "engines": {
- "node": ">=0.8.0"
- }
- },
"node_modules/file-entry-cache": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@@ -9141,12 +9076,13 @@
"dev": true
},
"node_modules/graphql": {
- "version": "15.8.0",
- "resolved": "https://registry.npmjs.org/graphql/-/graphql-15.8.0.tgz",
- "integrity": "sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw==",
+ "version": "16.9.0",
+ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.9.0.tgz",
+ "integrity": "sha512-GGTKBX4SD7Wdb8mqeDLni2oaRGYQWjWHGKPQ24ZMnUtKfcsVoiv4uX8+LJr1K6U5VW2Lu1BwJnj7uiori0YtRw==",
"dev": true,
+ "license": "MIT",
"engines": {
- "node": ">= 10.x"
+ "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
}
},
"node_modules/hard-rejection": {
@@ -9238,11 +9174,12 @@
"node": ">= 0.4"
}
},
- "node_modules/headers-utils": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/headers-utils/-/headers-utils-3.0.2.tgz",
- "integrity": "sha512-xAxZkM1dRyGV2Ou5bzMxBPNLoRCjcX+ya7KSWybQD2KwLphxsapUVK6x/02o7f4VU6GPSXch9vNY2+gkU8tYWQ==",
- "dev": true
+ "node_modules/headers-polyfill": {
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz",
+ "integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==",
+ "dev": true,
+ "license": "MIT"
},
"node_modules/highcharts": {
"version": "10.3.3",
@@ -9379,38 +9316,6 @@
"node": ">=0.4"
}
},
- "node_modules/iconv-lite": {
- "version": "0.4.24",
- "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
- "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
- "dev": true,
- "dependencies": {
- "safer-buffer": ">= 2.1.2 < 3"
- },
- "engines": {
- "node": ">=0.10.0"
- }
- },
- "node_modules/ieee754": {
- "version": "1.2.1",
- "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
- "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
- "dev": true,
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/feross"
- },
- {
- "type": "patreon",
- "url": "https://www.patreon.com/feross"
- },
- {
- "type": "consulting",
- "url": "https://feross.org/support"
- }
- ]
- },
"node_modules/ignore": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
@@ -9488,32 +9393,6 @@
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
"dev": true
},
- "node_modules/inquirer": {
- "version": "8.2.6",
- "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.6.tgz",
- "integrity": "sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==",
- "dev": true,
- "dependencies": {
- "ansi-escapes": "^4.2.1",
- "chalk": "^4.1.1",
- "cli-cursor": "^3.1.0",
- "cli-width": "^3.0.0",
- "external-editor": "^3.0.3",
- "figures": "^3.0.0",
- "lodash": "^4.17.21",
- "mute-stream": "0.0.8",
- "ora": "^5.4.1",
- "run-async": "^2.4.0",
- "rxjs": "^7.5.5",
- "string-width": "^4.1.0",
- "strip-ansi": "^6.0.0",
- "through": "^2.3.6",
- "wrap-ansi": "^6.0.1"
- },
- "engines": {
- "node": ">=12.0.0"
- }
- },
"node_modules/internal-slot": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz",
@@ -9736,15 +9615,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/is-interactive": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
- "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==",
- "dev": true,
- "engines": {
- "node": ">=8"
- }
- },
"node_modules/is-map": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz",
@@ -9773,7 +9643,8 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz",
"integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==",
- "dev": true
+ "dev": true,
+ "license": "MIT"
},
"node_modules/is-number": {
"version": "7.0.0",
@@ -9953,18 +9824,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/is-unicode-supported": {
- "version": "0.1.0",
- "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz",
- "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==",
- "dev": true,
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/is-weakmap": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz",
@@ -10232,15 +10091,6 @@
"jiti": "bin/jiti.js"
}
},
- "node_modules/js-levenshtein": {
- "version": "1.1.6",
- "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz",
- "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==",
- "dev": true,
- "engines": {
- "node": ">=0.10.0"
- }
- },
"node_modules/js-sha3": {
"version": "0.8.0",
"resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz",
@@ -10562,22 +10412,6 @@
"integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==",
"dev": true
},
- "node_modules/log-symbols": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz",
- "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==",
- "dev": true,
- "dependencies": {
- "chalk": "^4.1.0",
- "is-unicode-supported": "^0.1.0"
- },
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
"node_modules/long": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
@@ -10907,62 +10741,107 @@
"dev": true
},
"node_modules/msw": {
- "version": "0.36.8",
- "resolved": "https://registry.npmjs.org/msw/-/msw-0.36.8.tgz",
- "integrity": "sha512-K7lOQoYqhGhTSChsmHMQbf/SDCsxh/m0uhN6Ipt206lGoe81fpTmaGD0KLh4jUxCONMOUnwCSj0jtX2CM4pEdw==",
+ "version": "2.6.0",
+ "resolved": "https://registry.npmjs.org/msw/-/msw-2.6.0.tgz",
+ "integrity": "sha512-n3tx2w0MZ3H4pxY0ozrQ4sNPzK/dGtlr2cIIyuEsgq2Bhy4wvcW6ZH2w/gXM9+MEUY6HC1fWhqtcXDxVZr5Jxw==",
"dev": true,
"hasInstallScript": true,
+ "license": "MIT",
"dependencies": {
- "@mswjs/cookies": "^0.1.7",
- "@mswjs/interceptors": "^0.12.7",
- "@open-draft/until": "^1.0.3",
- "@types/cookie": "^0.4.1",
- "@types/inquirer": "^8.1.3",
- "@types/js-levenshtein": "^1.1.0",
- "chalk": "4.1.1",
- "chokidar": "^3.4.2",
- "cookie": "^0.4.1",
- "graphql": "^15.5.1",
- "headers-utils": "^3.0.2",
- "inquirer": "^8.2.0",
- "is-node-process": "^1.0.1",
- "js-levenshtein": "^1.1.6",
- "node-fetch": "^2.6.7",
- "path-to-regexp": "^8.0.0",
- "statuses": "^2.0.0",
- "strict-event-emitter": "^0.2.0",
- "type-fest": "^1.2.2",
- "yargs": "^17.3.0"
+ "@bundled-es-modules/cookie": "^2.0.0",
+ "@bundled-es-modules/statuses": "^1.0.1",
+ "@bundled-es-modules/tough-cookie": "^0.1.6",
+ "@inquirer/confirm": "^5.0.0",
+ "@mswjs/interceptors": "^0.36.5",
+ "@open-draft/deferred-promise": "^2.2.0",
+ "@open-draft/until": "^2.1.0",
+ "@types/cookie": "^0.6.0",
+ "@types/statuses": "^2.0.4",
+ "chalk": "^4.1.2",
+ "graphql": "^16.8.1",
+ "headers-polyfill": "^4.0.2",
+ "is-node-process": "^1.2.0",
+ "outvariant": "^1.4.3",
+ "path-to-regexp": "^6.3.0",
+ "strict-event-emitter": "^0.5.1",
+ "type-fest": "^4.26.1",
+ "yargs": "^17.7.2"
},
"bin": {
"msw": "cli/index.js"
},
+ "engines": {
+ "node": ">=18"
+ },
"funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/mswjs"
+ "url": "https://github.com/sponsors/mswjs"
+ },
+ "peerDependencies": {
+ "typescript": ">= 4.8.x"
+ },
+ "peerDependenciesMeta": {
+ "typescript": {
+ "optional": true
+ }
}
},
- "node_modules/msw/node_modules/chalk": {
- "version": "4.1.1",
- "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
- "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+ "node_modules/msw/node_modules/@inquirer/confirm": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.0.1.tgz",
+ "integrity": "sha512-6ycMm7k7NUApiMGfVc32yIPp28iPKxhGRMqoNDiUjq2RyTAkbs5Fx0TdzBqhabcKvniDdAAvHCmsRjnNfTsogw==",
"dev": true,
+ "license": "MIT",
"dependencies": {
- "ansi-styles": "^4.1.0",
- "supports-color": "^7.1.0"
+ "@inquirer/core": "^10.0.1",
+ "@inquirer/type": "^3.0.0"
},
"engines": {
- "node": ">=10"
+ "node": ">=18"
},
- "funding": {
- "url": "https://github.com/chalk/chalk?sponsor=1"
+ "peerDependencies": {
+ "@types/node": ">=18"
+ }
+ },
+ "node_modules/msw/node_modules/@inquirer/type": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.0.tgz",
+ "integrity": "sha512-YYykfbw/lefC7yKj7nanzQXILM7r3suIvyFlCcMskc99axmsSewXWkAfXKwMbgxL76iAFVmRwmYdwNZNc8gjog==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@types/node": ">=18"
+ }
+ },
+ "node_modules/msw/node_modules/@types/node": {
+ "version": "22.9.0",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.0.tgz",
+ "integrity": "sha512-vuyHg81vvWA1Z1ELfvLko2c8f34gyA0zaic0+Rllc5lbCnbSyuvb2Oxpm6TAUAC/2xZN3QGqxBNggD1nNR2AfQ==",
+ "dev": true,
+ "license": "MIT",
+ "peer": true,
+ "dependencies": {
+ "undici-types": "~6.19.8"
}
},
+ "node_modules/msw/node_modules/path-to-regexp": {
+ "version": "6.3.0",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
+ "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/mute-stream": {
- "version": "0.0.8",
- "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
- "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
- "dev": true
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz",
+ "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==",
+ "dev": true,
+ "license": "ISC",
+ "engines": {
+ "node": "^18.17.0 || >=20.5.0"
+ }
},
"node_modules/mz": {
"version": "2.7.0",
@@ -10984,15 +10863,16 @@
}
},
"node_modules/nanoid": {
- "version": "3.3.7",
- "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
- "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
+ "version": "3.3.8",
+ "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz",
+ "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/ai"
}
],
+ "license": "MIT",
"bin": {
"nanoid": "bin/nanoid.cjs"
},
@@ -13948,43 +13828,12 @@
"node": ">= 0.8.0"
}
},
- "node_modules/ora": {
- "version": "5.4.1",
- "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz",
- "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==",
- "dev": true,
- "dependencies": {
- "bl": "^4.1.0",
- "chalk": "^4.1.0",
- "cli-cursor": "^3.1.0",
- "cli-spinners": "^2.5.0",
- "is-interactive": "^1.0.0",
- "is-unicode-supported": "^0.1.0",
- "log-symbols": "^4.1.0",
- "strip-ansi": "^6.0.0",
- "wcwidth": "^1.0.1"
- },
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
- },
- "node_modules/os-tmpdir": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
- "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==",
- "dev": true,
- "engines": {
- "node": ">=0.10.0"
- }
- },
"node_modules/outvariant": {
- "version": "1.4.2",
- "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.2.tgz",
- "integrity": "sha512-Ou3dJ6bA/UJ5GVHxah4LnqDwZRwAmWxrG3wtrHrbGnP4RnLCtA64A4F+ae7Y8ww660JaddSoArUR5HjipWSHAQ==",
- "dev": true
+ "version": "1.4.3",
+ "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz",
+ "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==",
+ "dev": true,
+ "license": "MIT"
},
"node_modules/p-limit": {
"version": "3.1.0",
@@ -15712,20 +15561,6 @@
"node": ">=8"
}
},
- "node_modules/readable-stream": {
- "version": "3.6.2",
- "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
- "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
- "dev": true,
- "dependencies": {
- "inherits": "^2.0.3",
- "string_decoder": "^1.1.1",
- "util-deprecate": "^1.0.1"
- },
- "engines": {
- "node": ">= 6"
- }
- },
"node_modules/readdirp": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -15950,19 +15785,6 @@
"resolved": "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz",
"integrity": "sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng=="
},
- "node_modules/restore-cursor": {
- "version": "3.1.0",
- "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz",
- "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==",
- "dev": true,
- "dependencies": {
- "onetime": "^5.1.0",
- "signal-exit": "^3.0.2"
- },
- "engines": {
- "node": ">=8"
- }
- },
"node_modules/reusify": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
@@ -16034,15 +15856,6 @@
"integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==",
"dev": true
},
- "node_modules/run-async": {
- "version": "2.4.1",
- "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz",
- "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==",
- "dev": true,
- "engines": {
- "node": ">=0.12.0"
- }
- },
"node_modules/run-parallel": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -16071,15 +15884,6 @@
"resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz",
"integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ=="
},
- "node_modules/rxjs": {
- "version": "7.8.1",
- "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz",
- "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==",
- "dev": true,
- "dependencies": {
- "tslib": "^2.1.0"
- }
- },
"node_modules/safe-array-concat": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz",
@@ -16274,12 +16078,6 @@
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
"dev": true
},
- "node_modules/set-cookie-parser": {
- "version": "2.6.0",
- "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz",
- "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==",
- "dev": true
- },
"node_modules/set-function-length": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
@@ -16488,6 +16286,7 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
"integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
"dev": true,
+ "license": "MIT",
"engines": {
"node": ">= 0.8"
}
@@ -16499,13 +16298,11 @@
"dev": true
},
"node_modules/strict-event-emitter": {
- "version": "0.2.8",
- "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.2.8.tgz",
- "integrity": "sha512-KDf/ujU8Zud3YaLtMCcTI4xkZlZVIYxTLr+XIULexP+77EEVWixeXroLUXQXiVtH4XH2W7jr/3PT1v3zBuvc3A==",
+ "version": "0.5.1",
+ "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz",
+ "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==",
"dev": true,
- "dependencies": {
- "events": "^3.3.0"
- }
+ "license": "MIT"
},
"node_modules/strict-uri-encode": {
"version": "2.0.0",
@@ -16515,35 +16312,6 @@
"node": ">=4"
}
},
- "node_modules/string_decoder": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
- "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
- "dev": true,
- "dependencies": {
- "safe-buffer": "~5.2.0"
- }
- },
- "node_modules/string_decoder/node_modules/safe-buffer": {
- "version": "5.2.1",
- "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
- "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
- "dev": true,
- "funding": [
- {
- "type": "github",
- "url": "https://github.com/sponsors/feross"
- },
- {
- "type": "patreon",
- "url": "https://www.patreon.com/feross"
- },
- {
- "type": "consulting",
- "url": "https://feross.org/support"
- }
- ]
- },
"node_modules/string-natural-compare": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/string-natural-compare/-/string-natural-compare-3.0.1.tgz",
@@ -17158,12 +16926,6 @@
"node": ">=0.8"
}
},
- "node_modules/through": {
- "version": "2.3.8",
- "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
- "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==",
- "dev": true
- },
"node_modules/tiny-invariant": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
@@ -17198,18 +16960,6 @@
"node": ">=14.0.0"
}
},
- "node_modules/tmp": {
- "version": "0.0.33",
- "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
- "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
- "dev": true,
- "dependencies": {
- "os-tmpdir": "~1.0.2"
- },
- "engines": {
- "node": ">=0.6.0"
- }
- },
"node_modules/to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
@@ -17363,12 +17113,13 @@
}
},
"node_modules/type-fest": {
- "version": "1.4.0",
- "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz",
- "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
+ "version": "4.26.1",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz",
+ "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==",
"dev": true,
+ "license": "(MIT OR CC0-1.0)",
"engines": {
- "node": ">=10"
+ "node": ">=16"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
@@ -17505,6 +17256,14 @@
"integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==",
"dev": true
},
+ "node_modules/undici-types": {
+ "version": "6.19.8",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+ "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+ "dev": true,
+ "license": "MIT",
+ "peer": true
+ },
"node_modules/unicode-canonical-property-names-ecmascript": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz",
@@ -17874,15 +17633,6 @@
"node": ">=14"
}
},
- "node_modules/wcwidth": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
- "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==",
- "dev": true,
- "dependencies": {
- "defaults": "^1.0.3"
- }
- },
"node_modules/web-vitals": {
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-3.5.2.tgz",
@@ -18126,6 +17876,7 @@
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
"integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
"dev": true,
+ "license": "MIT",
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
@@ -18285,6 +18036,19 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/yoctocolors-cjs": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz",
+ "integrity": "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/zustand": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/zustand/-/zustand-3.7.2.tgz",
diff --git a/web/vtadmin/package.json b/web/vtadmin/package.json
index 78d14ea9e9c..5d4d5dc787a 100644
--- a/web/vtadmin/package.json
+++ b/web/vtadmin/package.json
@@ -89,7 +89,7 @@
"eslint-config-react-app": "^7.0.1",
"i": "^0.3.7",
"jsdom": "^21.1.1",
- "msw": "^0.36.8",
+ "msw": "^2.5.2",
"npm": "^10.8.0",
"postcss": "^8.4.31",
"prettier": "^2.2.1",
diff --git a/web/vtadmin/src/api/http.test.ts b/web/vtadmin/src/api/http.test.ts
index 140b1a74c70..8da5d91c55d 100644
--- a/web/vtadmin/src/api/http.test.ts
+++ b/web/vtadmin/src/api/http.test.ts
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { rest } from 'msw';
+import { http, HttpResponse } from 'msw';
import * as api from './http';
import {
@@ -32,7 +32,7 @@ vi.mock('../errors/errorHandler');
// for all requests made against the given `endpoint`.
const mockServerJson = (endpoint: string, json: object, status: number = 200) => {
const apiAddr = import.meta.env.VITE_VTADMIN_API_ADDRESS;
- global.server.use(rest.get(`${apiAddr}${endpoint}`, (req, res, ctx) => res(ctx.status(status), ctx.json(json))));
+ global.server.use(http.get(`${apiAddr}${endpoint}`, (info) => HttpResponse.json(json, { status: status })));
};
describe('api/http', () => {
@@ -80,8 +80,10 @@ describe('api/http', () => {
errorHandler.notify.mockReset();
const endpoint = `/api/tablets`;
global.server.use(
- rest.get(`${import.meta.env.VITE_VTADMIN_API_ADDRESS}${endpoint}`, (req, res, ctx) =>
- res(ctx.status(504), ctx.body('504 Gateway Time-out'))
+ http.get(`${import.meta.env.VITE_VTADMIN_API_ADDRESS}${endpoint}`, (info) =>
+ HttpResponse.html('504 Gateway Time-out', {
+ status: 504,
+ })
)
);
@@ -94,7 +96,7 @@ describe('api/http', () => {
/* eslint-disable jest/no-conditional-expect */
expect(e.name).toEqual(MALFORMED_HTTP_RESPONSE_ERROR);
expect(e.message).toContain(
- '[status 504] /api/tablets: invalid json response body at http://test-api.com/api/tablets'
+ `[status 504] /api/tablets: Unexpected token '<', " {
+ const { result } = await vtfetch(`/api/transaction/${clusterID}/${dtid}/info`);
+
+ const err = vtctldata.GetTransactionInfoResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.GetTransactionInfoResponse.create(result);
+};
+
export interface FetchTransactionsParams {
clusterID: string;
keyspace: string;
+ abandonAge?: string;
}
-export const fetchTransactions = async ({ clusterID, keyspace }: FetchTransactionsParams) => {
- const { result } = await vtfetch(`/api/transactions/${clusterID}/${keyspace}`);
+export const fetchTransactions = async ({ clusterID, keyspace, abandonAge = '' }: FetchTransactionsParams) => {
+ const req = new URLSearchParams();
+ req.append('abandon_age', abandonAge);
+
+ const { result } = await vtfetch(`/api/transactions/${clusterID}/${keyspace}?${req}`);
const err = vtctldata.GetUnresolvedTransactionsResponse.verify(result);
if (err) throw Error(err);
@@ -435,6 +453,19 @@ export const fetchTransactions = async ({ clusterID, keyspace }: FetchTransactio
return vtctldata.GetUnresolvedTransactionsResponse.create(result);
};
+export interface ConcludeTransactionParams {
+ clusterID: string;
+ dtid: string;
+}
+
+export const concludeTransaction = async ({ clusterID, dtid }: ConcludeTransactionParams) => {
+ const { result } = await vtfetch(`/api/transaction/${clusterID}/${dtid}/conclude`);
+ const err = vtctldata.ConcludeTransactionResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.ConcludeTransactionResponse.create(result);
+};
+
export const fetchWorkflows = async () => {
const { result } = await vtfetch(`/api/workflows`);
@@ -462,6 +493,29 @@ export const fetchWorkflowStatus = async (params: { clusterID: string; keyspace:
return vtctldata.WorkflowStatusResponse.create(result);
};
+export interface CreateMaterializeParams {
+ clusterID: string;
+ tableSettings: string;
+ request: vtctldata.IMaterializeCreateRequest;
+}
+
+export const createMaterialize = async ({ clusterID, tableSettings, request }: CreateMaterializeParams) => {
+ const body = {
+ table_settings: tableSettings,
+ request: request,
+ };
+
+ const { result } = await vtfetch(`/api/workflow/${clusterID}/materialize`, {
+ body: JSON.stringify(body),
+ method: 'post',
+ });
+
+ const err = vtctldata.MaterializeCreateResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.MaterializeCreateResponse.create(result);
+};
+
export interface CreateMoveTablesParams {
clusterID: string;
request: vtctldata.IMoveTablesCreateRequest;
@@ -479,6 +533,23 @@ export const createMoveTables = async ({ clusterID, request }: CreateMoveTablesP
return vtctldata.WorkflowStatusResponse.create(result);
};
+export interface CreateReshardParams {
+ clusterID: string;
+ request: vtctldata.IReshardCreateRequest;
+}
+
+export const createReshard = async ({ clusterID, request }: CreateReshardParams) => {
+ const { result } = await vtfetch(`/api/workflow/${clusterID}/reshard`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+
+ const err = vtctldata.WorkflowStatusResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.WorkflowStatusResponse.create(result);
+};
+
export interface WorkflowActionParams {
clusterID: string;
keyspace: string;
@@ -501,6 +572,54 @@ export const stopWorkflow = async ({ clusterID, keyspace, name }: WorkflowAction
return vtctldata.WorkflowUpdateResponse.create(result);
};
+export interface MoveTablesCompleteParams {
+ clusterID: string;
+ request: vtctldata.IMoveTablesCompleteRequest;
+}
+
+export const completeMoveTables = async ({ clusterID, request }: MoveTablesCompleteParams) => {
+ const { result } = await vtfetch(`/api/movetables/${clusterID}/complete`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+ const err = vtctldata.MoveTablesCompleteResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.MoveTablesCompleteResponse.create(result);
+};
+
+export interface WorkflowSwitchTrafficParams {
+ clusterID: string;
+ request: vtctldata.IWorkflowSwitchTrafficRequest;
+}
+
+export const workflowSwitchTraffic = async ({ clusterID, request }: WorkflowSwitchTrafficParams) => {
+ const { result } = await vtfetch(`/api/workflow/${clusterID}/switchtraffic`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+ const err = vtctldata.WorkflowSwitchTrafficResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.WorkflowSwitchTrafficResponse.create(result);
+};
+
+export interface WorkflowDeleteParams {
+ clusterID: string;
+ request: vtctldata.IWorkflowDeleteRequest;
+}
+
+export const workflowDelete = async ({ clusterID, request }: WorkflowDeleteParams) => {
+ const { result } = await vtfetch(`/api/workflow/${clusterID}/delete`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+ const err = vtctldata.WorkflowDeleteResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.WorkflowDeleteResponse.create(result);
+};
+
export const fetchVTExplain = async ({ cluster, keyspace, sql }: R) => {
// As an easy enhancement for later, we can also validate the request parameters on the front-end
// instead of defaulting to '', to save a round trip.
@@ -915,3 +1034,75 @@ export const validateVersionShard = async (params: ValidateVersionShardParams) =
return vtctldata.ValidateVersionShardResponse.create(result);
};
+
+export interface CreateVDiffParams {
+ clusterID: string;
+ request: vtctldata.IVDiffCreateRequest;
+}
+
+export const createVDiff = async ({ clusterID, request }: CreateVDiffParams) => {
+ const { result } = await vtfetch(`/api/vdiff/${clusterID}/`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+
+ const err = vtctldata.VDiffCreateResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.VDiffCreateResponse.create(result);
+};
+
+export interface ShowVDiffParams {
+ clusterID: string;
+ request: vtctldata.IVDiffShowRequest;
+}
+
+export const showVDiff = async ({ clusterID, request }: ShowVDiffParams) => {
+ const { result } = await vtfetch(`/api/vdiff/${clusterID}/show`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+
+ const err = vtadmin.VDiffShowResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtadmin.VDiffShowResponse.create(result);
+};
+
+export const fetchSchemaMigrations = async (request: vtadmin.IGetSchemaMigrationsRequest) => {
+ const { result } = await vtfetch(`/api/migrations/`, {
+ body: JSON.stringify(request),
+ method: 'post',
+ });
+
+ const err = vtadmin.GetSchemaMigrationsResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtadmin.GetSchemaMigrationsResponse.create(result);
+};
+
+export interface ApplySchemaParams {
+ clusterID: string;
+ keyspace: string;
+ callerID: string;
+ sql: string;
+ request: vtctldata.IApplySchemaRequest;
+}
+
+export const applySchema = async ({ clusterID, keyspace, callerID, sql, request }: ApplySchemaParams) => {
+ const body = {
+ sql,
+ caller_id: callerID,
+ request,
+ };
+
+ const { result } = await vtfetch(`/api/migration/${clusterID}/${keyspace}`, {
+ body: JSON.stringify(body),
+ method: 'post',
+ });
+
+ const err = vtctldata.ApplySchemaResponse.verify(result);
+ if (err) throw Error(err);
+
+ return vtctldata.ApplySchemaResponse.create(result);
+};
diff --git a/web/vtadmin/src/components/ActionPanel.test.tsx b/web/vtadmin/src/components/ActionPanel.test.tsx
index 6d0bc02b30f..6f1bed60603 100644
--- a/web/vtadmin/src/components/ActionPanel.test.tsx
+++ b/web/vtadmin/src/components/ActionPanel.test.tsx
@@ -16,7 +16,7 @@
import { render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
-import { rest } from 'msw';
+import { delay, http, HttpResponse } from 'msw';
import { QueryClient, QueryClientProvider, useMutation } from 'react-query';
import { describe, it, expect, vi } from 'vitest';
@@ -33,11 +33,7 @@ describe('ActionPanel', () => {
* provides such a function and should be `render`ed in the context QueryClientProvider.
*/
const Wrapper: React.FC = (props) => {
- const mutation = useMutation(() => fetch(new URL(props['url']), { method: 'post' }), {
- onError: (error) => {
- console.log('ERROR: ', error);
- },
- });
+ const mutation = useMutation(() => fetch(new URL(props['url']), { method: 'post' }));
return ;
};
@@ -46,8 +42,9 @@ describe('ActionPanel', () => {
const url = `${import.meta.env.VITE_VTADMIN_API_ADDRESS}/api/test`;
global.server.use(
- rest.post(url, (req, res, ctx) => {
- return res(ctx.json({ ok: true }));
+ http.post(url, async (info) => {
+ await delay();
+ return HttpResponse.json({ ok: true });
})
);
diff --git a/web/vtadmin/src/components/App.tsx b/web/vtadmin/src/components/App.tsx
index e65bbfde32b..fd0f772ae19 100644
--- a/web/vtadmin/src/components/App.tsx
+++ b/web/vtadmin/src/components/App.tsx
@@ -42,6 +42,12 @@ import { Topology } from './routes/topology/Topology';
import { ClusterTopology } from './routes/topology/ClusterTopology';
import { CreateMoveTables } from './routes/createWorkflow/CreateMoveTables';
import { Transactions } from './routes/Transactions';
+import { Transaction } from './routes/transaction/Transaction';
+import { CreateReshard } from './routes/createWorkflow/CreateReshard';
+import { CreateMaterialize } from './routes/createWorkflow/CreateMaterialize';
+import { TopologyTree } from './routes/topologyTree/TopologyTree';
+import { SchemaMigrations } from './routes/SchemaMigrations';
+import { CreateSchemaMigration } from './routes/createSchemaMigration/CreateSchemaMigration';
export const App = () => {
return (
@@ -117,6 +123,18 @@ export const App = () => {
)}
+ {!isReadOnlyMode() && (
+
+
+
+ )}
+
+ {!isReadOnlyMode() && (
+
+
+
+ )}
+
@@ -125,14 +143,32 @@ export const App = () => {
+
+
+
+
+ {!isReadOnlyMode() && (
+
+
+
+ )}
+
+
+
+
+
+
+
+
+
diff --git a/web/vtadmin/src/components/NavRail.tsx b/web/vtadmin/src/components/NavRail.tsx
index 9f9e1bf1681..b30cd165684 100644
--- a/web/vtadmin/src/components/NavRail.tsx
+++ b/web/vtadmin/src/components/NavRail.tsx
@@ -65,6 +65,9 @@ export const NavRail = () => {
+
+
+
diff --git a/web/vtadmin/src/components/dataTable/DataTable.tsx b/web/vtadmin/src/components/dataTable/DataTable.tsx
index ed14067bd34..9e863a6aa54 100644
--- a/web/vtadmin/src/components/dataTable/DataTable.tsx
+++ b/web/vtadmin/src/components/dataTable/DataTable.tsx
@@ -31,6 +31,10 @@ interface Props {
pageSize?: number;
renderRows: (rows: T[]) => JSX.Element[];
title?: string;
+ // Pass a unique `pageKey` for each DataTable, in case multiple
+ // DataTables access the same URL. This will be used to
+ // access page number from the URL.
+ pageKey?: string;
}
// Generally, page sizes of ~100 rows are fine in terms of performance,
@@ -43,12 +47,15 @@ export const DataTable = ({
pageSize = DEFAULT_PAGE_SIZE,
renderRows,
title,
+ pageKey = '',
}: Props) => {
const { pathname } = useLocation();
const urlQuery = useURLQuery();
+ const pageQueryKey = `${pageKey}page`;
+
const totalPages = Math.ceil(data.length / pageSize);
- const { page } = useURLPagination({ totalPages });
+ const { page } = useURLPagination({ totalPages, pageQueryKey });
const startIndex = (page - 1) * pageSize;
const endIndex = startIndex + pageSize;
@@ -59,7 +66,7 @@ export const DataTable = ({
const formatPageLink = (p: number) => ({
pathname,
- search: stringify({ ...urlQuery.query, page: p === 1 ? undefined : p }),
+ search: stringify({ ...urlQuery.query, [pageQueryKey]: p === 1 ? undefined : p }),
});
return (
diff --git a/web/vtadmin/src/components/routes/createWorkflow/ErrorDialog.tsx b/web/vtadmin/src/components/dialog/ErrorDialog.tsx
similarity index 94%
rename from web/vtadmin/src/components/routes/createWorkflow/ErrorDialog.tsx
rename to web/vtadmin/src/components/dialog/ErrorDialog.tsx
index 25ac5dedb0b..087876e4cd2 100644
--- a/web/vtadmin/src/components/routes/createWorkflow/ErrorDialog.tsx
+++ b/web/vtadmin/src/components/dialog/ErrorDialog.tsx
@@ -14,8 +14,8 @@
* limitations under the License.
*/
import React from 'react';
-import Dialog from '../../dialog/Dialog';
-import { Icon, Icons } from '../../Icon';
+import Dialog from './Dialog';
+import { Icon, Icons } from '../Icon';
export interface ErrorDialogProps {
errorTitle?: string;
diff --git a/web/vtadmin/src/components/inputs/Select.tsx b/web/vtadmin/src/components/inputs/Select.tsx
index d1fdd83af6e..588c350073c 100644
--- a/web/vtadmin/src/components/inputs/Select.tsx
+++ b/web/vtadmin/src/components/inputs/Select.tsx
@@ -37,6 +37,7 @@ interface Props {
size?: 'large';
description?: string;
required?: boolean;
+ disableClearSelection?: boolean;
}
/**
@@ -60,6 +61,7 @@ export const Select = ({
size,
description,
required,
+ disableClearSelection,
}: Props) => {
const _itemToString = React.useCallback(
(item: T | null): string => {
@@ -146,7 +148,7 @@ export const Select = ({
{content}
- {selectedItem && (
+ {selectedItem && !disableClearSelection && (
diff --git a/web/vtadmin/src/components/links/TransactionLink.tsx b/web/vtadmin/src/components/links/TransactionLink.tsx
new file mode 100644
index 00000000000..64f9d12ddf4
--- /dev/null
+++ b/web/vtadmin/src/components/links/TransactionLink.tsx
@@ -0,0 +1,40 @@
+/**
+ * Copyright 2024 The Vitess Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { Link } from 'react-router-dom';
+
+interface Props {
+ className?: string;
+ clusterID: string | null | undefined;
+ dtid: string | null | undefined;
+}
+
+export const TransactionLink: React.FunctionComponent = ({ children, className, clusterID, dtid, ...props }) => {
+ if (!clusterID || !dtid) {
+ return {children};
+ }
+
+ const to = {
+ pathname: `/transaction/${clusterID}/${dtid}`,
+ };
+
+ return (
+
+ {children}
+
+ );
+};
diff --git a/web/vtadmin/src/components/routes/SchemaMigrations.tsx b/web/vtadmin/src/components/routes/SchemaMigrations.tsx
new file mode 100644
index 00000000000..1761d26de49
--- /dev/null
+++ b/web/vtadmin/src/components/routes/SchemaMigrations.tsx
@@ -0,0 +1,195 @@
+/**
+ * Copyright 2024 The Vitess Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { useEffect, useState } from 'react';
+import { useKeyspaces, useSchemaMigrations } from '../../hooks/api';
+import { DataCell } from '../dataTable/DataCell';
+import { DataTable } from '../dataTable/DataTable';
+import { ContentContainer } from '../layout/ContentContainer';
+import { WorkspaceHeader } from '../layout/WorkspaceHeader';
+import { WorkspaceTitle } from '../layout/WorkspaceTitle';
+import { QueryLoadingPlaceholder } from '../placeholders/QueryLoadingPlaceholder';
+import { useDocumentTitle } from '../../hooks/useDocumentTitle';
+import { vtadmin } from '../../proto/vtadmin';
+import { Select } from '../inputs/Select';
+import { ShardLink } from '../links/ShardLink';
+import { formatDateTime } from '../../util/time';
+import { ReadOnlyGate } from '../ReadOnlyGate';
+import { formatSchemaMigrationStatus } from '../../util/schemaMigrations';
+import { Link } from 'react-router-dom';
+import { TabletLink } from '../links/TabletLink';
+import { formatAlias } from '../../util/tablets';
+import { useURLQuery } from '../../hooks/useURLQuery';
+
+const COLUMNS = ['UUID', 'Status', 'DDL Action', 'Timestamps', 'Stage', 'Progress'];
+
+export const SchemaMigrations = () => {
+ useDocumentTitle('Schema Migrations');
+
+ const { query, replaceQuery } = useURLQuery();
+ const urlKeyspace = query['keyspace'];
+ const urlCluster = query['cluster'];
+
+ const keyspacesQuery = useKeyspaces();
+ const { data: keyspaces = [], ...ksQuery } = keyspacesQuery;
+
+ const [selectedKeyspace, setSelectedKeypsace] = useState();
+
+ const request: vtadmin.IGetSchemaMigrationsRequest = {
+ cluster_requests: [
+ {
+ cluster_id: selectedKeyspace && selectedKeyspace.cluster?.id,
+ request: {
+ keyspace: selectedKeyspace && selectedKeyspace.keyspace?.name,
+ },
+ },
+ ],
+ };
+
+ const schemaMigrationsQuery = useSchemaMigrations(request, {
+ enabled: !!selectedKeyspace,
+ });
+
+ const schemaMigrations = schemaMigrationsQuery.data ? schemaMigrationsQuery.data.schema_migrations : [];
+
+ const handleKeyspaceChange = (ks: vtadmin.Keyspace | null | undefined) => {
+ setSelectedKeypsace(ks);
+
+ if (ks) {
+ replaceQuery({ keyspace: ks.keyspace?.name, cluster: ks.cluster?.id });
+ } else {
+ replaceQuery({ keyspace: undefined, cluster: undefined });
+ }
+ };
+
+ useEffect(() => {
+ if (urlKeyspace && urlCluster) {
+ const keyspace = keyspaces.find(
+ (ks) => ks.cluster?.id === String(urlCluster) && ks.keyspace?.name === String(urlKeyspace)
+ );
+
+ if (keyspace) {
+ setSelectedKeypsace(keyspace);
+ } else if (!ksQuery.isLoading) {
+ replaceQuery({ keyspace: undefined, cluster: undefined });
+ }
+ } else {
+ setSelectedKeypsace(undefined);
+ }
+ }, [urlKeyspace, urlCluster, keyspaces, ksQuery.isLoading, replaceQuery]);
+
+ const renderRows = (rows: vtadmin.ISchemaMigration[]) => {
+ return rows.map((row) => {
+ const migrationInfo = row.schema_migration;
+
+ if (!migrationInfo) return <>;
+
+ return (
+