From 072d1fadb63c51334aa2c4fc90571a8ec11e0323 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:09:41 +0530 Subject: [PATCH 01/28] Fix errant GTID detection in VTOrc to also work with a replica not connected to any primary (#17267) Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/general/vtorc_test.go | 58 +++++++++ go/vt/vtorc/inst/instance_dao.go | 82 ++++++++----- go/vt/vtorc/inst/instance_dao_test.go | 118 +++++++++++++++++++ 3 files changed, 229 insertions(+), 29 deletions(-) diff --git a/go/test/endtoend/vtorc/general/vtorc_test.go b/go/test/endtoend/vtorc/general/vtorc_test.go index 329601deb0c..2ec2fd4b0ae 100644 --- a/go/test/endtoend/vtorc/general/vtorc_test.go +++ b/go/test/endtoend/vtorc/general/vtorc_test.go @@ -62,6 +62,64 @@ func TestPrimaryElection(t *testing.T) { require.Len(t, res.Rows, 1, "There should only be 1 primary tablet which was elected") } +// TestErrantGTIDOnPreviousPrimary tests that VTOrc is able to detect errant GTIDs on a previously demoted primary +// if it has an errant GTID. +func TestErrantGTIDOnPreviousPrimary(t *testing.T) { + defer utils.PrintVTOrcLogsOnFailure(t, clusterInfo.ClusterInstance) + defer cluster.PanicHandler(t) + utils.SetupVttabletsAndVTOrcs(t, clusterInfo, 3, 0, []string{"--change-tablets-with-errant-gtid-to-drained"}, cluster.VTOrcConfiguration{}, 1, "") + keyspace := &clusterInfo.ClusterInstance.Keyspaces[0] + shard0 := &keyspace.Shards[0] + + // find primary from topo + curPrimary := utils.ShardPrimaryTablet(t, clusterInfo, keyspace, shard0) + assert.NotNil(t, curPrimary, "should have elected a primary") + vtOrcProcess := clusterInfo.ClusterInstance.VTOrcProcesses[0] + utils.WaitForSuccessfulRecoveryCount(t, vtOrcProcess, logic.ElectNewPrimaryRecoveryName, 1) + utils.WaitForSuccessfulPRSCount(t, vtOrcProcess, keyspace.Name, shard0.Name, 1) + + var replica, otherReplica *cluster.Vttablet + for _, tablet := range shard0.Vttablets { + // we know we have only two tablets, so the "other" one must be the new primary + if tablet.Alias != curPrimary.Alias { + if replica == nil { + replica = tablet + } else { + otherReplica = tablet + } + } + } + require.NotNil(t, replica, "should be able to find a replica") + require.NotNil(t, otherReplica, "should be able to find 2nd replica") + utils.CheckReplication(t, clusterInfo, curPrimary, []*cluster.Vttablet{replica, otherReplica}, 15*time.Second) + + // Disable global recoveries for the cluster. + vtOrcProcess.DisableGlobalRecoveries(t) + + // Run PRS to promote a different replica. + output, err := clusterInfo.ClusterInstance.VtctldClientProcess.ExecuteCommandWithOutput( + "PlannedReparentShard", + fmt.Sprintf("%s/%s", keyspace.Name, shard0.Name), + "--new-primary", replica.Alias) + require.NoError(t, err, "error in PlannedReparentShard output - %s", output) + + // Stop replicatin on the previous primary to simulate it not reparenting properly. + // Also insert an errant GTID on the previous primary. + err = utils.RunSQLs(t, []string{ + "STOP REPLICA", + "RESET REPLICA ALL", + "set global read_only=OFF", + "insert into vt_ks.vt_insert_test(id, msg) values (10173, 'test 178342')", + }, curPrimary, "") + require.NoError(t, err) + + // Wait for VTOrc to detect the errant GTID and change the tablet to a drained type. + vtOrcProcess.EnableGlobalRecoveries(t) + + // Wait for the tablet to be drained. + utils.WaitForTabletType(t, curPrimary, "drained") +} + // Cases to test: // 1. create cluster with 1 replica and 1 rdonly, let orc choose primary // verify rdonly is not elected, only replica diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index bd4438dd05f..bd437d1ddac 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -357,35 +357,7 @@ Cleanup: // Add replication group ancestry UUID as well. Otherwise, VTOrc thinks there are errant GTIDs in group // members and its replicas, even though they are not. instance.AncestryUUID = strings.Trim(instance.AncestryUUID, ",") - if instance.ExecutedGtidSet != "" && instance.primaryExecutedGtidSet != "" { - // Compare primary & replica GTID sets, but ignore the sets that present the primary's UUID. - // This is because vtorc may pool primary and replica at an inconvenient timing, - // such that the replica may _seems_ to have more entries than the primary, when in fact - // it's just that the primary's probing is stale. - redactedExecutedGtidSet, _ := NewOracleGtidSet(instance.ExecutedGtidSet) - for _, uuid := range strings.Split(instance.AncestryUUID, ",") { - if uuid != instance.ServerUUID { - redactedExecutedGtidSet.RemoveUUID(uuid) - } - if instance.IsCoPrimary && uuid == instance.ServerUUID { - // If this is a co-primary, then this server is likely to show its own generated GTIDs as errant, - // because its co-primary has not applied them yet - redactedExecutedGtidSet.RemoveUUID(uuid) - } - } - // Avoid querying the database if there's no point: - if !redactedExecutedGtidSet.IsEmpty() { - redactedPrimaryExecutedGtidSet, _ := NewOracleGtidSet(instance.primaryExecutedGtidSet) - redactedPrimaryExecutedGtidSet.RemoveUUID(instance.SourceUUID) - - instance.GtidErrant, err = replication.Subtract(redactedExecutedGtidSet.String(), redactedPrimaryExecutedGtidSet.String()) - if err == nil { - var gtidCount int64 - gtidCount, err = replication.GTIDCount(instance.GtidErrant) - currentErrantGTIDCount.Set(tabletAlias, gtidCount) - } - } - } + err = detectErrantGTIDs(tabletAlias, instance, tablet) } latency.Stop("instance") @@ -412,6 +384,58 @@ Cleanup: return nil, err } +// detectErrantGTIDs detects the errant GTIDs on an instance. +func detectErrantGTIDs(tabletAlias string, instance *Instance, tablet *topodatapb.Tablet) (err error) { + // If the tablet is not replicating from anyone, then it could be the previous primary. + // We should check for errant GTIDs by finding the difference with the shard's current primary. + if instance.primaryExecutedGtidSet == "" && instance.SourceHost == "" { + var primaryInstance *Instance + primaryAlias, _, _ := ReadShardPrimaryInformation(tablet.Keyspace, tablet.Shard) + if primaryAlias != "" { + primaryInstance, _, _ = ReadInstance(primaryAlias) + } + // Only run errant GTID detection, if we are sure that the data read of the current primary + // is up-to-date enough to reflect that it has been promoted. This is needed to prevent + // flagging incorrect errant GTIDs. If we were to use old data, we could have some GTIDs + // accepted by the old primary (this tablet) that don't show in the new primary's set. + if primaryInstance != nil { + if primaryInstance.SourceHost == "" { + instance.primaryExecutedGtidSet = primaryInstance.ExecutedGtidSet + } + } + } + if instance.ExecutedGtidSet != "" && instance.primaryExecutedGtidSet != "" { + // Compare primary & replica GTID sets, but ignore the sets that present the primary's UUID. + // This is because vtorc may pool primary and replica at an inconvenient timing, + // such that the replica may _seems_ to have more entries than the primary, when in fact + // it's just that the primary's probing is stale. + redactedExecutedGtidSet, _ := NewOracleGtidSet(instance.ExecutedGtidSet) + for _, uuid := range strings.Split(instance.AncestryUUID, ",") { + if uuid != instance.ServerUUID { + redactedExecutedGtidSet.RemoveUUID(uuid) + } + if instance.IsCoPrimary && uuid == instance.ServerUUID { + // If this is a co-primary, then this server is likely to show its own generated GTIDs as errant, + // because its co-primary has not applied them yet + redactedExecutedGtidSet.RemoveUUID(uuid) + } + } + // Avoid querying the database if there's no point: + if !redactedExecutedGtidSet.IsEmpty() { + redactedPrimaryExecutedGtidSet, _ := NewOracleGtidSet(instance.primaryExecutedGtidSet) + redactedPrimaryExecutedGtidSet.RemoveUUID(instance.SourceUUID) + + instance.GtidErrant, err = replication.Subtract(redactedExecutedGtidSet.String(), redactedPrimaryExecutedGtidSet.String()) + if err == nil { + var gtidCount int64 + gtidCount, err = replication.GTIDCount(instance.GtidErrant) + currentErrantGTIDCount.Set(tabletAlias, gtidCount) + } + } + } + return err +} + // getKeyspaceShardName returns a single string having both the keyspace and shard func getKeyspaceShardName(keyspace, shard string) string { return fmt.Sprintf("%v:%v", keyspace, shard) diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index 2416c1abb90..cb2064d10c0 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -14,6 +14,7 @@ import ( "vitess.io/vitess/go/vt/external/golib/sqlutils" "vitess.io/vitess/go/vt/log" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtorc/config" "vitess.io/vitess/go/vt/vtorc/db" @@ -773,3 +774,120 @@ func TestExpireTableData(t *testing.T) { }) } } + +func TestDetectErrantGTIDs(t *testing.T) { + tests := []struct { + name string + instance *Instance + primaryInstance *Instance + wantErr bool + wantErrantGTID string + }{ + { + name: "No errant GTIDs", + instance: &Instance{ + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34", + primaryExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10591,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34", + AncestryUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff,230ea8ea-81e3-11e4-972a-e25ec4bd140a", + ServerUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + SourceUUID: "230ea8ea-81e3-11e4-972a-e25ec4bd140a", + }, + }, { + name: "Errant GTIDs on replica", + instance: &Instance{ + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:34", + primaryExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10591,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34", + AncestryUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff,230ea8ea-81e3-11e4-972a-e25ec4bd140a", + ServerUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + SourceUUID: "230ea8ea-81e3-11e4-972a-e25ec4bd140a", + }, + wantErrantGTID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff:34", + }, + { + name: "No errant GTIDs on old primary", + instance: &Instance{ + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-341", + AncestryUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + ServerUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + }, + primaryInstance: &Instance{ + SourceHost: "", + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10589,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-341", + }, + }, + { + name: "Errant GTIDs on old primary", + instance: &Instance{ + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-342", + AncestryUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + ServerUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + }, + primaryInstance: &Instance{ + SourceHost: "", + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10589,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-341", + }, + wantErrantGTID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff:342", + }, { + name: "Old information for new primary", + instance: &Instance{ + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-342", + AncestryUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + ServerUUID: "316d193c-70e5-11e5-adb2-ecf4bb2262ff", + }, + primaryInstance: &Instance{ + SourceHost: "localhost", + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10539,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-311", + }, + }, + } + + keyspaceName := "ks" + shardName := "0" + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone-1", + Uid: 100, + }, + Keyspace: keyspaceName, + Shard: shardName, + } + primaryTablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone-1", + Uid: 100, + }, + Keyspace: keyspaceName, + Shard: shardName, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Clear the database after the test. The easiest way to do that is to run all the initialization commands again. + defer func() { + db.ClearVTOrcDatabase() + }() + db.ClearVTOrcDatabase() + + // Save shard record for the primary tablet. + err := SaveShard(topo.NewShardInfo(keyspaceName, shardName, &topodatapb.Shard{ + PrimaryAlias: primaryTablet.Alias, + }, nil)) + require.NoError(t, err) + + if tt.primaryInstance != nil { + tt.primaryInstance.InstanceAlias = topoproto.TabletAliasString(primaryTablet.Alias) + err = SaveTablet(primaryTablet) + require.NoError(t, err) + err = WriteInstance(tt.primaryInstance, true, nil) + require.NoError(t, err) + } + + err = detectErrantGTIDs(topoproto.TabletAliasString(tablet.Alias), tt.instance, tablet) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, tt.wantErrantGTID, tt.instance.GtidErrant) + }) + } +} From 145ff6b09e223e4fd89eaf4813be81ec46d04b2e Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 25 Nov 2024 19:03:55 +0100 Subject: [PATCH 02/28] Specify Ubuntu 24.04 for all jobs (#17278) Signed-off-by: Dirkjan Bussink --- .github/workflows/assign_milestone.yml | 2 +- .github/workflows/auto_approve_pr.yml | 2 +- .github/workflows/check_label.yml | 2 +- .../check_make_vtadmin_authz_testgen.yml | 4 +-- .../check_make_vtadmin_web_proto.yml | 2 +- .../workflows/close_stale_pull_requests.yml | 2 +- .github/workflows/cluster_endtoend_12.yml | 13 ++++++-- .github/workflows/cluster_endtoend_13.yml | 13 ++++++-- .github/workflows/cluster_endtoend_15.yml | 13 ++++++-- .github/workflows/cluster_endtoend_18.yml | 13 ++++++-- .github/workflows/cluster_endtoend_21.yml | 13 ++++++-- .../cluster_endtoend_backup_pitr.yml | 13 ++++++-- ...luster_endtoend_backup_pitr_mysqlshell.yml | 13 ++++++-- ...luster_endtoend_backup_pitr_xtrabackup.yml | 2 +- ...ter_endtoend_ers_prs_newfeatures_heavy.yml | 13 ++++++-- .../workflows/cluster_endtoend_mysql80.yml | 13 ++++++-- .../cluster_endtoend_mysql_server_vault.yml | 13 ++++++-- .../cluster_endtoend_onlineddl_revert.yml | 13 ++++++-- .../cluster_endtoend_onlineddl_scheduler.yml | 13 ++++++-- .../cluster_endtoend_onlineddl_vrepl.yml | 13 ++++++-- ...luster_endtoend_onlineddl_vrepl_stress.yml | 13 ++++++-- ..._endtoend_onlineddl_vrepl_stress_suite.yml | 13 ++++++-- ...cluster_endtoend_onlineddl_vrepl_suite.yml | 13 ++++++-- .../cluster_endtoend_schemadiff_vrepl.yml | 13 ++++++-- .../cluster_endtoend_tabletmanager_consul.yml | 13 ++++++-- ...cluster_endtoend_tabletmanager_tablegc.yml | 13 ++++++-- ..._endtoend_tabletmanager_throttler_topo.yml | 13 ++++++-- ...cluster_endtoend_topo_connection_cache.yml | 13 ++++++-- ...dtoend_vreplication_across_db_versions.yml | 13 ++++++-- .../cluster_endtoend_vreplication_basic.yml | 13 ++++++-- ...luster_endtoend_vreplication_cellalias.yml | 13 ++++++-- ...er_endtoend_vreplication_copy_parallel.yml | 13 ++++++-- ...dtoend_vreplication_foreign_key_stress.yml | 13 ++++++-- ...endtoend_vreplication_mariadb_to_mysql.yml | 13 ++++++-- .../cluster_endtoend_vreplication_migrate.yml | 13 ++++++-- ...ter_endtoend_vreplication_multi_tenant.yml | 13 ++++++-- ...ion_partial_movetables_and_materialize.yml | 13 ++++++-- .../cluster_endtoend_vreplication_v2.yml | 13 ++++++-- ...tion_vtctldclient_vdiff2_movetables_tz.yml | 13 ++++++-- .../workflows/cluster_endtoend_vstream.yml | 13 ++++++-- .../workflows/cluster_endtoend_vtbackup.yml | 13 ++++++-- ..._vtctlbackup_sharded_clustertest_heavy.yml | 13 ++++++-- .../cluster_endtoend_vtgate_concurrentdml.yml | 13 ++++++-- ...ster_endtoend_vtgate_foreignkey_stress.yml | 13 ++++++-- .../cluster_endtoend_vtgate_gen4.yml | 13 ++++++-- .../cluster_endtoend_vtgate_general_heavy.yml | 13 ++++++-- .../cluster_endtoend_vtgate_godriver.yml | 13 ++++++-- ...uster_endtoend_vtgate_partial_keyspace.yml | 13 ++++++-- .../cluster_endtoend_vtgate_queries.yml | 13 ++++++-- ...cluster_endtoend_vtgate_readafterwrite.yml | 13 ++++++-- .../cluster_endtoend_vtgate_reservedconn.yml | 13 ++++++-- .../cluster_endtoend_vtgate_schema.yml | 13 ++++++-- ...cluster_endtoend_vtgate_schema_tracker.yml | 13 ++++++-- ...dtoend_vtgate_tablet_healthcheck_cache.yml | 13 ++++++-- .../cluster_endtoend_vtgate_topo.yml | 13 ++++++-- .../cluster_endtoend_vtgate_topo_consul.yml | 13 ++++++-- .../cluster_endtoend_vtgate_topo_etcd.yml | 13 ++++++-- .../cluster_endtoend_vtgate_transaction.yml | 13 ++++++-- .../cluster_endtoend_vtgate_unsharded.yml | 13 ++++++-- .../cluster_endtoend_vtgate_vindex_heavy.yml | 13 ++++++-- .../cluster_endtoend_vtgate_vschema.yml | 13 ++++++-- .github/workflows/cluster_endtoend_vtorc.yml | 13 ++++++-- .../cluster_endtoend_vttablet_prscomplex.yml | 13 ++++++-- .../workflows/cluster_endtoend_xb_backup.yml | 2 +- .../cluster_endtoend_xb_recovery.yml | 2 +- .github/workflows/code_freeze.yml | 2 +- .github/workflows/codecov.yml | 2 +- .github/workflows/codeql_analysis.yml | 6 ++-- .github/workflows/create_release.yml | 2 +- .github/workflows/docker_build_images.yml | 8 ++--- .github/workflows/docker_test_cluster_10.yml | 2 +- .github/workflows/docker_test_cluster_25.yml | 2 +- .github/workflows/e2e_race.yml | 6 ++-- .github/workflows/endtoend.yml | 4 +-- .github/workflows/local_example.yml | 22 ++----------- .github/workflows/region_example.yml | 22 ++----------- .github/workflows/scorecards.yml | 2 +- .github/workflows/static_checks_etc.yml | 4 +-- .github/workflows/unit_race.yml | 2 +- .github/workflows/unit_race_evalengine.yml | 2 +- .../unit_test_evalengine_mysql57.yml | 25 +++++++++----- .../unit_test_evalengine_mysql80.yml | 18 +++++----- .../unit_test_evalengine_mysql84.yml | 18 +++++----- .github/workflows/unit_test_mysql57.yml | 25 +++++++++----- .github/workflows/unit_test_mysql80.yml | 18 +++++----- .github/workflows/unit_test_mysql84.yml | 18 +++++----- .../workflows/update_golang_dependencies.yml | 2 +- .github/workflows/update_golang_version.yml | 2 +- .../upgrade_downgrade_test_backups_e2e.yml | 4 +-- ...owngrade_test_backups_e2e_next_release.yml | 4 +-- .../upgrade_downgrade_test_backups_manual.yml | 6 ++-- ...grade_test_backups_manual_next_release.yml | 6 ++-- .../upgrade_downgrade_test_onlineddl_flow.yml | 6 ++-- ...e_downgrade_test_query_serving_queries.yml | 6 ++-- ...est_query_serving_queries_next_release.yml | 6 ++-- ...de_downgrade_test_query_serving_schema.yml | 6 ++-- ...test_query_serving_schema_next_release.yml | 6 ++-- ...rade_downgrade_test_reparent_new_vtctl.yml | 6 ++-- ...e_downgrade_test_reparent_new_vttablet.yml | 6 ++-- ...rade_downgrade_test_reparent_old_vtctl.yml | 6 ++-- ...e_downgrade_test_reparent_old_vttablet.yml | 6 ++-- .../upgrade_downgrade_test_semi_sync.yml | 4 +-- .github/workflows/vitess_tester_vtgate.yml | 4 +-- .github/workflows/vtadmin_web_build.yml | 2 +- .github/workflows/vtadmin_web_lint.yml | 2 +- .github/workflows/vtadmin_web_unit_tests.yml | 2 +- test/templates/cluster_endtoend_test.tpl | 13 ++++++-- .../cluster_endtoend_test_docker.tpl | 2 +- .../cluster_endtoend_test_mysql57.tpl | 8 +++-- test/templates/cluster_vitess_tester.tpl | 4 +-- test/templates/unit_test.tpl | 33 +++++++++++-------- 111 files changed, 806 insertions(+), 304 deletions(-) diff --git a/.github/workflows/assign_milestone.yml b/.github/workflows/assign_milestone.yml index 3ae0a89b4a9..6687a239b57 100644 --- a/.github/workflows/assign_milestone.yml +++ b/.github/workflows/assign_milestone.yml @@ -12,7 +12,7 @@ env: jobs: build: name: Assign Milestone - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: pull-requests: write diff --git a/.github/workflows/auto_approve_pr.yml b/.github/workflows/auto_approve_pr.yml index de553694927..e584337f78b 100644 --- a/.github/workflows/auto_approve_pr.yml +++ b/.github/workflows/auto_approve_pr.yml @@ -9,7 +9,7 @@ permissions: jobs: auto_approve: name: Auto Approve Pull Request - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: pull-requests: write # only given on local PRs, forks run with `read` access diff --git a/.github/workflows/check_label.yml b/.github/workflows/check_label.yml index 15b3ae3ec15..2c6439a29f9 100644 --- a/.github/workflows/check_label.yml +++ b/.github/workflows/check_label.yml @@ -9,7 +9,7 @@ jobs: check_pull_request_labels: name: Check Pull Request labels timeout-minutes: 10 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.repository == 'vitessio/vitess' steps: - name: Release Notes label diff --git a/.github/workflows/check_make_vtadmin_authz_testgen.yml b/.github/workflows/check_make_vtadmin_authz_testgen.yml index f69df918be1..26543c8a948 100644 --- a/.github/workflows/check_make_vtadmin_authz_testgen.yml +++ b/.github/workflows/check_make_vtadmin_authz_testgen.yml @@ -6,7 +6,7 @@ permissions: read-all jobs: build: name: Check Make vtadmin_authz_testgen - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | @@ -61,7 +61,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' run: | sudo apt-get update - sudo apt-get install -y make unzip g++ etcd curl git wget + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget sudo service etcd stop go mod download go install golang.org/x/tools/cmd/goimports@latest diff --git a/.github/workflows/check_make_vtadmin_web_proto.yml b/.github/workflows/check_make_vtadmin_web_proto.yml index 0f8d404a256..1c1bf2cae4f 100644 --- a/.github/workflows/check_make_vtadmin_web_proto.yml +++ b/.github/workflows/check_make_vtadmin_web_proto.yml @@ -6,7 +6,7 @@ permissions: read-all jobs: build: name: Check Make VTAdmin Web Proto - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | diff --git a/.github/workflows/close_stale_pull_requests.yml b/.github/workflows/close_stale_pull_requests.yml index 17fef550aa3..dbb71ac79b9 100644 --- a/.github/workflows/close_stale_pull_requests.yml +++ b/.github/workflows/close_stale_pull_requests.yml @@ -9,7 +9,7 @@ permissions: read-all jobs: close_stale_pull_requests: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: pull-requests: write diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index 236e6e2835c..7c98855b783 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (12) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index 945d3448287..947d393bbbb 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (13) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index 0c57e651013..aedc1bbd691 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (15) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index 0c04f3e1b76..9886b617d51 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (18) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index 7d14a8b99ed..5d703099a3d 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (21) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_backup_pitr.yml b/.github/workflows/cluster_endtoend_backup_pitr.yml index 99047933d69..1f744e85004 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (backup_pitr) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index 8da7e054666..6ed73f260df 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (backup_pitr_mysqlshell) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml index 4012d1b1e4d..9bb6a4b56f8 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (backup_pitr_xtrabackup) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index 177fc77a5c4..e859749ceca 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (ers_prs_newfeatures_heavy) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index d0ff268c5fb..653db141b7d 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (mysql80) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_mysql_server_vault.yml b/.github/workflows/cluster_endtoend_mysql_server_vault.yml index 38cdcbadf95..9de4aacd499 100644 --- a/.github/workflows/cluster_endtoend_mysql_server_vault.yml +++ b/.github/workflows/cluster_endtoend_mysql_server_vault.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (mysql_server_vault) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index dd9990b4761..54c85944190 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (onlineddl_revert) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index 144e06915ce..9e117ac0f04 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (onlineddl_scheduler) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 5cd394f2d42..5056e46d895 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index 993a2d5193e..3597b37ede7 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index 015db29723e..a5322578981 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 550408f5def..12bde509048 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index 9c2cac74843..e452d28606a 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (schemadiff_vrepl) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -97,10 +97,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index 4846fdcc2ac..3e8ffce0ac5 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_consul) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 435d28a722a..cc75ef22937 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_tablegc) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml index 6cf480aa1f1..0837be5356e 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_throttler_topo) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index 056bfb6d8a4..da6ee53ad53 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (topo_connection_cache) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index aacf9d157fd..a38506ddd7d 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_across_db_versions) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index ead3b816df2..283d0451a34 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_basic) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index d9f1aeac45e..b316c075614 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_cellalias) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml index 9cdc4a9091a..48f506709e7 100644 --- a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml +++ b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_copy_parallel) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml index f95f70e8550..a1293e3688e 100644 --- a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml +++ b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_foreign_key_stress) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml index 2a6af9b91bd..2013d89a83d 100644 --- a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml +++ b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_mariadb_to_mysql) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index 6b6768eb676..e7a8a8cb5ce 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_migrate) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml index 78b6c616691..c0c80a8dc61 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_multi_tenant) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml index 30367611574..0b80b82fd7f 100644 --- a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml +++ b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_partial_movetables_and_materialize) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index 23871ada3b8..fcbe7057b4c 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_v2) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml index dd1eb76ca0e..40f2002f9a3 100644 --- a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml +++ b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vreplication_vtctldclient_vdiff2_movetables_tz) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vstream.yml b/.github/workflows/cluster_endtoend_vstream.yml index 3c99a687abc..acfefb40758 100644 --- a/.github/workflows/cluster_endtoend_vstream.yml +++ b/.github/workflows/cluster_endtoend_vstream.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vstream) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtbackup.yml b/.github/workflows/cluster_endtoend_vtbackup.yml index e47e8fbad47..947598bc307 100644 --- a/.github/workflows/cluster_endtoend_vtbackup.yml +++ b/.github/workflows/cluster_endtoend_vtbackup.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtbackup) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index 8f9a27bc750..c6f8712a35a 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtctlbackup_sharded_clustertest_heavy) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index db9ea451081..2cd81fe04ef 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_concurrentdml) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml index 2fada310f7e..19620d9e3a2 100644 --- a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml +++ b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_foreignkey_stress) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index 04813704cd2..06961af4ee7 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_gen4) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index bdb6b8b149d..e0b21df421a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_general_heavy) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index e23742a5650..23c9da550c2 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_godriver) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml index 4d082900790..f5d1850aca3 100644 --- a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml +++ b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_partial_keyspace) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index 46242a6b7f4..80462086a57 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_queries) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index b39aa3693e6..686db1e12ab 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_readafterwrite) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index 0032704b240..0f4ae20b0a7 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_reservedconn) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index be619a0815c..2dca36298e1 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_schema) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index a02b4ec5435..711f38253cb 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_schema_tracker) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index ff0c6990512..55996d7fa86 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_tablet_healthcheck_cache) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index e0f4397d56c..f47e0a8dd63 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index a10a9947218..1b62ac79041 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo_consul) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index 42e3b3d919c..f88d3b9af3a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo_etcd) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index bc400dde33f..9ae7b465695 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_transaction) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index 8c128e95b0f..d348606dc31 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_unsharded) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index e260443a242..16bdee60217 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_vindex_heavy) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index 72e9ac45949..fb176119769 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtgate_vschema) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 23544b28d01..92c2ea3d5d7 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vtorc) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -105,10 +105,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml index eaf6583a22d..9868c382a69 100644 --- a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml +++ b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (vttablet_prscomplex) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -96,10 +96,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index dae2eca922f..26dbfea2def 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (xb_backup) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index 0dad33aee7f..43360690d6b 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (xb_recovery) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI diff --git a/.github/workflows/code_freeze.yml b/.github/workflows/code_freeze.yml index 92ef037ee6b..a66fb6e8b2b 100644 --- a/.github/workflows/code_freeze.yml +++ b/.github/workflows/code_freeze.yml @@ -7,7 +7,7 @@ permissions: read-all jobs: build: name: Code Freeze - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Fail if Code Freeze is enabled run: | diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 7846749a013..fc30974212b 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -9,7 +9,7 @@ permissions: read-all jobs: test: name: Code Coverage - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Check out code diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index e1a474ce9a2..c523e34c697 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -14,7 +14,7 @@ permissions: read-all jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: read contents: read @@ -59,13 +59,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index a42bbc0ff2d..424cc5bfe15 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -12,7 +12,7 @@ permissions: read-all jobs: build: name: Create Release - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write steps: diff --git a/.github/workflows/docker_build_images.yml b/.github/workflows/docker_build_images.yml index 827946a43dc..2024d677205 100644 --- a/.github/workflows/docker_build_images.yml +++ b/.github/workflows/docker_build_images.yml @@ -16,7 +16,7 @@ permissions: read-all jobs: build_and_push_vttestserver: name: Build and push vttestserver - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 if: github.repository == 'vitessio/vitess' strategy: @@ -70,7 +70,7 @@ jobs: build_and_push_lite: name: Build and push lite - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.repository == 'vitessio/vitess' strategy: @@ -132,7 +132,7 @@ jobs: build_and_push_components: name: Build and push - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 if: github.repository == 'vitessio/vitess' && needs.build_and_push_lite.result == 'success' needs: - build_and_push_lite @@ -226,7 +226,7 @@ jobs: slack_notification: name: Slack Notification if failed - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - build_and_push_vttestserver - build_and_push_lite diff --git a/.github/workflows/docker_test_cluster_10.yml b/.github/workflows/docker_test_cluster_10.yml index ce1c9e7277b..10a7c1551c0 100644 --- a/.github/workflows/docker_test_cluster_10.yml +++ b/.github/workflows/docker_test_cluster_10.yml @@ -5,7 +5,7 @@ jobs: build: name: Docker Test Cluster 10 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI diff --git a/.github/workflows/docker_test_cluster_25.yml b/.github/workflows/docker_test_cluster_25.yml index eb2319b8c9f..34463ce3830 100644 --- a/.github/workflows/docker_test_cluster_25.yml +++ b/.github/workflows/docker_test_cluster_25.yml @@ -5,7 +5,7 @@ jobs: build: name: Docker Test Cluster 25 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 78c8d0b1753..50e1e4b9485 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -5,7 +5,7 @@ jobs: build: name: End-to-End Test (Race) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | @@ -65,13 +65,13 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml index 9e06e4ee44b..f061e8bee68 100644 --- a/.github/workflows/endtoend.yml +++ b/.github/workflows/endtoend.yml @@ -5,7 +5,7 @@ jobs: build: name: End-to-End Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | @@ -63,7 +63,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index 65669158415..fc9724aa6ac 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -4,8 +4,8 @@ permissions: read-all jobs: build: - name: Local example using ${{ matrix.topo }} on ubuntu-22.04 - runs-on: gh-hosted-runners-16cores-1 + name: Local example using ${{ matrix.topo }} on Ubuntu + runs-on: gh-hosted-runners-16cores-1-24.04 strategy: matrix: topo: [consul,etcd,zk2] @@ -67,24 +67,6 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' run: | - if [ ${{matrix.os}} = "ubuntu-22.04" ]; then - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - - # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop - sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - elif [ ${{matrix.os}} = "macos-latest" ]; then - brew install mysql@5.7 make unzip etcd curl git wget - fi go mod download - name: Run make minimaltools diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index c031ea315b5..c17f43eec11 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -4,8 +4,8 @@ permissions: read-all jobs: build: - name: Region Sharding example using ${{ matrix.topo }} on ubuntu-22.04 - runs-on: gh-hosted-runners-16cores-1 + name: Region Sharding example using ${{ matrix.topo }} on Ubuntu + runs-on: gh-hosted-runners-16cores-1-24.04 strategy: matrix: topo: [etcd] @@ -67,24 +67,6 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' run: | - if [ ${{matrix.os}} = "ubuntu-22.04" ]; then - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - - # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop - sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - elif [ ${{matrix.os}} = "macos-latest" ]; then - brew install mysql@5.7 make unzip etcd curl git wget - fi go mod download - name: Run make minimaltools diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 7d20694b850..b879ec8c144 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -14,7 +14,7 @@ permissions: read-all jobs: analysis: name: Scorecard analysis - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: # Needed to upload the results to code-scanning dashboard. security-events: write diff --git a/.github/workflows/static_checks_etc.yml b/.github/workflows/static_checks_etc.yml index cd896d5fd29..cd2cce93827 100644 --- a/.github/workflows/static_checks_etc.yml +++ b/.github/workflows/static_checks_etc.yml @@ -9,7 +9,7 @@ permissions: read-all jobs: build: name: Static Code Checks Etc - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -152,7 +152,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.parser_changes == 'true' || steps.changes.outputs.go_files == 'true') run: | sudo apt-get update - sudo apt-get install -y make unzip g++ etcd curl git wget + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget sudo service etcd stop go mod download diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml index e764cd33d9d..8ffb74bd2cb 100644 --- a/.github/workflows/unit_race.yml +++ b/.github/workflows/unit_race.yml @@ -15,7 +15,7 @@ jobs: build: name: Unit Test (Race) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI run: | diff --git a/.github/workflows/unit_race_evalengine.yml b/.github/workflows/unit_race_evalengine.yml index 5a5cb637681..d6d248f066e 100644 --- a/.github/workflows/unit_race_evalengine.yml +++ b/.github/workflows/unit_race_evalengine.yml @@ -15,7 +15,7 @@ jobs: build: name: Unit Test (Evalengine_Race) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI run: | diff --git a/.github/workflows/unit_test_evalengine_mysql57.yml b/.github/workflows/unit_test_evalengine_mysql57.yml index 09cb392c794..ecc366b38fe 100644 --- a/.github/workflows/unit_test_evalengine_mysql57.yml +++ b/.github/workflows/unit_test_evalengine_mysql57.yml @@ -16,7 +16,7 @@ env: jobs: test: name: Unit Test (evalengine_mysql57) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -89,30 +89,37 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 # packages for Jammy. echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 + sudo apt-get update + # We have to install this old version of libaio1. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/unit_test_evalengine_mysql80.yml b/.github/workflows/unit_test_evalengine_mysql80.yml index 5578829d48d..e6e802b52d8 100644 --- a/.github/workflows/unit_test_evalengine_mysql80.yml +++ b/.github/workflows/unit_test_evalengine_mysql80.yml @@ -16,7 +16,7 @@ env: jobs: test: name: Unit Test (evalengine_mysql80) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -89,27 +89,27 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/unit_test_evalengine_mysql84.yml b/.github/workflows/unit_test_evalengine_mysql84.yml index 2a86d5bfabd..46736dac349 100644 --- a/.github/workflows/unit_test_evalengine_mysql84.yml +++ b/.github/workflows/unit_test_evalengine_mysql84.yml @@ -16,7 +16,7 @@ env: jobs: test: name: Unit Test (evalengine_mysql84) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -89,27 +89,27 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index bd4b6ed8c3c..3eaf02d1538 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -16,7 +16,7 @@ env: jobs: test: name: Unit Test (mysql57) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -89,30 +89,37 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 # packages for Jammy. echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 + sudo apt-get update + # We have to install this old version of libaio1. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 03f31a25cc8..c036e6dd477 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -16,7 +16,7 @@ env: jobs: test: name: Unit Test (mysql80) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -89,27 +89,27 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/unit_test_mysql84.yml b/.github/workflows/unit_test_mysql84.yml index 95a56aeff81..84447ce390b 100644 --- a/.github/workflows/unit_test_mysql84.yml +++ b/.github/workflows/unit_test_mysql84.yml @@ -16,7 +16,7 @@ env: jobs: test: name: Unit Test (mysql84) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -89,27 +89,27 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/update_golang_dependencies.yml b/.github/workflows/update_golang_dependencies.yml index a614a634d83..33c5157097c 100644 --- a/.github/workflows/update_golang_dependencies.yml +++ b/.github/workflows/update_golang_dependencies.yml @@ -14,7 +14,7 @@ jobs: contents: write pull-requests: write name: Update Golang Dependencies - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Set up Go uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml index 503cad50ce8..09bbf451451 100644 --- a/.github/workflows/update_golang_version.yml +++ b/.github/workflows/update_golang_version.yml @@ -17,7 +17,7 @@ jobs: matrix: branch: [ main, release-21.0, release-20.0, release-19.0 ] name: Update Golang Version - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index f1c4f335f0d..a56aad2f523 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -13,7 +13,7 @@ jobs: upgrade_downgrade_test_e2e: timeout-minutes: 60 name: Run Upgrade Downgrade Test - Backups - E2E - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -87,7 +87,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index 73c24c5e72e..7ee6c2db799 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -14,7 +14,7 @@ jobs: upgrade_downgrade_test_e2e: timeout-minutes: 60 name: Run Upgrade Downgrade Test - Backups - E2E - Next Release - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -89,7 +89,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index 9c866affb0b..00aab1b78ff 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -15,7 +15,7 @@ jobs: upgrade_downgrade_test_manual: timeout-minutes: 40 name: Run Upgrade Downgrade Test - Backups - Manual - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -102,14 +102,14 @@ jobs: # Install MySQL 8.0 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml index 802ee78d940..768268271a1 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml @@ -15,7 +15,7 @@ jobs: upgrade_downgrade_test_manual: timeout-minutes: 40 name: Run Upgrade Downgrade Test - Backups - Manual - Next Release - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -103,14 +103,14 @@ jobs: # Install MySQL 8.0 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml index 89e5f7c4d25..72426e70a61 100644 --- a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml +++ b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Online DDL flow - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -108,13 +108,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml index f7d68dfee04..5ac1a55334c 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Query Serving (Queries) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -100,13 +100,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml index e3a34bd43f8..c65fff08c52 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Query Serving (Queries) Next Release - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -101,13 +101,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml index a9162d2f199..4b5fad0ab29 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Query Serving (Schema) - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -100,13 +100,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml index bf5f63d8836..6157079db6f 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Query Serving (Schema) Next Release - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -101,13 +101,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml index ac6615b5788..931d9174c31 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Reparent New Vtctl - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -101,13 +101,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml index e163ab35ea4..d77b7be53b3 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Reparent New VTTablet - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -101,13 +101,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml index 93e94466b74..701025d7ecc 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Reparent Old Vtctl - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -100,13 +100,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml index 12f6b514aa1..8b121d4af10 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -16,7 +16,7 @@ jobs: upgrade_downgrade_test: name: Run Upgrade Downgrade Test - Reparent Old VTTablet - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -100,13 +100,13 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml index 16dac6e7fd2..f12e323654e 100644 --- a/.github/workflows/upgrade_downgrade_test_semi_sync.yml +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -13,7 +13,7 @@ jobs: upgrade_downgrade_test_e2e: timeout-minutes: 60 name: Run Semi Sync Upgrade Downgrade Test - runs-on: gh-hosted-runners-16cores-1 + runs-on: gh-hosted-runners-16cores-1-24.04 steps: - name: Skip CI @@ -87,7 +87,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/vitess_tester_vtgate.yml b/.github/workflows/vitess_tester_vtgate.yml index 5f8c52d454b..7d5cf89a3ec 100644 --- a/.github/workflows/vitess_tester_vtgate.yml +++ b/.github/workflows/vitess_tester_vtgate.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Vitess Tester (vtgate) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -95,7 +95,7 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update diff --git a/.github/workflows/vtadmin_web_build.yml b/.github/workflows/vtadmin_web_build.yml index 096b70cd002..822fa8a0920 100644 --- a/.github/workflows/vtadmin_web_build.yml +++ b/.github/workflows/vtadmin_web_build.yml @@ -16,7 +16,7 @@ permissions: read-all jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | diff --git a/.github/workflows/vtadmin_web_lint.yml b/.github/workflows/vtadmin_web_lint.yml index 5435b4b54fe..694c24734a6 100644 --- a/.github/workflows/vtadmin_web_lint.yml +++ b/.github/workflows/vtadmin_web_lint.yml @@ -16,7 +16,7 @@ permissions: read-all jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | diff --git a/.github/workflows/vtadmin_web_unit_tests.yml b/.github/workflows/vtadmin_web_unit_tests.yml index b765a68fa76..72a887c1926 100644 --- a/.github/workflows/vtadmin_web_unit_tests.yml +++ b/.github/workflows/vtadmin_web_unit_tests.yml @@ -16,7 +16,7 @@ permissions: read-all jobs: unit-tests: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI run: | diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 01f4555e303..6fe58fae361 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on {{.Name}} - runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1{{else}}ubuntu-latest{{end}} + runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}} steps: - name: Skip CI @@ -124,10 +124,19 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + + # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + # Install everything else we need, and configure sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 diff --git a/test/templates/cluster_endtoend_test_docker.tpl b/test/templates/cluster_endtoend_test_docker.tpl index f68e4223c75..f7e8aa2c1d8 100644 --- a/test/templates/cluster_endtoend_test_docker.tpl +++ b/test/templates/cluster_endtoend_test_docker.tpl @@ -6,7 +6,7 @@ permissions: read-all jobs: build: name: Run endtoend tests on {{.Name}} - runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1{{else}}ubuntu-latest{{end}} + runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}} steps: - name: Skip CI diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl index b51ffc9110e..f4152c939b0 100644 --- a/test/templates/cluster_endtoend_test_mysql57.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -19,7 +19,7 @@ env: jobs: build: name: Run endtoend tests on {{.Name}} - runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1{{else}}ubuntu-latest{{end}} + runs-on: {{if .Cores16}}gh-hosted-runners-16cores-1-24.04{{else}}ubuntu-24.04{{end}} steps: - name: Skip CI @@ -126,13 +126,17 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 # packages for Jammy. echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + # We have to install this old version of libaio1. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata diff --git a/test/templates/cluster_vitess_tester.tpl b/test/templates/cluster_vitess_tester.tpl index f0b5838d8e8..b8d77754ba6 100644 --- a/test/templates/cluster_vitess_tester.tpl +++ b/test/templates/cluster_vitess_tester.tpl @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on {{.Name}} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -93,7 +93,7 @@ jobs: # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index c47b7a1d796..3704aebac4e 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -14,7 +14,7 @@ env: jobs: test: name: {{.Name}} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Skip CI @@ -87,20 +87,20 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get -qq update + sudo apt-get update # Uninstall any previously installed MySQL first sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -qq -y autoremove - sudo apt-get -qq -y autoclean + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean sudo deluser mysql sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb {{if (eq .Platform "mysql57")}} # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 @@ -108,25 +108,32 @@ jobs: echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 + sudo apt-get update + # We have to install this old version of libaio1. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 {{end}} {{if (eq .Platform "mysql80")}} echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client {{end}} {{if (eq .Platform "mysql84")}} echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - sudo DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y mysql-server mysql-client + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client {{end}} - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata sudo service mysql stop sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ From 0439d89f5b59550b0d4602aaeeee1844fb76e11c Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 25 Nov 2024 20:14:06 +0200 Subject: [PATCH 03/28] `Applyschema` uses `ExecuteMultiFetchAsDba` (#17078) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schemamanager/local_controller.go | 6 ++++-- go/vt/schemamanager/local_controller_test.go | 4 ++-- go/vt/schemamanager/schemamanager.go | 4 ++-- go/vt/schemamanager/schemamanager_test.go | 7 +++++++ go/vt/schemamanager/tablet_executor.go | 14 ++++++++------ go/vt/vtctl/grpcvtctldserver/server.go | 4 +++- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/go/vt/schemamanager/local_controller.go b/go/vt/schemamanager/local_controller.go index b95bb1398bf..92b7afa83c0 100644 --- a/go/vt/schemamanager/local_controller.go +++ b/go/vt/schemamanager/local_controller.go @@ -208,8 +208,10 @@ func (controller *LocalController) writeToLogDir(ctx context.Context, result *Ex rowsReturned := uint64(0) rowsAffected := uint64(0) for _, queryResult := range result.SuccessShards { - rowsReturned += uint64(len(queryResult.Result.Rows)) - rowsAffected += queryResult.Result.RowsAffected + for _, result := range queryResult.Results { + rowsReturned += uint64(len(result.Rows)) + rowsAffected += result.RowsAffected + } } logFile.WriteString(fmt.Sprintf("-- Rows returned: %d\n", rowsReturned)) logFile.WriteString(fmt.Sprintf("-- Rows affected: %d\n", rowsAffected)) diff --git a/go/vt/schemamanager/local_controller_test.go b/go/vt/schemamanager/local_controller_test.go index 1784a76e133..74d66033c63 100644 --- a/go/vt/schemamanager/local_controller_test.go +++ b/go/vt/schemamanager/local_controller_test.go @@ -138,8 +138,8 @@ func TestLocalControllerSchemaChange(t *testing.T) { result := &ExecuteResult{ Sqls: []string{"create table test_table (id int)"}, SuccessShards: []ShardResult{{ - Shard: "0", - Result: &querypb.QueryResult{}, + Shard: "0", + Results: []*querypb.QueryResult{{}}, }}, } logPath := path.Join(controller.logDir, controller.sqlFilename) diff --git a/go/vt/schemamanager/schemamanager.go b/go/vt/schemamanager/schemamanager.go index 1ccc52d6a6d..8a695a1164e 100644 --- a/go/vt/schemamanager/schemamanager.go +++ b/go/vt/schemamanager/schemamanager.go @@ -85,8 +85,8 @@ type ShardWithError struct { // ShardResult contains sql execute information on a particular shard type ShardResult struct { - Shard string - Result *querypb.QueryResult + Shard string + Results []*querypb.QueryResult // Position is a replication position that is guaranteed to be after the // schema change was applied. It can be used to wait for replicas to receive // the schema change via replication. diff --git a/go/vt/schemamanager/schemamanager_test.go b/go/vt/schemamanager/schemamanager_test.go index 129600d0527..76b638960fe 100644 --- a/go/vt/schemamanager/schemamanager_test.go +++ b/go/vt/schemamanager/schemamanager_test.go @@ -293,6 +293,13 @@ func (client *fakeTabletManagerClient) ExecuteFetchAsDba(ctx context.Context, ta return client.TabletManagerClient.ExecuteFetchAsDba(ctx, tablet, usePool, req) } +func (client *fakeTabletManagerClient) ExecuteMultiFetchAsDba(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, req *tabletmanagerdatapb.ExecuteMultiFetchAsDbaRequest) ([]*querypb.QueryResult, error) { + if client.EnableExecuteFetchAsDbaError { + return nil, fmt.Errorf("ExecuteMultiFetchAsDba occur an unknown error") + } + return client.TabletManagerClient.ExecuteMultiFetchAsDba(ctx, tablet, usePool, req) +} + // newFakeTopo returns a topo with: // - a keyspace named 'test_keyspace'. // - 3 shards named '1', '2', '3'. diff --git a/go/vt/schemamanager/tablet_executor.go b/go/vt/schemamanager/tablet_executor.go index 68270e0babc..bd521703723 100644 --- a/go/vt/schemamanager/tablet_executor.go +++ b/go/vt/schemamanager/tablet_executor.go @@ -572,13 +572,15 @@ func (exec *TabletExecutor) executeOneTablet( errChan chan ShardWithError, successChan chan ShardResult) { - var result *querypb.QueryResult + var results []*querypb.QueryResult var err error if viaQueryService { - result, err = exec.tmc.ExecuteQuery(ctx, tablet, &tabletmanagerdatapb.ExecuteQueryRequest{ + result, reserr := exec.tmc.ExecuteQuery(ctx, tablet, &tabletmanagerdatapb.ExecuteQueryRequest{ Query: []byte(sql), MaxRows: 10, }) + results = []*querypb.QueryResult{result} + err = reserr } else { if exec.ddlStrategySetting != nil && exec.ddlStrategySetting.IsAllowZeroInDateFlag() { // --allow-zero-in-date Applies to DDLs @@ -588,14 +590,14 @@ func (exec *TabletExecutor) executeOneTablet( return } } - request := &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{ - Query: []byte(sql), + request := &tabletmanagerdatapb.ExecuteMultiFetchAsDbaRequest{ + Sql: []byte(sql), MaxRows: 10, } if exec.ddlStrategySetting != nil && exec.ddlStrategySetting.IsAllowForeignKeysFlag() { request.DisableForeignKeyChecks = true } - result, err = exec.tmc.ExecuteFetchAsDba(ctx, tablet, false, request) + results, err = exec.tmc.ExecuteMultiFetchAsDba(ctx, tablet, false, request) } if err != nil { @@ -614,7 +616,7 @@ func (exec *TabletExecutor) executeOneTablet( } successChan <- ShardResult{ Shard: tablet.Shard, - Result: result, + Results: results, Position: pos, } } diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index e280a410e02..3bfce2204a2 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -303,7 +303,9 @@ func (s *VtctldServer) ApplySchema(ctx context.Context, req *vtctldatapb.ApplySc } for _, shard := range execResult.SuccessShards { - resp.RowsAffectedByShard[shard.Shard] = shard.Result.RowsAffected + for _, result := range shard.Results { + resp.RowsAffectedByShard[shard.Shard] += result.RowsAffected + } } return resp, err From ebf321ef957d52eb760ccecf8780ebda50c12045 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 25 Nov 2024 20:17:45 +0200 Subject: [PATCH 04/28] Online DDL: publish `vreplication_lag_seconds` from vreplication progress (#17263) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../schema/onlineddl/schema_migrations.sql | 1 + go/vt/vttablet/onlineddl/executor.go | 86 +++++++++---------- go/vt/vttablet/onlineddl/schema.go | 5 ++ go/vt/vttablet/onlineddl/vrepl.go | 14 +++ 4 files changed, 61 insertions(+), 45 deletions(-) diff --git a/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql b/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql index 91c0323e54b..3e2a72d6ae5 100644 --- a/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql +++ b/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql @@ -45,6 +45,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations `message_timestamp` timestamp(6) NULL DEFAULT NULL, `eta_seconds` bigint NOT NULL DEFAULT '-1', `rows_copied` bigint unsigned NOT NULL DEFAULT '0', + `vreplication_lag_seconds` bigint unsigned NOT NULL DEFAULT '0', `table_rows` bigint NOT NULL DEFAULT '0', `added_unique_keys` int unsigned NOT NULL DEFAULT '0', `removed_unique_keys` int unsigned NOT NULL DEFAULT '0', diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index 002cd1fb6d0..f8b5cfd9b8d 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -3583,53 +3583,36 @@ func (e *Executor) isPreserveForeignKeySupported(ctx context.Context) (isSupport // and is up to date with the binlogs. func (e *Executor) isVReplMigrationReadyToCutOver(ctx context.Context, onlineDDL *schema.OnlineDDL, s *VReplStream) (isReady bool, err error) { // Check all the cases where migration is still running: - { - // when ready to cut-over, pos must have some value - if s.pos == "" { - return false, nil - } + // when ready to cut-over, pos must have some value + if s.pos == "" { + return false, nil } - { - // Both time_updated and transaction_timestamp must be in close proximity to each - // other and to the time now, otherwise that means we're lagging and it's not a good time - // to cut-over - durationDiff := func(t1, t2 time.Time) time.Duration { - return t1.Sub(t2).Abs() - } - timeNow := time.Now() - timeUpdated := time.Unix(s.timeUpdated, 0) - if durationDiff(timeNow, timeUpdated) > onlineDDL.CutOverThreshold { - return false, nil - } - // Let's look at transaction timestamp. This gets written by any ongoing - // writes on the server (whether on this table or any other table) - transactionTimestamp := time.Unix(s.transactionTimestamp, 0) - if durationDiff(timeNow, transactionTimestamp) > onlineDDL.CutOverThreshold { - return false, nil - } + // Both time_updated and transaction_timestamp must be in close proximity to each + // other and to the time now, otherwise that means we're lagging and it's not a good time + // to cut-over + if s.Lag() > onlineDDL.CutOverThreshold { + return false, nil } - { - // copy_state must have no entries for this vreplication id: if entries are - // present that means copy is still in progress - query, err := sqlparser.ParseAndBind(sqlReadCountCopyState, - sqltypes.Int32BindVariable(s.id), - ) - if err != nil { - return false, err - } - r, err := e.execQuery(ctx, query) - if err != nil { - return false, err - } - csRow := r.Named().Row() - if csRow == nil { - return false, err - } - count := csRow.AsInt64("cnt", 0) - if count > 0 { - // Still copying - return false, nil - } + // copy_state must have no entries for this vreplication id: if entries are + // present that means copy is still in progress + query, err := sqlparser.ParseAndBind(sqlReadCountCopyState, + sqltypes.Int32BindVariable(s.id), + ) + if err != nil { + return false, err + } + r, err := e.execQuery(ctx, query) + if err != nil { + return false, err + } + csRow := r.Named().Row() + if csRow == nil { + return false, err + } + count := csRow.AsInt64("cnt", 0) + if count > 0 { + // Still copying + return false, nil } return true, nil @@ -3776,6 +3759,7 @@ func (e *Executor) reviewRunningMigrations(ctx context.Context) (countRunnning i } _ = e.updateRowsCopied(ctx, uuid, s.rowsCopied) _ = e.updateMigrationProgressByRowsCopied(ctx, uuid, s.rowsCopied) + _ = e.updateMigrationVreplicationLagSeconds(ctx, uuid, int64(s.Lag().Seconds())) _ = e.updateMigrationETASecondsByProgress(ctx, uuid) if s.timeThrottled != 0 { // Avoid creating a 0000-00-00 00:00:00 timestamp @@ -4534,6 +4518,18 @@ func (e *Executor) updateRowsCopied(ctx context.Context, uuid string, rowsCopied return err } +func (e *Executor) updateMigrationVreplicationLagSeconds(ctx context.Context, uuid string, vreplicationLagSeconds int64) error { + query, err := sqlparser.ParseAndBind(sqlUpdateMigrationVreplicationLagSeconds, + sqltypes.Int64BindVariable(vreplicationLagSeconds), + sqltypes.StringBindVariable(uuid), + ) + if err != nil { + return err + } + _, err = e.execQuery(ctx, query) + return err +} + func (e *Executor) updateVitessLivenessIndicator(ctx context.Context, uuid string, livenessIndicator int64) error { query, err := sqlparser.ParseAndBind(sqlUpdateMigrationVitessLivenessIndicator, sqltypes.Int64BindVariable(livenessIndicator), diff --git a/go/vt/vttablet/onlineddl/schema.go b/go/vt/vttablet/onlineddl/schema.go index bc3fa4f2bc9..943a3b1df07 100644 --- a/go/vt/vttablet/onlineddl/schema.go +++ b/go/vt/vttablet/onlineddl/schema.go @@ -87,6 +87,11 @@ const ( WHERE migration_uuid=%a ` + sqlUpdateMigrationVreplicationLagSeconds = `UPDATE _vt.schema_migrations + SET vreplication_lag_seconds=%a + WHERE + migration_uuid=%a + ` sqlUpdateMigrationIsView = `UPDATE _vt.schema_migrations SET is_view=%a WHERE diff --git a/go/vt/vttablet/onlineddl/vrepl.go b/go/vt/vttablet/onlineddl/vrepl.go index 26eb614e95a..2761c27c801 100644 --- a/go/vt/vttablet/onlineddl/vrepl.go +++ b/go/vt/vttablet/onlineddl/vrepl.go @@ -30,6 +30,7 @@ import ( "net/url" "strconv" "strings" + "time" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" @@ -96,6 +97,19 @@ func (v *VReplStream) hasError() (isTerminal bool, vreplError error) { return false, nil } +// Lag returns the vreplication lag, as determined by the higher of the transaction timestamp and the time updated. +func (s *VReplStream) Lag() time.Duration { + durationDiff := func(t1, t2 time.Time) time.Duration { + return t1.Sub(t2).Abs() + } + timeNow := time.Now() + timeUpdated := time.Unix(s.timeUpdated, 0) + // Let's look at transaction timestamp. This gets written by any ongoing + // writes on the server (whether on this table or any other table) + transactionTimestamp := time.Unix(s.transactionTimestamp, 0) + return max(durationDiff(timeNow, timeUpdated), durationDiff(timeNow, transactionTimestamp)) +} + // VRepl is an online DDL helper for VReplication based migrations (ddl_strategy="online") type VRepl struct { workflow string From 96f4ac7562c4ae93319a28bda7a73bdf9041259a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 25 Nov 2024 20:23:49 +0200 Subject: [PATCH 05/28] A couple `endtoend` cluster tests enhancement (#17247) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/cluster/vtctldclient_process.go | 5 ++++- go/test/endtoend/cluster/vttablet_process.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/cluster/vtctldclient_process.go b/go/test/endtoend/cluster/vtctldclient_process.go index 892f03e27fe..8b401f59070 100644 --- a/go/test/endtoend/cluster/vtctldclient_process.go +++ b/go/test/endtoend/cluster/vtctldclient_process.go @@ -41,6 +41,7 @@ type VtctldClientProcess struct { TempDirectory string ZoneName string VtctldClientMajorVersion int + Quiet bool } // ExecuteCommand executes any vtctldclient command @@ -72,7 +73,9 @@ func (vtctldclient *VtctldClientProcess) ExecuteCommandWithOutput(args ...string filterDoubleDashArgs(pArgs, vtctldclient.VtctldClientMajorVersion)..., ) msg := binlogplayer.LimitString(strings.Join(tmpProcess.Args, " "), 256) // limit log line length - log.Infof("Executing vtctldclient with command: %v (attempt %d of %d)", msg, i+1, retries) + if !vtctldclient.Quiet { + log.Infof("Executing vtctldclient with command: %v (attempt %d of %d)", msg, i+1, retries) + } resultByte, err = tmpProcess.CombinedOutput() resultStr = string(resultByte) if err == nil || !shouldRetry(resultStr) { diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 6c7a85ec533..6bc6e6b8d7f 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -459,6 +459,16 @@ func (vttablet *VttabletProcess) QueryTablet(query string, keyspace string, useD return executeQuery(conn, query) } +// MultiQueryTablet lets you execute a query in this tablet and get the result +func (vttablet *VttabletProcess) MultiQueryTablet(sql string, keyspace string, useDb bool) error { + conn, err := vttablet.TabletConn(keyspace, useDb) + if err != nil { + return err + } + defer conn.Close() + return executeMultiQuery(conn, sql) +} + // SemiSyncExtensionLoaded returns what type of semi-sync extension is loaded func (vttablet *VttabletProcess) SemiSyncExtensionLoaded() (mysql.SemiSyncType, error) { conn, err := vttablet.TabletConn("", false) From 69b381fe185b28bf6ed68eba63821f169d64559c Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 26 Nov 2024 12:39:05 +0100 Subject: [PATCH 06/28] `vtorc`: make SQL formatting consistent (#17154) Signed-off-by: Tim Vaillancourt Signed-off-by: Manan Gupta Co-authored-by: Manan Gupta --- go/vt/vtorc/db/db.go | 18 +- go/vt/vtorc/inst/analysis_dao.go | 62 ++--- go/vt/vtorc/inst/audit_dao.go | 24 +- go/vt/vtorc/inst/instance_dao.go | 276 +++++++++++---------- go/vt/vtorc/inst/instance_dao_test.go | 46 ++-- go/vt/vtorc/inst/tablet_dao.go | 44 ++-- go/vt/vtorc/logic/disable_recovery.go | 33 ++- go/vt/vtorc/logic/tablet_discovery.go | 2 +- go/vt/vtorc/logic/topology_recovery_dao.go | 157 ++++++------ go/vt/vtorc/process/health.go | 9 +- 10 files changed, 358 insertions(+), 313 deletions(-) diff --git a/go/vt/vtorc/db/db.go b/go/vt/vtorc/db/db.go index 64143477645..4bb22b65b5a 100644 --- a/go/vt/vtorc/db/db.go +++ b/go/vt/vtorc/db/db.go @@ -58,13 +58,13 @@ func OpenVTOrc() (db *sql.DB, err error) { // registerVTOrcDeployment updates the vtorc_db_deployments table upon successful deployment func registerVTOrcDeployment(db *sql.DB) error { - query := ` - replace into vtorc_db_deployments ( - deployed_version, deployed_timestamp - ) values ( - ?, datetime('now') - ) - ` + query := `REPLACE INTO vtorc_db_deployments ( + deployed_version, + deployed_timestamp + ) VALUES ( + ?, + DATETIME('now') + )` if _, err := execInternal(db, query, ""); err != nil { log.Fatalf("Unable to write to vtorc_db_deployments: %+v", err) } @@ -116,9 +116,7 @@ func initVTOrcDB(db *sql.DB) error { // execInternal func execInternal(db *sql.DB, query string, args ...any) (sql.Result, error) { - var err error - res, err := sqlutils.ExecNoPrepare(db, query, args...) - return res, err + return sqlutils.ExecNoPrepare(db, query, args...) } // ExecVTOrc will execute given query on the vtorc backend database. diff --git a/go/vt/vtorc/inst/analysis_dao.go b/go/vt/vtorc/inst/analysis_dao.go index 25d93a6864b..e44538e694c 100644 --- a/go/vt/vtorc/inst/analysis_dao.go +++ b/go/vt/vtorc/inst/analysis_dao.go @@ -69,8 +69,7 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna // TODO(sougou); deprecate ReduceReplicationAnalysisCount args := sqlutils.Args(config.Config.ReasonableReplicationLagSeconds, ValidSecondsFromSeenToLastAttemptedCheck(), config.Config.ReasonableReplicationLagSeconds, keyspace, shard) - query := ` - SELECT + query := `SELECT vitess_tablet.info AS tablet_info, vitess_tablet.tablet_type, vitess_tablet.primary_timestamp, @@ -91,13 +90,13 @@ func GetReplicationAnalysis(keyspace string, shard string, hints *ReplicationAna IFNULL( primary_instance.binary_log_file = database_instance_stale_binlog_coordinates.binary_log_file AND primary_instance.binary_log_pos = database_instance_stale_binlog_coordinates.binary_log_pos - AND database_instance_stale_binlog_coordinates.first_seen < datetime('now', printf('-%d second', ?)), + AND database_instance_stale_binlog_coordinates.first_seen < DATETIME('now', PRINTF('-%d SECOND', ?)), 0 ) ) AS is_stale_binlog_coordinates, MIN( primary_instance.last_checked <= primary_instance.last_seen - and primary_instance.last_attempted_check <= datetime(primary_instance.last_seen, printf('+%d second', ?)) + and primary_instance.last_attempted_check <= DATETIME(primary_instance.last_seen, PRINTF('+%d SECOND', ?)) ) = 1 AS is_last_check_valid, /* To be considered a primary, traditional async replication must not be present/valid AND the host should either */ /* not be a replication group member OR be the primary of the replication group */ @@ -655,13 +654,13 @@ func auditInstanceAnalysisInChangelog(tabletAlias string, analysisCode AnalysisC // Find if the lastAnalysisHasChanged or not while updating the row if it has. lastAnalysisChanged := false { - sqlResult, err := db.ExecVTOrc(` - update database_instance_last_analysis set + sqlResult, err := db.ExecVTOrc(`UPDATE database_instance_last_analysis + SET analysis = ?, - analysis_timestamp = datetime('now') - where + analysis_timestamp = DATETIME('now') + WHERE alias = ? - and analysis != ? + AND analysis != ? `, string(analysisCode), tabletAlias, string(analysisCode), ) @@ -682,13 +681,16 @@ func auditInstanceAnalysisInChangelog(tabletAlias string, analysisCode AnalysisC firstInsertion := false if !lastAnalysisChanged { // The insert only returns more than 1 row changed if this is the first insertion. - sqlResult, err := db.ExecVTOrc(` - insert or ignore into database_instance_last_analysis ( - alias, analysis_timestamp, analysis - ) values ( - ?, datetime('now'), ? - ) - `, + sqlResult, err := db.ExecVTOrc(`INSERT OR IGNORE + INTO database_instance_last_analysis ( + alias, + analysis_timestamp, + analysis + ) VALUES ( + ?, + DATETIME('now'), + ? + )`, tabletAlias, string(analysisCode), ) if err != nil { @@ -708,13 +710,16 @@ func auditInstanceAnalysisInChangelog(tabletAlias string, analysisCode AnalysisC return nil } - _, err := db.ExecVTOrc(` - insert into database_instance_analysis_changelog ( - alias, analysis_timestamp, analysis - ) values ( - ?, datetime('now'), ? - ) - `, + _, err := db.ExecVTOrc(`INSERT + INTO database_instance_analysis_changelog ( + alias, + analysis_timestamp, + analysis + ) VALUES ( + ?, + DATETIME('now'), + ? + )`, tabletAlias, string(analysisCode), ) if err == nil { @@ -727,12 +732,11 @@ func auditInstanceAnalysisInChangelog(tabletAlias string, analysisCode AnalysisC // ExpireInstanceAnalysisChangelog removes old-enough analysis entries from the changelog func ExpireInstanceAnalysisChangelog() error { - _, err := db.ExecVTOrc(` - delete - from database_instance_analysis_changelog - where - analysis_timestamp < datetime('now', printf('-%d hour', ?)) - `, + _, err := db.ExecVTOrc(`DELETE + FROM database_instance_analysis_changelog + WHERE + analysis_timestamp < DATETIME('now', PRINTF('-%d HOUR', ?)) + `, config.UnseenInstanceForgetHours, ) if err != nil { diff --git a/go/vt/vtorc/inst/audit_dao.go b/go/vt/vtorc/inst/audit_dao.go index 642fb187509..cbfd771e81c 100644 --- a/go/vt/vtorc/inst/audit_dao.go +++ b/go/vt/vtorc/inst/audit_dao.go @@ -55,14 +55,22 @@ func AuditOperation(auditType string, tabletAlias string, message string) error }() } if config.Config.AuditToBackendDB { - _, err := db.ExecVTOrc(` - insert - into audit ( - audit_timestamp, audit_type, alias, keyspace, shard, message - ) VALUES ( - datetime('now'), ?, ?, ?, ?, ? - ) - `, + _, err := db.ExecVTOrc(`INSERT + INTO audit ( + audit_timestamp, + audit_type, + alias, + keyspace, + shard, + message + ) VALUES ( + DATETIME('now'), + ?, + ?, + ?, + ?, + ? + )`, auditType, tabletAlias, keyspace, diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index bd437d1ddac..c8ff218710f 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -114,10 +114,15 @@ func ExecDBWriteFunc(f func() error) error { func ExpireTableData(tableName string, timestampColumn string) error { writeFunc := func() error { - _, err := db.ExecVTOrc( - fmt.Sprintf("delete from %s where %s < datetime('now', printf('-%%d DAY', ?))", tableName, timestampColumn), - config.Config.AuditPurgeDays, + query := fmt.Sprintf(`DELETE + FROM %s + WHERE + %s < DATETIME('now', PRINTF('-%%d DAY', ?)) + `, + tableName, + timestampColumn, ) + _, err := db.ExecVTOrc(query, config.Config.AuditPurgeDays) return err } return ExecDBWriteFunc(writeFunc) @@ -463,16 +468,16 @@ func ReadInstanceClusterAttributes(instance *Instance) (err error) { var primaryExecutedGtidSet string primaryDataFound := false - query := ` - select - replication_depth, - source_host, - source_port, - ancestry_uuid, - executed_gtid_set - from database_instance - where hostname=? and port=? - ` + query := `SELECT + replication_depth, + source_host, + source_port, + ancestry_uuid, + executed_gtid_set + FROM database_instance + WHERE + hostname = ? + AND port = ?` primaryHostname := instance.SourceHost primaryPort := instance.SourcePort args := sqlutils.Args(primaryHostname, primaryPort) @@ -604,20 +609,22 @@ func readInstancesByCondition(condition string, args []any, sort string) ([](*In if sort == "" { sort = `alias` } - query := fmt.Sprintf(` - select - *, - strftime('%%s', 'now') - strftime('%%s', last_checked) as seconds_since_last_checked, - ifnull(last_checked <= last_seen, 0) as is_last_check_valid, - strftime('%%s', 'now') - strftime('%%s', last_seen) as seconds_since_last_seen - from - vitess_tablet - left join database_instance using (alias, hostname, port) - where - %s - order by - %s - `, condition, sort) + query := fmt.Sprintf(`SELECT + *, + STRFTIME('%%s', 'now') - STRFTIME('%%s', last_checked) AS seconds_since_last_checked, + IFNULL(last_checked <= last_seen, 0) AS is_last_check_valid, + STRFTIME('%%s', 'now') - STRFTIME('%%s', last_seen) AS seconds_since_last_seen + FROM + vitess_tablet + LEFT JOIN database_instance USING (alias, hostname, port) + WHERE + %s + ORDER BY + %s + `, + condition, + sort, + ) err := db.QueryVTOrc(query, args, func(m sqlutils.RowMap) error { instance := readInstanceRow(m) @@ -638,9 +645,7 @@ func readInstancesByCondition(condition string, args []any, sort string) ([](*In // ReadInstance reads an instance from the vtorc backend database func ReadInstance(tabletAlias string) (*Instance, bool, error) { - condition := ` - alias = ? - ` + condition := `alias = ?` instances, err := readInstancesByCondition(condition, sqlutils.Args(tabletAlias), "") // We know there will be at most one (alias is the PK). // And we expect to find one. @@ -657,18 +662,17 @@ func ReadInstance(tabletAlias string) (*Instance, bool, error) { // ReadProblemInstances reads all instances with problems func ReadProblemInstances(keyspace string, shard string) ([](*Instance), error) { condition := ` - keyspace LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) - and shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) - and ( - (last_seen < last_checked) - or (strftime('%%s', 'now') - strftime('%%s', last_checked) > ?) - or (replication_sql_thread_state not in (-1 ,1)) - or (replication_io_thread_state not in (-1 ,1)) - or (abs(cast(replication_lag_seconds as integer) - cast(sql_delay as integer)) > ?) - or (abs(cast(replica_lag_seconds as integer) - cast(sql_delay as integer)) > ?) - or (gtid_errant != '') - ) - ` + keyspace LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + AND shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + AND ( + (last_seen < last_checked) + OR (STRFTIME('%%s', 'now') - STRFTIME('%%s', last_checked) > ?) + OR (replication_sql_thread_state NOT IN (-1 ,1)) + OR (replication_io_thread_state NOT IN (-1 ,1)) + OR (ABS(CAST(replication_lag_seconds AS integer) - CAST(sql_delay AS integer)) > ?) + OR (ABS(CAST(replica_lag_seconds AS integer) - CAST(sql_delay AS integer)) > ?) + OR (gtid_errant != '') + )` args := sqlutils.Args(keyspace, keyspace, shard, shard, config.Config.InstancePollSeconds*5, config.Config.ReasonableReplicationLagSeconds, config.Config.ReasonableReplicationLagSeconds) return readInstancesByCondition(condition, args, "") @@ -677,10 +681,9 @@ func ReadProblemInstances(keyspace string, shard string) ([](*Instance), error) // ReadInstancesWithErrantGTIds reads all instances with errant GTIDs func ReadInstancesWithErrantGTIds(keyspace string, shard string) ([]*Instance, error) { condition := ` - keyspace LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) - and shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) - and gtid_errant != '' - ` + keyspace LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + AND shard LIKE (CASE WHEN ? = '' THEN '%' ELSE ? END) + AND gtid_errant != ''` args := sqlutils.Args(keyspace, keyspace, shard, shard) return readInstancesByCondition(condition, args, "") @@ -688,15 +691,14 @@ func ReadInstancesWithErrantGTIds(keyspace string, shard string) ([]*Instance, e // GetKeyspaceShardName gets the keyspace shard name for the given instance key func GetKeyspaceShardName(tabletAlias string) (keyspace string, shard string, err error) { - query := ` - select - keyspace, - shard - from - vitess_tablet - where - alias = ? - ` + query := `SELECT + keyspace, + shard + FROM + vitess_tablet + WHERE + alias = ? + ` err = db.QueryVTOrc(query, sqlutils.Args(tabletAlias), func(m sqlutils.RowMap) error { keyspace = m.GetString("keyspace") shard = m.GetString("shard") @@ -719,27 +721,26 @@ func GetKeyspaceShardName(tabletAlias string) (keyspace string, shard string, er // the instance. func ReadOutdatedInstanceKeys() ([]string, error) { var res []string - query := ` - SELECT - alias - FROM - database_instance - WHERE - CASE - WHEN last_attempted_check <= last_checked - THEN last_checked < datetime('now', printf('-%d second', ?)) - ELSE last_checked < datetime('now', printf('-%d second', ?)) - END - UNION - SELECT - vitess_tablet.alias - FROM - vitess_tablet LEFT JOIN database_instance ON ( - vitess_tablet.alias = database_instance.alias - ) - WHERE - database_instance.alias IS NULL - ` + query := `SELECT + alias + FROM + database_instance + WHERE + CASE + WHEN last_attempted_check <= last_checked + THEN last_checked < DATETIME('now', PRINTF('-%d SECOND', ?)) + ELSE last_checked < DATETIME('now', PRINTF('-%d SECOND', ?)) + END + UNION + SELECT + vitess_tablet.alias + FROM + vitess_tablet LEFT JOIN database_instance ON ( + vitess_tablet.alias = database_instance.alias + ) + WHERE + database_instance.alias IS NULL + ` args := sqlutils.Args(config.Config.InstancePollSeconds, 2*config.Config.InstancePollSeconds) err := db.QueryVTOrc(query, args, func(m sqlutils.RowMap) error { @@ -782,12 +783,17 @@ func mkInsert(table string, columns []string, values []string, nrRows int, inser } col := strings.Join(columns, ", ") - q.WriteString(fmt.Sprintf(`%s %s - (%s) - VALUES - %s - `, - insertStr, table, col, val.String())) + query := fmt.Sprintf(`%s %s + (%s) + VALUES + %s + `, + insertStr, + table, + col, + val.String(), + ) + q.WriteString(query) return q.String(), nil } @@ -873,13 +879,13 @@ func mkInsertForInstances(instances []*Instance, instanceWasActuallyFound bool, for i := range columns { values[i] = "?" } - values[3] = "datetime('now')" // last_checked - values[4] = "datetime('now')" // last_attempted_check + values[3] = "DATETIME('now')" // last_checked + values[4] = "DATETIME('now')" // last_attempted_check values[5] = "1" // last_check_partial_success if updateLastSeen { columns = append(columns, "last_seen") - values = append(values, "datetime('now')") + values = append(values, "DATETIME('now')") } var args []any @@ -995,14 +1001,13 @@ func WriteInstance(instance *Instance, instanceWasActuallyFound bool, lastError // for a given instance func UpdateInstanceLastChecked(tabletAlias string, partialSuccess bool) error { writeFunc := func() error { - _, err := db.ExecVTOrc(` - update - database_instance - set - last_checked = datetime('now'), - last_check_partial_success = ? - where - alias = ?`, + _, err := db.ExecVTOrc(`UPDATE database_instance + SET + last_checked = DATETIME('now'), + last_check_partial_success = ? + WHERE + alias = ? + `, partialSuccess, tabletAlias, ) @@ -1024,13 +1029,12 @@ func UpdateInstanceLastChecked(tabletAlias string, partialSuccess bool) error { // we have a "hanging" issue. func UpdateInstanceLastAttemptedCheck(tabletAlias string) error { writeFunc := func() error { - _, err := db.ExecVTOrc(` - update - database_instance - set - last_attempted_check = datetime('now') - where - alias = ?`, + _, err := db.ExecVTOrc(`UPDATE database_instance + SET + last_attempted_check = DATETIME('now') + WHERE + alias = ? + `, tabletAlias, ) if err != nil { @@ -1061,11 +1065,11 @@ func ForgetInstance(tabletAlias string) error { currentErrantGTIDCount.Reset(tabletAlias) // Delete from the 'vitess_tablet' table. - _, err := db.ExecVTOrc(` - delete - from vitess_tablet - where - alias = ?`, + _, err := db.ExecVTOrc(`DELETE + FROM vitess_tablet + WHERE + alias = ? + `, tabletAlias, ) if err != nil { @@ -1074,11 +1078,11 @@ func ForgetInstance(tabletAlias string) error { } // Also delete from the 'database_instance' table. - sqlResult, err := db.ExecVTOrc(` - delete - from database_instance - where - alias = ?`, + sqlResult, err := db.ExecVTOrc(`DELETE + FROM database_instance + WHERE + alias = ? + `, tabletAlias, ) if err != nil { @@ -1102,11 +1106,11 @@ func ForgetInstance(tabletAlias string) error { // ForgetLongUnseenInstances will remove entries of all instances that have long since been last seen. func ForgetLongUnseenInstances() error { - sqlResult, err := db.ExecVTOrc(` - delete - from database_instance - where - last_seen < datetime('now', printf('-%d hour', ?))`, + sqlResult, err := db.ExecVTOrc(`DELETE + FROM database_instance + WHERE + last_seen < DATETIME('now', PRINTF('-%d HOUR', ?)) + `, config.UnseenInstanceForgetHours, ) if err != nil { @@ -1127,18 +1131,26 @@ func ForgetLongUnseenInstances() error { // SnapshotTopologies records topology graph for all existing topologies func SnapshotTopologies() error { writeFunc := func() error { - _, err := db.ExecVTOrc(` - insert or ignore into - database_instance_topology_history (snapshot_unix_timestamp, - alias, hostname, port, source_host, source_port, keyspace, shard, version) - select - strftime('%s', 'now'), - vitess_tablet.alias, vitess_tablet.hostname, vitess_tablet.port, - database_instance.source_host, database_instance.source_port, + _, err := db.ExecVTOrc(`INSERT OR IGNORE + INTO database_instance_topology_history ( + snapshot_unix_timestamp, + alias, + hostname, + port, + source_host, + source_port, + keyspace, + shard, + version + ) + SELECT + STRFTIME('%s', 'now'), + vitess_tablet.alias, vitess_tablet.hostname, vitess_tablet.port, + database_instance.source_host, database_instance.source_port, vitess_tablet.keyspace, vitess_tablet.shard, database_instance.version - from - vitess_tablet left join database_instance using (alias, hostname, port) - `, + FROM + vitess_tablet LEFT JOIN database_instance USING (alias, hostname, port) + `, ) if err != nil { log.Error(err) @@ -1156,10 +1168,12 @@ func ExpireStaleInstanceBinlogCoordinates() error { expireSeconds = config.StaleInstanceCoordinatesExpireSeconds } writeFunc := func() error { - _, err := db.ExecVTOrc(` - delete from database_instance_stale_binlog_coordinates - where first_seen < datetime('now', printf('-%d second', ?)) - `, expireSeconds, + _, err := db.ExecVTOrc(`DELETE + FROM database_instance_stale_binlog_coordinates + WHERE + first_seen < DATETIME('now', PRINTF('-%d SECOND', ?)) + `, + expireSeconds, ) if err != nil { log.Error(err) @@ -1181,7 +1195,7 @@ func GetDatabaseState() (string, error) { ts := tableState{ TableName: tableName, } - err := db.QueryVTOrc("select * from "+tableName, nil, func(rowMap sqlutils.RowMap) error { + err := db.QueryVTOrc("SELECT * FROM "+tableName, nil, func(rowMap sqlutils.RowMap) error { ts.Rows = append(ts.Rows, rowMap) return nil }) diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index cb2064d10c0..f248ded5e2b 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -66,7 +66,7 @@ func TestMkInsertSingle(t *testing.T) { replica_sql_running, replica_io_running, replication_sql_thread_state, replication_io_thread_state, has_replication_filters, supports_oracle_gtid, oracle_gtid, source_uuid, ancestry_uuid, executed_gtid_set, gtid_mode, gtid_purged, gtid_errant, mariadb_gtid, pseudo_gtid, source_log_file, read_source_log_pos, relay_source_log_file, exec_source_log_pos, relay_log_file, relay_log_pos, last_sql_error, last_io_error, replication_lag_seconds, replica_lag_seconds, sql_delay, data_center, region, physical_environment, replication_depth, is_co_primary, has_replication_credentials, allow_tls, semi_sync_enforced, semi_sync_primary_enabled, semi_sync_primary_timeout, semi_sync_primary_wait_for_replica_count, semi_sync_replica_enabled, semi_sync_primary_status, semi_sync_primary_clients, semi_sync_replica_status, last_discovery_latency, last_seen) VALUES - (?, ?, ?, datetime('now'), datetime('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now')) + (?, ?, ?, DATETIME('now'), DATETIME('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, DATETIME('now')) ` a1 := `zone1-i710, i710, 3306, 710, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, 0, 0, @@ -89,9 +89,9 @@ func TestMkInsertThree(t *testing.T) { replica_sql_running, replica_io_running, replication_sql_thread_state, replication_io_thread_state, has_replication_filters, supports_oracle_gtid, oracle_gtid, source_uuid, ancestry_uuid, executed_gtid_set, gtid_mode, gtid_purged, gtid_errant, mariadb_gtid, pseudo_gtid, source_log_file, read_source_log_pos, relay_source_log_file, exec_source_log_pos, relay_log_file, relay_log_pos, last_sql_error, last_io_error, replication_lag_seconds, replica_lag_seconds, sql_delay, data_center, region, physical_environment, replication_depth, is_co_primary, has_replication_credentials, allow_tls, semi_sync_enforced, semi_sync_primary_enabled, semi_sync_primary_timeout, semi_sync_primary_wait_for_replica_count, semi_sync_replica_enabled, semi_sync_primary_status, semi_sync_primary_clients, semi_sync_replica_status, last_discovery_latency, last_seen) VALUES - (?, ?, ?, datetime('now'), datetime('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now')), - (?, ?, ?, datetime('now'), datetime('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now')), - (?, ?, ?, datetime('now'), datetime('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now')) + (?, ?, ?, DATETIME('now'), DATETIME('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, DATETIME('now')), + (?, ?, ?, DATETIME('now'), DATETIME('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, DATETIME('now')), + (?, ?, ?, DATETIME('now'), DATETIME('now'), 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, DATETIME('now')) ` a3 := ` zone1-i710, i710, 3306, 710, , 5.6.7, 5.6, MySQL, false, false, STATEMENT, FULL, false, false, , 0, , 0, 0, 0, false, false, 0, 0, false, false, false, , , , , , , false, false, , 0, mysql.000007, 10, , 0, , , {0 false}, {0 false}, 0, , , , 0, false, false, false, false, false, 0, 0, false, false, 0, false, 0, @@ -429,27 +429,27 @@ func TestReadOutdatedInstanceKeys(t *testing.T) { }{ { name: "No problems", - sql: []string{"update database_instance set last_checked = datetime('now')"}, + sql: []string{"update database_instance set last_checked = DATETIME('now')"}, instancesRequired: nil, }, { name: "One instance is outdated", sql: []string{ - "update database_instance set last_checked = datetime('now')", - "update database_instance set last_checked = datetime('now', '-1 hour') where alias = 'zone1-0000000100'", + "update database_instance set last_checked = DATETIME('now')", + "update database_instance set last_checked = DATETIME('now', '-1 hour') where alias = 'zone1-0000000100'", }, instancesRequired: []string{"zone1-0000000100"}, }, { name: "One instance doesn't have myql data", sql: []string{ - "update database_instance set last_checked = datetime('now')", + "update database_instance set last_checked = DATETIME('now')", `INSERT INTO vitess_tablet VALUES('zone1-0000000103','localhost',7706,'ks','0','zone1',2,'0001-01-01 00:00:00+00:00','');`, }, instancesRequired: []string{"zone1-0000000103"}, }, { name: "One instance doesn't have myql data and one is outdated", sql: []string{ - "update database_instance set last_checked = datetime('now')", - "update database_instance set last_checked = datetime('now', '-1 hour') where alias = 'zone1-0000000100'", + "update database_instance set last_checked = DATETIME('now')", + "update database_instance set last_checked = DATETIME('now', '-1 hour') where alias = 'zone1-0000000100'", `INSERT INTO vitess_tablet VALUES('zone1-0000000103','localhost',7706,'ks','0','zone1',2,'0001-01-01 00:00:00+00:00','');`, }, instancesRequired: []string{"zone1-0000000103", "zone1-0000000100"}, @@ -486,10 +486,10 @@ func TestReadOutdatedInstanceKeys(t *testing.T) { errInDataCollection := db.QueryVTOrcRowsMap(`select alias, last_checked, last_attempted_check, -ROUND((JULIANDAY(datetime('now')) - JULIANDAY(last_checked)) * 86400) AS difference, +ROUND((JULIANDAY(DATETIME('now')) - JULIANDAY(last_checked)) * 86400) AS difference, last_attempted_check <= last_checked as use1, -last_checked < datetime('now', '-1500 second') as is_outdated1, -last_checked < datetime('now', '-3000 second') as is_outdated2 +last_checked < DATETIME('now', '-1500 second') as is_outdated1, +last_checked < DATETIME('now', '-3000 second') as is_outdated2 from database_instance`, func(rowMap sqlutils.RowMap) error { log.Errorf("Row in database_instance - %+v", rowMap) return nil @@ -513,12 +513,12 @@ func TestUpdateInstanceLastChecked(t *testing.T) { name: "Verify updated last checked", tabletAlias: "zone1-0000000100", partialSuccess: false, - conditionToCheck: "last_checked >= datetime('now', '-30 second') and last_check_partial_success = false", + conditionToCheck: "last_checked >= DATETIME('now', '-30 second') and last_check_partial_success = false", }, { name: "Verify partial success", tabletAlias: "zone1-0000000100", partialSuccess: true, - conditionToCheck: "last_checked >= datetime('now', '-30 second') and last_check_partial_success = true", + conditionToCheck: "last_checked >= DATETIME('now', '-30 second') and last_check_partial_success = true", }, { name: "Verify no error on unknown tablet", tabletAlias: "unknown tablet", @@ -564,7 +564,7 @@ func TestUpdateInstanceLastAttemptedCheck(t *testing.T) { { name: "Verify updated last checked", tabletAlias: "zone1-0000000100", - conditionToCheck: "last_attempted_check >= datetime('now', '-30 second')", + conditionToCheck: "last_attempted_check >= DATETIME('now', '-30 second')", }, { name: "Verify no error on unknown tablet", tabletAlias: "unknown tablet", @@ -737,19 +737,19 @@ func TestExpireTableData(t *testing.T) { tableName: "audit", timestampColumn: "audit_timestamp", expectedRowCount: 1, - insertQuery: `insert into audit (audit_id, audit_timestamp, audit_type, alias, message, keyspace, shard) values -(1, datetime('now', '-50 DAY'), 'a','a','a','a','a'), -(2, datetime('now', '-5 DAY'), 'a','a','a','a','a')`, + insertQuery: `INSERT INTO audit (audit_id, audit_timestamp, audit_type, alias, message, keyspace, shard) VALUES +(1, DATETIME('now', '-50 DAY'), 'a','a','a','a','a'), +(2, DATETIME('now', '-5 DAY'), 'a','a','a','a','a')`, }, { name: "ExpireRecoveryDetectionHistory", tableName: "recovery_detection", timestampColumn: "detection_timestamp", expectedRowCount: 2, - insertQuery: `insert into recovery_detection (detection_id, detection_timestamp, alias, analysis, keyspace, shard) values -(1, datetime('now', '-3 DAY'),'a','a','a','a'), -(2, datetime('now', '-5 DAY'),'a','a','a','a'), -(3, datetime('now', '-15 DAY'),'a','a','a','a')`, + insertQuery: `INSERT INTO recovery_detection (detection_id, detection_timestamp, alias, analysis, keyspace, shard) VALUES +(1, DATETIME('now', '-3 DAY'),'a','a','a','a'), +(2, DATETIME('now', '-5 DAY'),'a','a','a','a'), +(3, DATETIME('now', '-15 DAY'),'a','a','a','a')`, }, } for _, tt := range tests { diff --git a/go/vt/vtorc/inst/tablet_dao.go b/go/vt/vtorc/inst/tablet_dao.go index af304292a70..f48f2b97370 100644 --- a/go/vt/vtorc/inst/tablet_dao.go +++ b/go/vt/vtorc/inst/tablet_dao.go @@ -56,13 +56,13 @@ func fullStatus(tabletAlias string) (*replicationdatapb.FullStatus, error) { // ReadTablet reads the vitess tablet record. func ReadTablet(tabletAlias string) (*topodatapb.Tablet, error) { - query := ` - select - info - from - vitess_tablet - where alias = ? - ` + query := `SELECT + info + FROM + vitess_tablet + WHERE + alias = ? + ` args := sqlutils.Args(tabletAlias) tablet := &topodatapb.Tablet{} opts := prototext.UnmarshalOptions{DiscardUnknown: true} @@ -84,14 +84,28 @@ func SaveTablet(tablet *topodatapb.Tablet) error { if err != nil { return err } - _, err = db.ExecVTOrc(` - replace - into vitess_tablet ( - alias, hostname, port, cell, keyspace, shard, tablet_type, primary_timestamp, info - ) values ( - ?, ?, ?, ?, ?, ?, ?, ?, ? - ) - `, + _, err = db.ExecVTOrc(`REPLACE + INTO vitess_tablet ( + alias, + hostname, + port, + cell, + keyspace, + shard, + tablet_type, + primary_timestamp, + info + ) VALUES ( + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ? + )`, topoproto.TabletAliasString(tablet.Alias), tablet.MysqlHostname, int(tablet.MysqlPort), diff --git a/go/vt/vtorc/logic/disable_recovery.go b/go/vt/vtorc/logic/disable_recovery.go index 60650798876..c5446eeb9ff 100644 --- a/go/vt/vtorc/logic/disable_recovery.go +++ b/go/vt/vtorc/logic/disable_recovery.go @@ -40,14 +40,13 @@ import ( // IsRecoveryDisabled returns true if Recoveries are disabled globally func IsRecoveryDisabled() (disabled bool, err error) { - query := ` - SELECT - COUNT(*) as mycount - FROM - global_recovery_disable - WHERE - disable_recovery=? - ` + query := `SELECT + COUNT(*) AS mycount + FROM + global_recovery_disable + WHERE + disable_recovery = ? + ` err = db.QueryVTOrc(query, sqlutils.Args(1), func(m sqlutils.RowMap) error { mycount := m.GetInt("mycount") disabled = (mycount > 0) @@ -63,21 +62,19 @@ func IsRecoveryDisabled() (disabled bool, err error) { // DisableRecovery ensures recoveries are disabled globally func DisableRecovery() error { - _, err := db.ExecVTOrc(` - INSERT OR IGNORE INTO global_recovery_disable - (disable_recovery) - VALUES (1) - `, - ) + _, err := db.ExecVTOrc(`INSERT OR IGNORE + INTO global_recovery_disable ( + disable_recovery + ) VALUES (1)`) return err } // EnableRecovery ensures recoveries are enabled globally func EnableRecovery() error { // The "WHERE" clause is just to avoid full-scan reports by monitoring tools - _, err := db.ExecVTOrc(` - DELETE FROM global_recovery_disable WHERE disable_recovery >= 0 - `, - ) + _, err := db.ExecVTOrc(`DELETE + FROM global_recovery_disable + WHERE + disable_recovery >= 0`) return err } diff --git a/go/vt/vtorc/logic/tablet_discovery.go b/go/vt/vtorc/logic/tablet_discovery.go index e9bbcee35cb..1ee9e2cfefa 100644 --- a/go/vt/vtorc/logic/tablet_discovery.go +++ b/go/vt/vtorc/logic/tablet_discovery.go @@ -66,7 +66,7 @@ func OpenTabletDiscovery() <-chan time.Time { ts = topo.Open() tmc = inst.InitializeTMC() // Clear existing cache and perform a new refresh. - if _, err := db.ExecVTOrc("delete from vitess_tablet"); err != nil { + if _, err := db.ExecVTOrc("DELETE FROM vitess_tablet"); err != nil { log.Error(err) } // We refresh all information from the topo once before we start the ticks to do it on a timer. diff --git a/go/vt/vtorc/logic/topology_recovery_dao.go b/go/vt/vtorc/logic/topology_recovery_dao.go index 730e6b2a158..137251c4fc8 100644 --- a/go/vt/vtorc/logic/topology_recovery_dao.go +++ b/go/vt/vtorc/logic/topology_recovery_dao.go @@ -30,21 +30,20 @@ import ( // InsertRecoveryDetection inserts the recovery analysis that has been detected. func InsertRecoveryDetection(analysisEntry *inst.ReplicationAnalysis) error { - sqlResult, err := db.ExecVTOrc(` - insert or ignore - into recovery_detection ( - alias, - analysis, - keyspace, - shard, - detection_timestamp - ) values ( - ?, - ?, - ?, - ?, - datetime('now') - )`, + sqlResult, err := db.ExecVTOrc(`INSERT OR IGNORE + INTO recovery_detection ( + alias, + analysis, + keyspace, + shard, + detection_timestamp + ) VALUES ( + ?, + ?, + ?, + ?, + DATETIME('now') + )`, analysisEntry.AnalyzedInstanceAlias, string(analysisEntry.Analysis), analysisEntry.ClusterDetails.Keyspace, @@ -65,26 +64,24 @@ func InsertRecoveryDetection(analysisEntry *inst.ReplicationAnalysis) error { func writeTopologyRecovery(topologyRecovery *TopologyRecovery) (*TopologyRecovery, error) { analysisEntry := topologyRecovery.AnalysisEntry - sqlResult, err := db.ExecVTOrc(` - insert or ignore - into topology_recovery ( - recovery_id, - alias, - start_recovery, - analysis, - keyspace, - shard, - detection_id - ) values ( - ?, - ?, - datetime('now'), - ?, - ?, - ?, - ? - ) - `, + sqlResult, err := db.ExecVTOrc(`INSERT OR IGNORE + INTO topology_recovery ( + recovery_id, + alias, + start_recovery, + analysis, + keyspace, + shard, + detection_id + ) VALUES ( + ?, + ?, + DATETIME('now'), + ?, + ?, + ?, + ? + )`, sqlutils.NilIfZero(topologyRecovery.ID), analysisEntry.AnalyzedInstanceAlias, string(analysisEntry.Analysis), @@ -138,15 +135,16 @@ func AttemptRecoveryRegistration(analysisEntry *inst.ReplicationAnalysis) (*Topo // ResolveRecovery is called on completion of a recovery process and updates the recovery status. // It does not clear the "active period" as this still takes place in order to avoid flapping. func writeResolveRecovery(topologyRecovery *TopologyRecovery) error { - _, err := db.ExecVTOrc(` - update topology_recovery set - is_successful = ?, - successor_alias = ?, - all_errors = ?, - end_recovery = datetime('now') - where - recovery_id = ? - `, topologyRecovery.IsSuccessful, + _, err := db.ExecVTOrc(`UPDATE topology_recovery + SET + is_successful = ?, + successor_alias = ?, + all_errors = ?, + end_recovery = DATETIME('now') + WHERE + recovery_id = ? + `, + topologyRecovery.IsSuccessful, topologyRecovery.SuccessorAlias, strings.Join(topologyRecovery.AllErrors, "\n"), topologyRecovery.ID, @@ -160,26 +158,27 @@ func writeResolveRecovery(topologyRecovery *TopologyRecovery) error { // readRecoveries reads recovery entry/audit entries from topology_recovery func readRecoveries(whereCondition string, limit string, args []any) ([]*TopologyRecovery, error) { res := []*TopologyRecovery{} - query := fmt.Sprintf(` - select - recovery_id, - alias, - start_recovery, - IFNULL(end_recovery, '') AS end_recovery, - is_successful, - ifnull(successor_alias, '') as successor_alias, - analysis, - keyspace, - shard, - all_errors, - detection_id - from + query := fmt.Sprintf(`SELECT + recovery_id, + alias, + start_recovery, + IFNULL(end_recovery, '') AS end_recovery, + is_successful, + IFNULL(successor_alias, '') AS successor_alias, + analysis, + keyspace, + shard, + all_errors, + detection_id + FROM topology_recovery %s - order by - recovery_id desc + ORDER BY recovery_id DESC %s - `, whereCondition, limit) + `, + whereCondition, + limit, + ) err := db.QueryVTOrc(query, args, func(m sqlutils.RowMap) error { topologyRecovery := *NewTopologyRecovery(inst.ReplicationAnalysis{}) topologyRecovery.ID = m.GetInt64("recovery_id") @@ -211,11 +210,10 @@ func readRecoveries(whereCondition string, limit string, args []any) ([]*Topolog // ReadActiveClusterRecoveries reads recoveries that are ongoing for the given cluster. func ReadActiveClusterRecoveries(keyspace string, shard string) ([]*TopologyRecovery, error) { - whereClause := ` - where - end_recovery IS NULL - and keyspace=? - and shard=?` + whereClause := `WHERE + end_recovery IS NULL + AND keyspace = ? + AND shard = ?` return readRecoveries(whereClause, ``, sqlutils.Args(keyspace, shard)) } @@ -225,23 +223,30 @@ func ReadRecentRecoveries(page int) ([]*TopologyRecovery, error) { whereClause := "" var args []any if len(whereConditions) > 0 { - whereClause = fmt.Sprintf("where %s", strings.Join(whereConditions, " and ")) + whereClause = fmt.Sprintf("WHERE %s", strings.Join(whereConditions, " AND ")) } - limit := ` - limit ? - offset ?` + limit := `LIMIT ? OFFSET ?` args = append(args, config.AuditPageSize, page*config.AuditPageSize) return readRecoveries(whereClause, limit, args) } // writeTopologyRecoveryStep writes down a single step in a recovery process func writeTopologyRecoveryStep(topologyRecoveryStep *TopologyRecoveryStep) error { - sqlResult, err := db.ExecVTOrc(` - insert or ignore - into topology_recovery_steps ( - recovery_step_id, recovery_id, audit_at, message - ) values (?, ?, datetime('now'), ?) - `, sqlutils.NilIfZero(topologyRecoveryStep.ID), topologyRecoveryStep.RecoveryID, topologyRecoveryStep.Message, + sqlResult, err := db.ExecVTOrc(`INSERT OR IGNORE + INTO topology_recovery_steps ( + recovery_step_id, + recovery_id, + audit_at, + message + ) VALUES ( + ?, + ?, + DATETIME('now'), + ? + )`, + sqlutils.NilIfZero(topologyRecoveryStep.ID), + topologyRecoveryStep.RecoveryID, + topologyRecoveryStep.Message, ) if err != nil { log.Error(err) diff --git a/go/vt/vtorc/process/health.go b/go/vt/vtorc/process/health.go index 87a11733f66..f72d7b05210 100644 --- a/go/vt/vtorc/process/health.go +++ b/go/vt/vtorc/process/health.go @@ -35,12 +35,17 @@ var ThisNodeHealth = &NodeHealth{} // writeHealthToDatabase writes to the database and returns if it was successful. func writeHealthToDatabase() bool { - _, err := db.ExecVTOrc("delete from node_health") + _, err := db.ExecVTOrc("DELETE FROM node_health") if err != nil { log.Error(err) return false } - sqlResult, err := db.ExecVTOrc(`insert into node_health (last_seen_active) values (datetime('now'))`) + sqlResult, err := db.ExecVTOrc(`INSERT + INTO node_health ( + last_seen_active + ) VALUES ( + DATETIME('now') + )`) if err != nil { log.Error(err) return false From 1f23496f0fd88159c43923f080c0af6a991f9413 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:26:36 +0200 Subject: [PATCH 07/28] Multi-metrics throttler: adding InnoDB `history_list_length` metric (#17262) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../throttler_topo/throttler_test.go | 34 ++++++++++ .../tabletserver/throttle/base/metric_name.go | 1 + .../throttle/base/metric_name_test.go | 4 +- .../self_metric_innodb_history_list_length.go | 68 +++++++++++++++++++ .../tabletserver/throttle/throttler_test.go | 17 +++++ 5 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go diff --git a/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go b/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go index f96069c81b8..d5a1053d77d 100644 --- a/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go @@ -175,6 +175,16 @@ func throttledApps(tablet *cluster.Vttablet) (resp *http.Response, respBody stri return resp, respBody, err } +func vitessThrottleCheck(tablet *cluster.Vttablet, skipRequestHeartbeats bool) (*vtctldatapb.CheckThrottlerResponse, error) { + flags := &throttle.CheckFlags{ + Scope: base.ShardScope, + SkipRequestHeartbeats: skipRequestHeartbeats, + MultiMetricsEnabled: true, + } + resp, err := throttler.CheckThrottler(clusterInstance, tablet, throttlerapp.VitessName, flags) + return resp, err +} + func throttleCheck(tablet *cluster.Vttablet, skipRequestHeartbeats bool) (*vtctldatapb.CheckThrottlerResponse, error) { flags := &throttle.CheckFlags{ Scope: base.ShardScope, @@ -305,6 +315,17 @@ func TestInitialThrottler(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, tabletmanagerdatapb.CheckThrottlerResponseCode_THRESHOLD_EXCEEDED) }) t.Run("setting high threshold", func(t *testing.T) { + { + req := &vtctldatapb.UpdateThrottlerConfigRequest{MetricName: base.LoadAvgMetricName.String(), Threshold: 5555} + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil) + assert.NoError(t, err) + } + { + req := &vtctldatapb.UpdateThrottlerConfigRequest{MetricName: base.MysqldLoadAvgMetricName.String(), Threshold: 5555} + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil) + assert.NoError(t, err) + } + req := &vtctldatapb.UpdateThrottlerConfigRequest{Threshold: extremelyHighThreshold.Seconds()} _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil) assert.NoError(t, err) @@ -317,6 +338,19 @@ func TestInitialThrottler(t *testing.T) { t.Run("validating OK response from throttler with high threshold", func(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, tabletmanagerdatapb.CheckThrottlerResponseCode_OK) }) + t.Run("validating vitess app throttler check", func(t *testing.T) { + resp, err := vitessThrottleCheck(primaryTablet, true) + require.NoError(t, err) + for _, metricName := range base.KnownMetricNames { + t.Run(metricName.String(), func(t *testing.T) { + assert.Contains(t, resp.Check.Metrics, metricName.String()) + metric := resp.Check.Metrics[metricName.String()] + require.NotNil(t, metric) + assert.Equal(t, tabletmanagerdatapb.CheckThrottlerResponseCode_OK, metric.ResponseCode, "metric: %+v", metric) + }) + } + }) + t.Run("setting low threshold", func(t *testing.T) { req := &vtctldatapb.UpdateThrottlerConfigRequest{Threshold: throttler.DefaultThreshold.Seconds()} _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil) diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_name.go b/go/vt/vttablet/tabletserver/throttle/base/metric_name.go index 607192b9c0c..43bd2d17a8c 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/metric_name.go +++ b/go/vt/vttablet/tabletserver/throttle/base/metric_name.go @@ -65,6 +65,7 @@ const ( ThreadsRunningMetricName MetricName = "threads_running" CustomMetricName MetricName = "custom" LoadAvgMetricName MetricName = "loadavg" + HistoryListLengthMetricName MetricName = "history_list_length" MysqldLoadAvgMetricName MetricName = "mysqld-loadavg" MysqldDatadirUsedRatioMetricName MetricName = "mysqld-datadir-used-ratio" ) diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go b/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go index ffd7f674cc2..c2e2b44b36f 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go +++ b/go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go @@ -241,15 +241,17 @@ func TestKnownMetricNames(t *testing.T) { assert.Contains(t, KnownMetricNames, LoadAvgMetricName) assert.Contains(t, KnownMetricNames, CustomMetricName) assert.Contains(t, KnownMetricNames, DefaultMetricName) + assert.Contains(t, KnownMetricNames, HistoryListLengthMetricName) assert.Contains(t, KnownMetricNames, MysqldLoadAvgMetricName) assert.Contains(t, KnownMetricNames, MysqldDatadirUsedRatioMetricName) } -func TestSingleWordCamelKnownMetricNames(t *testing.T) { +func TestKnownMetricNamesPascalCase(t *testing.T) { expectCases := map[MetricName]string{ LagMetricName: "Lag", ThreadsRunningMetricName: "ThreadsRunning", LoadAvgMetricName: "Loadavg", + HistoryListLengthMetricName: "HistoryListLength", CustomMetricName: "Custom", DefaultMetricName: "Default", MysqldLoadAvgMetricName: "MysqldLoadavg", diff --git a/go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go b/go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go new file mode 100644 index 00000000000..2696b1750ea --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import ( + "context" + "math" + "sync/atomic" + "time" +) + +var ( + historyListLengthQuery = "select count as history_len from information_schema.INNODB_METRICS where name = 'trx_rseg_history_len'" + + cachedHistoryListLengthMetric atomic.Pointer[ThrottleMetric] + historyListLengthCacheDuration = 5 * time.Second + historyListLengthDefaultThreshold = math.Pow10(9) +) + +var _ SelfMetric = registerSelfMetric(&HistoryListLengthSelfMetric{}) + +type HistoryListLengthSelfMetric struct { +} + +func (m *HistoryListLengthSelfMetric) Name() MetricName { + return HistoryListLengthMetricName +} + +func (m *HistoryListLengthSelfMetric) DefaultScope() Scope { + return SelfScope +} + +func (m *HistoryListLengthSelfMetric) DefaultThreshold() float64 { + return historyListLengthDefaultThreshold +} + +func (m *HistoryListLengthSelfMetric) RequiresConn() bool { + return true +} + +func (m *HistoryListLengthSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric { + // This function will be called sequentially, and therefore does not need strong mutex protection. Still, we use atomics + // to ensure correctness in case an external goroutine tries to read the metric concurrently. + metric := cachedHistoryListLengthMetric.Load() + if metric != nil { + return metric + } + metric = ReadSelfMySQLThrottleMetric(ctx, params.Conn, historyListLengthQuery) + cachedHistoryListLengthMetric.Store(metric) + time.AfterFunc(historyListLengthCacheDuration, func() { + cachedHistoryListLengthMetric.Store(nil) + }) + return metric +} diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 0a1162b02d3..352e641fa35 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -71,6 +71,12 @@ var ( Value: 2.718, Err: nil, }, + base.HistoryListLengthMetricName: &base.ThrottleMetric{ + Scope: base.SelfScope, + Alias: "", + Value: 5, + Err: nil, + }, base.MysqldLoadAvgMetricName: &base.ThrottleMetric{ Scope: base.SelfScope, Alias: "", @@ -105,6 +111,11 @@ var ( ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK, Value: 5.1, }, + base.HistoryListLengthMetricName.String(): { + StatusCode: http.StatusOK, + ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK, + Value: 6, + }, base.MysqldLoadAvgMetricName.String(): { StatusCode: http.StatusOK, ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK, @@ -1853,6 +1864,7 @@ func TestChecks(t *testing.T) { assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because flags.Scope is set assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because flags.Scope is set assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because flags.Scope is set + assert.EqualValues(t, 5, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // self value, because flags.Scope is set assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // self value, because flags.Scope is set assert.EqualValues(t, 0.85, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // self value, because flags.Scope is set for _, metric := range checkResult.Metrics { @@ -1914,6 +1926,7 @@ func TestChecks(t *testing.T) { assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, because flags.Scope is set assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value, because flags.Scope is set assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set + assert.EqualValues(t, 6, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // shard value, because flags.Scope is set assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set assert.EqualValues(t, 0.87, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // shard value, because flags.Scope is set for _, metric := range checkResult.Metrics { @@ -1948,6 +1961,7 @@ func TestChecks(t *testing.T) { assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because "self" is the default scope for threads_running assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because "self" is the default scope for custom assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg + assert.EqualValues(t, 5, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // self value, because "self" is the default scope for loadavg assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg assert.EqualValues(t, 0.85, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // self value, because "self" is the default scope for loadavg assert.EqualValues(t, base.ShardScope.String(), checkResult.Metrics[base.LagMetricName.String()].Scope) @@ -1970,6 +1984,7 @@ func TestChecks(t *testing.T) { base.MetricName("custom"), base.MetricName("shard/loadavg"), base.MetricName("shard/mysqld-loadavg"), + base.MetricName("self/history_list_length"), base.MetricName("self/mysqld-datadir-used-ratio"), base.MetricName("default"), } @@ -1986,6 +2001,7 @@ func TestChecks(t *testing.T) { assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, even though scope name is in metric name assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value because flags.Scope is set assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set + assert.EqualValues(t, 6, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // shard value, even though scope name is in metric name assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set assert.EqualValues(t, 0.87, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // shard value, even though scope name is in metric name for _, metric := range checkResult.Metrics { @@ -2257,6 +2273,7 @@ func TestReplica(t *testing.T) { assert.Error(t, metricResult.Error, "metricName=%v, value=%v, threshold=%v", metricName, metricResult.Value, metricResult.Threshold) assert.ErrorIs(t, metricResult.Error, base.ErrThresholdExceeded) case base.ThreadsRunningMetricName, + base.HistoryListLengthMetricName, base.MysqldLoadAvgMetricName, base.MysqldDatadirUsedRatioMetricName: assert.NoError(t, metricResult.Error, "metricName=%v, value=%v, threshold=%v", metricName, metricResult.Value, metricResult.Threshold) From 8648264f9049cd979669d6942731fe95dec79e15 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 26 Nov 2024 16:33:27 +0100 Subject: [PATCH 08/28] `vtorc`: cleanup init db handling (#17198) Signed-off-by: Tim Vaillancourt Signed-off-by: Manan Gupta Co-authored-by: Manan Gupta --- go/vt/vtorc/db/db.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/go/vt/vtorc/db/db.go b/go/vt/vtorc/db/db.go index 4bb22b65b5a..470e5364680 100644 --- a/go/vt/vtorc/db/db.go +++ b/go/vt/vtorc/db/db.go @@ -47,7 +47,9 @@ func OpenVTOrc() (db *sql.DB, err error) { db, fromCache, err = sqlutils.GetSQLiteDB(config.Config.SQLite3DataFile) if err == nil && !fromCache { log.Infof("Connected to vtorc backend: sqlite on %v", config.Config.SQLite3DataFile) - _ = initVTOrcDB(db) + if err := initVTOrcDB(db); err != nil { + log.Fatalf("Cannot initiate vtorc: %+v", err) + } } if db != nil { db.SetMaxOpenConns(1) @@ -76,19 +78,14 @@ func registerVTOrcDeployment(db *sql.DB) error { func deployStatements(db *sql.DB, queries []string) error { tx, err := db.Begin() if err != nil { - log.Fatal(err.Error()) return err } for _, query := range queries { if _, err := tx.Exec(query); err != nil { - log.Fatalf("Cannot initiate vtorc: %+v; query=%+v", err, query) return err } } - if err := tx.Commit(); err != nil { - log.Fatal(err.Error()) - } - return nil + return tx.Commit() } // ClearVTOrcDatabase is used to clear the VTOrc database. This function is meant to be used by tests to clear the @@ -96,7 +93,9 @@ func deployStatements(db *sql.DB, queries []string) error { func ClearVTOrcDatabase() { db, _, _ := sqlutils.GetSQLiteDB(config.Config.SQLite3DataFile) if db != nil { - _ = initVTOrcDB(db) + if err := initVTOrcDB(db); err != nil { + log.Fatalf("Cannot re-initiate vtorc: %+v", err) + } } } @@ -105,12 +104,18 @@ func ClearVTOrcDatabase() { func initVTOrcDB(db *sql.DB) error { log.Info("Initializing vtorc") log.Info("Migrating database schema") - _ = deployStatements(db, vtorcBackend) - _ = registerVTOrcDeployment(db) - - _, _ = ExecVTOrc(`PRAGMA journal_mode = WAL`) - _, _ = ExecVTOrc(`PRAGMA synchronous = NORMAL`) - + if err := deployStatements(db, vtorcBackend); err != nil { + return err + } + if err := registerVTOrcDeployment(db); err != nil { + return err + } + if _, err := ExecVTOrc(`PRAGMA journal_mode = WAL`); err != nil { + return err + } + if _, err := ExecVTOrc(`PRAGMA synchronous = NORMAL`); err != nil { + return err + } return nil } From 16fa7a3641af3cbfbce7525c57e4dc462e2c555b Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:10:01 +0530 Subject: [PATCH 09/28] Errant GTID detection fix for VTOrc (#17287) Signed-off-by: Manan Gupta --- go/vt/vtorc/inst/instance_dao.go | 11 ++++-- go/vt/vtorc/inst/instance_dao_test.go | 49 +++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index c8ff218710f..45291394c56 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -362,7 +362,7 @@ Cleanup: // Add replication group ancestry UUID as well. Otherwise, VTOrc thinks there are errant GTIDs in group // members and its replicas, even though they are not. instance.AncestryUUID = strings.Trim(instance.AncestryUUID, ",") - err = detectErrantGTIDs(tabletAlias, instance, tablet) + err = detectErrantGTIDs(instance, tablet) } latency.Stop("instance") @@ -390,13 +390,18 @@ Cleanup: } // detectErrantGTIDs detects the errant GTIDs on an instance. -func detectErrantGTIDs(tabletAlias string, instance *Instance, tablet *topodatapb.Tablet) (err error) { +func detectErrantGTIDs(instance *Instance, tablet *topodatapb.Tablet) (err error) { // If the tablet is not replicating from anyone, then it could be the previous primary. // We should check for errant GTIDs by finding the difference with the shard's current primary. if instance.primaryExecutedGtidSet == "" && instance.SourceHost == "" { var primaryInstance *Instance primaryAlias, _, _ := ReadShardPrimaryInformation(tablet.Keyspace, tablet.Shard) if primaryAlias != "" { + // Check if the current tablet is the primary. + // If it is, then we don't need to run errant gtid detection on it. + if primaryAlias == instance.InstanceAlias { + return nil + } primaryInstance, _, _ = ReadInstance(primaryAlias) } // Only run errant GTID detection, if we are sure that the data read of the current primary @@ -434,7 +439,7 @@ func detectErrantGTIDs(tabletAlias string, instance *Instance, tablet *topodatap if err == nil { var gtidCount int64 gtidCount, err = replication.GTIDCount(instance.GtidErrant) - currentErrantGTIDCount.Set(tabletAlias, gtidCount) + currentErrantGTIDCount.Set(instance.InstanceAlias, gtidCount) } } } diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index f248ded5e2b..0d59de0588f 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -854,7 +854,7 @@ func TestDetectErrantGTIDs(t *testing.T) { primaryTablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone-1", - Uid: 100, + Uid: 101, }, Keyspace: keyspaceName, Shard: shardName, @@ -881,7 +881,8 @@ func TestDetectErrantGTIDs(t *testing.T) { require.NoError(t, err) } - err = detectErrantGTIDs(topoproto.TabletAliasString(tablet.Alias), tt.instance, tablet) + tt.instance.InstanceAlias = topoproto.TabletAliasString(tablet.Alias) + err = detectErrantGTIDs(tt.instance, tablet) if tt.wantErr { require.Error(t, err) return @@ -891,3 +892,47 @@ func TestDetectErrantGTIDs(t *testing.T) { }) } } + +// TestPrimaryErrantGTIDs tests that we don't run Errant GTID detection on the primary tablet itself! +func TestPrimaryErrantGTIDs(t *testing.T) { + // Clear the database after the test. The easiest way to do that is to run all the initialization commands again. + defer func() { + db.ClearVTOrcDatabase() + }() + db.ClearVTOrcDatabase() + keyspaceName := "ks" + shardName := "0" + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone-1", + Uid: 100, + }, + Keyspace: keyspaceName, + Shard: shardName, + } + instance := &Instance{ + SourceHost: "", + ExecutedGtidSet: "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10589,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-341", + InstanceAlias: topoproto.TabletAliasString(tablet.Alias), + } + + // Save shard record for the primary tablet. + err := SaveShard(topo.NewShardInfo(keyspaceName, shardName, &topodatapb.Shard{ + PrimaryAlias: tablet.Alias, + }, nil)) + require.NoError(t, err) + + // Store the tablet record and the instance. + err = SaveTablet(tablet) + require.NoError(t, err) + err = WriteInstance(instance, true, nil) + require.NoError(t, err) + + // After this if we read a new information for the record that updates its + // gtid set further, we shouldn't be detecting errant GTIDs on it since it is the primary! + // We shouldn't be comparing it with a previous version of itself! + instance.ExecutedGtidSet = "230ea8ea-81e3-11e4-972a-e25ec4bd140a:1-10589,8bc65c84-3fe4-11ed-a912-257f0fcdd6c9:1-34,316d193c-70e5-11e5-adb2-ecf4bb2262ff:1-351" + err = detectErrantGTIDs(instance, tablet) + require.NoError(t, err) + require.EqualValues(t, "", instance.GtidErrant) +} From 0726ea6ad92792da0fb998504c4e61add3999be9 Mon Sep 17 00:00:00 2001 From: Florent Poinsard <35779988+frouioui@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:58:55 -0600 Subject: [PATCH 10/28] Fix how we cancel the context in the builtin backup engine (#17285) Signed-off-by: Florent Poinsard --- go/vt/mysqlctl/builtinbackupengine.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index f3cbe5364a0..5aa759f6f7a 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -604,7 +604,15 @@ func (be *BuiltinBackupEngine) backupFiles( wg := sync.WaitGroup{} ctxCancel, cancel := context.WithCancel(ctx) - defer cancel() + defer func() { + // We may still have operations in flight that require a valid context, such as adding files to S3. + // Unless we encountered an error, we should not cancel the context, this is taken care of later + // in the process. If we encountered an error however, we can safely cancel the context as we should + // no longer work on anything and exit fast. + if finalErr != nil { + cancel() + } + }() for i := range fes { wg.Add(1) @@ -1037,7 +1045,15 @@ func (be *BuiltinBackupEngine) restoreFiles(ctx context.Context, params RestoreP wg := sync.WaitGroup{} ctxCancel, cancel := context.WithCancel(ctx) - defer cancel() + defer func() { + // We may still have operations in flight that require a valid context, such as adding files to S3. + // Unless we encountered an error, we should not cancel the context. This is taken care of later + // in the process. If we encountered an error however, we can safely cancel the context as we should + // no longer work on anything and exit fast. + if err != nil { + cancel() + } + }() for i := range fes { wg.Add(1) From a1830b37ccc46e15a0619e9efcb8c9266ca7c143 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 28 Nov 2024 18:53:27 +0530 Subject: [PATCH 11/28] vttablet debugenv changes (#17161) Signed-off-by: Harshit Gangal Signed-off-by: Manan Gupta Co-authored-by: Manan Gupta --- go/vt/vttablet/endtoend/config_test.go | 4 +- go/vt/vttablet/tabletserver/debugenv.go | 211 ++++++++++++++---------- 2 files changed, 130 insertions(+), 85 deletions(-) diff --git a/go/vt/vttablet/endtoend/config_test.go b/go/vt/vttablet/endtoend/config_test.go index 4abf5b36c21..c3ad5f8a9db 100644 --- a/go/vt/vttablet/endtoend/config_test.go +++ b/go/vt/vttablet/endtoend/config_test.go @@ -36,7 +36,7 @@ import ( ) func TestPoolSize(t *testing.T) { - revert := changeVar(t, "PoolSize", "1") + revert := changeVar(t, "ReadPoolSize", "1") defer revert() vstart := framework.DebugVars() @@ -92,7 +92,7 @@ func TestTxPoolSize(t *testing.T) { defer client2.Rollback() verifyIntValue(t, framework.DebugVars(), "FoundRowsPoolAvailable", framework.FetchInt(vstart, "FoundRowsPoolAvailable")-1) - revert := changeVar(t, "TxPoolSize", "1") + revert := changeVar(t, "TransactionPoolSize", "1") defer revert() vend := framework.DebugVars() verifyIntValue(t, vend, "TransactionPoolAvailable", 0) diff --git a/go/vt/vttablet/tabletserver/debugenv.go b/go/vt/vttablet/tabletserver/debugenv.go index 54cf09db7d6..9a802a5d49c 100644 --- a/go/vt/vttablet/tabletserver/debugenv.go +++ b/go/vt/vttablet/tabletserver/debugenv.go @@ -70,90 +70,131 @@ func debugEnvHandler(tsv *TabletServer, w http.ResponseWriter, r *http.Request) return } + switch r.Method { + case http.MethodPost: + handlePost(tsv, w, r) + case http.MethodGet: + handleGet(tsv, w, r) + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} + +func handlePost(tsv *TabletServer, w http.ResponseWriter, r *http.Request) { + varname := r.FormValue("varname") + value := r.FormValue("value") + var msg string - if r.Method == "POST" { - varname := r.FormValue("varname") - value := r.FormValue("value") - setIntVal := func(f func(int)) { - ival, err := strconv.Atoi(value) - if err != nil { - msg = fmt.Sprintf("Failed setting value for %v: %v", varname, err) - return - } - f(ival) - msg = fmt.Sprintf("Setting %v to: %v", varname, value) + if varname == "" || value == "" { + http.Error(w, "Missing varname or value", http.StatusBadRequest) + return + } + + setIntVal := func(f func(int)) error { + ival, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid int value for %v: %v", varname, err) } - setIntValCtx := func(f func(context.Context, int) error) { - ival, err := strconv.Atoi(value) - if err == nil { - err = f(r.Context(), ival) - if err == nil { - msg = fmt.Sprintf("Setting %v to: %v", varname, value) - return - } - } - msg = fmt.Sprintf("Failed setting value for %v: %v", varname, err) + f(ival) + msg = fmt.Sprintf("Setting %v to: %v", varname, value) + return nil + } + + setIntValCtx := func(f func(context.Context, int) error) error { + ival, err := strconv.Atoi(value) + if err == nil { + err = f(r.Context(), ival) } - setInt64Val := func(f func(int64)) { - ival, err := strconv.ParseInt(value, 10, 64) - if err != nil { - msg = fmt.Sprintf("Failed setting value for %v: %v", varname, err) - return - } - f(ival) - msg = fmt.Sprintf("Setting %v to: %v", varname, value) + if err != nil { + return fmt.Errorf("failed setting value for %v: %v", varname, err) } - setDurationVal := func(f func(time.Duration)) { - durationVal, err := time.ParseDuration(value) - if err != nil { - msg = fmt.Sprintf("Failed setting value for %v: %v", varname, err) - return - } - f(durationVal) - msg = fmt.Sprintf("Setting %v to: %v", varname, value) + msg = fmt.Sprintf("Setting %v to: %v", varname, value) + return nil + } + + setInt64Val := func(f func(int64)) error { + ival, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("invalid int64 value for %v: %v", varname, err) } - setFloat64Val := func(f func(float64)) { - fval, err := strconv.ParseFloat(value, 64) - if err != nil { - msg = fmt.Sprintf("Failed setting value for %v: %v", varname, err) - return - } - f(fval) - msg = fmt.Sprintf("Setting %v to: %v", varname, value) + f(ival) + msg = fmt.Sprintf("Setting %v to: %v", varname, value) + return nil + } + + setDurationVal := func(f func(time.Duration)) error { + durationVal, err := time.ParseDuration(value) + if err != nil { + return fmt.Errorf("invalid duration value for %v: %v", varname, err) } - switch varname { - case "PoolSize": - setIntValCtx(tsv.SetPoolSize) - case "StreamPoolSize": - setIntValCtx(tsv.SetStreamPoolSize) - case "TxPoolSize": - setIntValCtx(tsv.SetTxPoolSize) - case "MaxResultSize": - setIntVal(tsv.SetMaxResultSize) - case "WarnResultSize": - setIntVal(tsv.SetWarnResultSize) - case "RowStreamerMaxInnoDBTrxHistLen": - setInt64Val(func(val int64) { tsv.Config().RowStreamer.MaxInnoDBTrxHistLen = val }) - case "RowStreamerMaxMySQLReplLagSecs": - setInt64Val(func(val int64) { tsv.Config().RowStreamer.MaxMySQLReplLagSecs = val }) - case "UnhealthyThreshold": - setDurationVal(func(d time.Duration) { tsv.Config().Healthcheck.UnhealthyThreshold = d }) - setDurationVal(tsv.hs.SetUnhealthyThreshold) - setDurationVal(tsv.sm.SetUnhealthyThreshold) - case "ThrottleMetricThreshold": - setFloat64Val(tsv.SetThrottleMetricThreshold) - case "Consolidator": - tsv.SetConsolidatorMode(value) - msg = fmt.Sprintf("Setting %v to: %v", varname, value) + f(durationVal) + msg = fmt.Sprintf("Setting %v to: %v", varname, value) + return nil + } + + setFloat64Val := func(f func(float64)) error { + fval, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid float64 value for %v: %v", varname, err) } + f(fval) + msg = fmt.Sprintf("Setting %v to: %v", varname, value) + return nil + } + + var err error + switch varname { + case "ReadPoolSize": + err = setIntValCtx(tsv.SetPoolSize) + case "StreamPoolSize": + err = setIntValCtx(tsv.SetStreamPoolSize) + case "TransactionPoolSize": + err = setIntValCtx(tsv.SetTxPoolSize) + case "MaxResultSize": + err = setIntVal(tsv.SetMaxResultSize) + case "WarnResultSize": + err = setIntVal(tsv.SetWarnResultSize) + case "RowStreamerMaxInnoDBTrxHistLen": + err = setInt64Val(func(val int64) { tsv.Config().RowStreamer.MaxInnoDBTrxHistLen = val }) + case "RowStreamerMaxMySQLReplLagSecs": + err = setInt64Val(func(val int64) { tsv.Config().RowStreamer.MaxMySQLReplLagSecs = val }) + case "UnhealthyThreshold": + err = setDurationVal(func(d time.Duration) { tsv.Config().Healthcheck.UnhealthyThreshold = d }) + case "ThrottleMetricThreshold": + err = setFloat64Val(tsv.SetThrottleMetricThreshold) + case "Consolidator": + tsv.SetConsolidatorMode(value) + msg = fmt.Sprintf("Setting %v to: %v", varname, value) + } + + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } + vars := getVars(tsv) + sendResponse(r, w, vars, msg) +} + +func handleGet(tsv *TabletServer, w http.ResponseWriter, r *http.Request) { + vars := getVars(tsv) + sendResponse(r, w, vars, "") +} + +func sendResponse(r *http.Request, w http.ResponseWriter, vars []envValue, msg string) { + format := r.FormValue("format") + if format == "json" { + respondWithJSON(w, vars, msg) + return + } + respondWithHTML(w, vars, msg) +} + +func getVars(tsv *TabletServer) []envValue { var vars []envValue - vars = addVar(vars, "PoolSize", tsv.PoolSize) + vars = addVar(vars, "ReadPoolSize", tsv.PoolSize) vars = addVar(vars, "StreamPoolSize", tsv.StreamPoolSize) - vars = addVar(vars, "TxPoolSize", tsv.TxPoolSize) - vars = addVar(vars, "QueryCacheCapacity", tsv.QueryPlanCacheCap) // QueryCacheCapacity is deprecated in v21, it is replaced by QueryEnginePlanCacheCapacity - vars = addVar(vars, "QueryEnginePlanCacheCapacity", tsv.QueryPlanCacheCap) + vars = addVar(vars, "TransactionPoolSize", tsv.TxPoolSize) vars = addVar(vars, "MaxResultSize", tsv.MaxResultSize) vars = addVar(vars, "WarnResultSize", tsv.WarnResultSize) vars = addVar(vars, "RowStreamerMaxInnoDBTrxHistLen", func() int64 { return tsv.Config().RowStreamer.MaxInnoDBTrxHistLen }) @@ -165,18 +206,22 @@ func debugEnvHandler(tsv *TabletServer, w http.ResponseWriter, r *http.Request) Value: tsv.ConsolidatorMode(), }) - format := r.FormValue("format") - if format == "json" { - mvars := make(map[string]string) - for _, v := range vars { - mvars[v.Name] = v.Value - } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(mvars) - return + return vars +} + +func respondWithJSON(w http.ResponseWriter, vars []envValue, msg string) { + mvars := make(map[string]string) + for _, v := range vars { + mvars[v.Name] = v.Value } + if msg != "" { + mvars["ResponseMessage"] = msg + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(mvars) +} - // gridTable is reused from twopcz.go. +func respondWithHTML(w http.ResponseWriter, vars []envValue, msg string) { w.Write(gridTable) w.Write([]byte("

Internal Variables

\n")) if msg != "" { From 2f281de91c1e5ed6fe17d8f53174a36a55018175 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Thu, 28 Nov 2024 18:59:55 +0530 Subject: [PATCH 12/28] Fix flakiness in `TestReadTransactionStatus` (#17277) Signed-off-by: Manan Gupta --- .github/CODEOWNERS | 1 + .../endtoend/transaction/twopc/twopc_test.go | 24 +++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6778ec7e0de..7b9b54a19bb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -28,6 +28,7 @@ go.sum @ajm188 @deepthi @harshit-gangal @mattlord @rohit-nayak-ps @systay @froui /go/test/endtoend/onlineddl @rohit-nayak-ps @shlomi-noach /go/test/endtoend/messaging @mattlord @rohit-nayak-ps @derekperkins /go/test/endtoend/schemadiff @shlomi-noach @mattlord +/go/test/endtoend/transaction @harshit-gangal @systay @frouioui @GuptaManan100 /go/test/endtoend/*throttler* @shlomi-noach @mattlord @timvaillancourt /go/test/endtoend/vtgate @harshit-gangal @systay @frouioui /go/test/endtoend/vtorc @deepthi @shlomi-noach @GuptaManan100 @timvaillancourt diff --git a/go/test/endtoend/transaction/twopc/twopc_test.go b/go/test/endtoend/transaction/twopc/twopc_test.go index 033d93f8792..df064fb16cd 100644 --- a/go/test/endtoend/transaction/twopc/twopc_test.go +++ b/go/test/endtoend/transaction/twopc/twopc_test.go @@ -1395,8 +1395,6 @@ func TestReadTransactionStatus(t *testing.T) { "insert into twopc_t1(id, col) values(6, 4)", "insert into twopc_t1(id, col) values(9, 4)", }) - // Allow enough time for the commit to have started. - time.Sleep(1 * time.Second) // Create a tablet manager client and use it to read the transaction state. tmc := grpctmclient.NewClient() @@ -1405,12 +1403,24 @@ func TestReadTransactionStatus(t *testing.T) { defer cancel() primaryTablet := getTablet(clusterInstance.Keyspaces[0].Shards[2].FindPrimaryTablet().GrpcPort) + // Wait for the transaction to show up in the unresolved list. var unresTransaction *querypb.TransactionMetadata - for _, shard := range clusterInstance.Keyspaces[0].Shards { - urtRes, err := tmc.GetUnresolvedTransactions(ctx, getTablet(shard.FindPrimaryTablet().GrpcPort), 1) - require.NoError(t, err) - if len(urtRes) > 0 { - unresTransaction = urtRes[0] + timeout := time.After(10 * time.Second) + for { + for _, shard := range clusterInstance.Keyspaces[0].Shards { + urtRes, err := tmc.GetUnresolvedTransactions(ctx, getTablet(shard.FindPrimaryTablet().GrpcPort), 1) + require.NoError(t, err) + if len(urtRes) > 0 { + unresTransaction = urtRes[0] + } + } + if unresTransaction != nil { + break + } + select { + case <-timeout: + require.Fail(t, "timed out waiting for unresolved transaction") + default: } } require.NotNil(t, unresTransaction) From aad2e469aee458443a884ad7f200cac6e215ed92 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Fri, 29 Nov 2024 00:45:43 +0530 Subject: [PATCH 13/28] Add CI for VTop example (#16007) Signed-off-by: Manan Gupta Signed-off-by: Florent Poinsard Signed-off-by: Rohit Nayak Co-authored-by: Florent Poinsard Co-authored-by: Rohit Nayak --- .github/workflows/vtop_example.yml | 97 ++++++++++++++++++++++ examples/operator/101_initial_cluster.yaml | 30 ++----- test/config.json | 11 +++ test/vtop_example.sh | 3 +- tools/get_kubectl_kind.sh | 4 +- 5 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/vtop_example.yml diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml new file mode 100644 index 00000000000..fb5ae87c101 --- /dev/null +++ b/.github/workflows/vtop_example.yml @@ -0,0 +1,97 @@ +name: vtop_example +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example') + cancel-in-progress: true + +jobs: + build: + name: VTop Example + runs-on: self-hosted + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + - 'test/**' + - '.github/workflows/vtop_example.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Install everything we need, and configure + sudo apt-get install -y eatmydata make + go mod download + + # needed for vtctldclient + - name: Build vitess + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make build + + - name: Install kubectl & kind + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make install_kubectl_kind + + - name: vtop_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 60 + run: | + source build.env + eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example \ No newline at end of file diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index c26219254f1..a4f9f447e21 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -15,7 +15,7 @@ spec: vtbackup: vitess/lite:latest vtorc: vitess/lite:latest mysqld: - mysql80Compatible: vitess/lite:latest + mysql80Compatible: mysql:8.0.30 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 @@ -155,23 +155,6 @@ stringData: # Vitess defaults ############################################################################### - # Vitess-internal database. - CREATE DATABASE IF NOT EXISTS _vt; - # Note that definitions of local_metadata and shard_metadata should be the same - # as in production which is defined in go/vt/mysqlctl/metadata_tables.go. - CREATE TABLE IF NOT EXISTS _vt.local_metadata ( - name VARCHAR(255) NOT NULL, - value VARCHAR(255) NOT NULL, - db_name VARBINARY(255) NOT NULL, - PRIMARY KEY (db_name, name) - ) ENGINE=InnoDB; - CREATE TABLE IF NOT EXISTS _vt.shard_metadata ( - name VARCHAR(255) NOT NULL, - value MEDIUMBLOB NOT NULL, - db_name VARBINARY(255) NOT NULL, - PRIMARY KEY (db_name, name) - ) ENGINE=InnoDB; - # Admin user with all privileges. CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; @@ -200,12 +183,10 @@ stringData: ON *.* TO 'vt_allprivs'@'localhost'; # User for slave replication connections. - # TODO: Should we set a password on this since it allows remote connections? CREATE USER 'vt_repl'@'%'; GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%'; - # User for Vitess filtered replication (binlog player). - # Same permissions as vt_app. + # User for Vitess VReplication (base vstreamers and vplayer). CREATE USER 'vt_filtered'@'localhost'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, @@ -213,6 +194,13 @@ stringData: SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER ON *.* TO 'vt_filtered'@'localhost'; + # User for general MySQL monitoring. + CREATE USER 'vt_monitoring'@'localhost'; + GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD + ON *.* TO 'vt_monitoring'@'localhost'; + GRANT SELECT, UPDATE, DELETE, DROP + ON performance_schema.* TO 'vt_monitoring'@'localhost'; + # custom sql is used to add custom scripts like creating users/passwords. We use it in our tests # {{custom_sql}} diff --git a/test/config.json b/test/config.json index c911232ce74..1e278546c7a 100644 --- a/test/config.json +++ b/test/config.json @@ -1238,6 +1238,17 @@ "RetryMax": 1, "Tags": [] }, + "vtop_example": { + "File": "", + "Args": [], + "Command": [ + "test/vtop_example.sh" + ], + "Manual": false, + "Shard": "", + "RetryMax": 1, + "Tags": [] + }, "vtorc_primary_failure": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtorc/primaryfailure"], diff --git a/test/vtop_example.sh b/test/vtop_example.sh index 5ff90a2be7e..c537c0f844c 100755 --- a/test/vtop_example.sh +++ b/test/vtop_example.sh @@ -482,11 +482,12 @@ EOF waitForKeyspaceToBeServing customer 80- 1 } +kind delete cluster --name kind || true # Build the docker image for vitess/lite using the local code docker build -f docker/lite/Dockerfile -t vitess/lite:pr . # Build the docker image for vitess/vtadmin using the local code -docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr . +docker build -f docker/binaries/vtadmin/Dockerfile --build-arg VT_BASE_VER=pr -t vitess/vtadmin:pr ./docker/binaries/vtadmin # Print the docker images available docker image ls diff --git a/tools/get_kubectl_kind.sh b/tools/get_kubectl_kind.sh index 57df414fdd8..169b120aaa0 100755 --- a/tools/get_kubectl_kind.sh +++ b/tools/get_kubectl_kind.sh @@ -12,7 +12,7 @@ source build.env mkdir -p "$VTROOT/bin" cd "$VTROOT/bin" -KUBE_VERSION="${KUBE_VERSION:-v1.21.1}" +KUBE_VERSION="${KUBE_VERSION:-v1.31.0}" KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" # Download kubectl if needed. @@ -28,7 +28,7 @@ ln -sf "kubectl-${KUBE_VERSION}" kubectl if ! command -v kind &> /dev/null then echo "Downloading kind..." - curl -L https://kind.sigs.k8s.io/dl/v0.12.0/kind-linux-amd64 > "kind" + curl -L https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64 > "kind" chmod +x "kind" echo "Installed kind" else From 9422e32d10eec94e355204af148053a15deef14c Mon Sep 17 00:00:00 2001 From: Rohit Nayak <57520317+rohit-nayak-ps@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:24:59 +0100 Subject: [PATCH 14/28] Tool to determine mapping from vindex and value to shard (#17290) Signed-off-by: Rohit Nayak --- tools/map-shard-for-value/Makefile | 22 ++ .../map-shard-for-value.go | 207 ++++++++++++++++++ .../map-shard-for-value.md | 47 ++++ .../map-shard-for-value_test.go | 90 ++++++++ 4 files changed, 366 insertions(+) create mode 100644 tools/map-shard-for-value/Makefile create mode 100755 tools/map-shard-for-value/map-shard-for-value.go create mode 100644 tools/map-shard-for-value/map-shard-for-value.md create mode 100644 tools/map-shard-for-value/map-shard-for-value_test.go diff --git a/tools/map-shard-for-value/Makefile b/tools/map-shard-for-value/Makefile new file mode 100644 index 00000000000..61bc88ac0ed --- /dev/null +++ b/tools/map-shard-for-value/Makefile @@ -0,0 +1,22 @@ +# Copyright 2024 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build: + go build map-shard-for-value.go + +test: + echo "1\n-1\n99" | go run map-shard-for-value.go --total_shards=4 --vindex=xxhash + +clean: + rm -f map-shard-for-value diff --git a/tools/map-shard-for-value/map-shard-for-value.go b/tools/map-shard-for-value/map-shard-for-value.go new file mode 100755 index 00000000000..18a092d1371 --- /dev/null +++ b/tools/map-shard-for-value/map-shard-for-value.go @@ -0,0 +1,207 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "context" + "encoding/hex" + "fmt" + "log" + "os" + "strconv" + "strings" + + flag "github.com/spf13/pflag" + + "vitess.io/vitess/go/vt/topo" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +/* + * This tool reads a list of values from stdin and prints the + * corresponding keyspace ID and shard for each value. It uses the given vindex + * and shard ranges to determine the shard. The vindex is expected to be a + * single-column vindex. The shard ranges are specified as a comma-separated + * list of key ranges, example "-80,80-". + * If you have uniformly distributed shards, you can specify the total number + * of shards using the -total_shards flag, and the tool will generate the shard ranges + * using the same logic as the Vitess operator does (using the key.GenerateShardRanges() function). + * + * Example usage: + * echo "1\n2\n3" | go run shard-from-id.go -vindex=hash -shards=-80,80- + * + * Currently tested only for integer values and hash/xxhash vindexes. + */ + +func mapShard(allShards []*topodata.ShardReference, ksid key.DestinationKeyspaceID) (string, error) { + foundShard := "" + addShard := func(shard string) error { + foundShard = shard + return nil + } + if err := ksid.Resolve(allShards, addShard); err != nil { + return "", fmt.Errorf("failed to resolve keyspace ID: %v:: %s", ksid.String(), err) + } + + if foundShard == "" { + return "", fmt.Errorf("no shard found for keyspace ID: %v", ksid) + } + return foundShard, nil +} + +func selectShard(vindex vindexes.Vindex, value sqltypes.Value, allShards []*topodata.ShardReference) (string, key.DestinationKeyspaceID, error) { + ctx := context.Background() + + destinations, err := vindexes.Map(ctx, vindex, nil, [][]sqltypes.Value{{value}}) + if err != nil { + return "", nil, fmt.Errorf("failed to map value to keyspace ID: %w", err) + } + + if len(destinations) != 1 { + return "", nil, fmt.Errorf("unexpected number of destinations: %d", len(destinations)) + } + + ksid, ok := destinations[0].(key.DestinationKeyspaceID) + if !ok { + return "", nil, fmt.Errorf("unexpected destination type: %T", destinations[0]) + } + + foundShard, err := mapShard(allShards, ksid) + if err != nil { + return "", nil, fmt.Errorf("failed to map shard, original value %v, keyspace id %s: %w", value, ksid, err) + } + return foundShard, ksid, nil +} + +func getValue(valueStr, valueType string) (sqltypes.Value, error) { + var value sqltypes.Value + + switch valueType { + case "int": + valueInt, err := strconv.ParseInt(valueStr, 10, 64) + if err != nil { + return value, fmt.Errorf("failed to parse int value: %w", err) + } + value = sqltypes.NewInt64(valueInt) + case "uint": + valueUint, err := strconv.ParseUint(valueStr, 10, 64) + if err != nil { + return value, fmt.Errorf("failed to parse uint value: %w", err) + } + value = sqltypes.NewUint64(valueUint) + case "string": + value = sqltypes.NewVarChar(valueStr) + default: + return value, fmt.Errorf("unsupported value type: %s", valueType) + } + + return value, nil +} + +func getShardMap(shardsCSV *string) []*topodata.ShardReference { + var allShards []*topodata.ShardReference + + for _, shard := range strings.Split(*shardsCSV, ",") { + _, keyRange, err := topo.ValidateShardName(shard) + if err != nil { + log.Fatalf("invalid shard range: %s", shard) + } + allShards = append(allShards, &topodata.ShardReference{ + Name: shard, + KeyRange: keyRange, + }) + } + return allShards +} + +type output struct { + Value string + KeyspaceID string + Shard string +} + +func processValues(scanner *bufio.Scanner, shardsCSV *string, vindexName string, valueType string) ([]output, error) { + allShards := getShardMap(shardsCSV) + + vindex, err := vindexes.CreateVindex(vindexName, vindexName, nil) + if err != nil { + return nil, fmt.Errorf("failed to create vindex: %v", err) + } + var outputs []output + for scanner.Scan() { + valueStr := scanner.Text() + if valueStr == "" { + continue + } + value, err := getValue(valueStr, valueType) + if err != nil { + return nil, fmt.Errorf("failed to get value for: %v, value_type %s:: %v", valueStr, valueType, err) + } + shard, ksid, err := selectShard(vindex, value, allShards) + if err != nil { + // ignore errors so that we can go ahead with the computation for other values + continue + } + outputs = append(outputs, output{Value: valueStr, KeyspaceID: hex.EncodeToString(ksid), Shard: shard}) + } + return outputs, nil +} + +func printOutput(outputs []output) { + fmt.Println("value,keyspaceID,shard") + for _, output := range outputs { + fmt.Printf("%s,%s,%s\n", output.Value, output.KeyspaceID, output.Shard) + } +} + +func main() { + // Explicitly configuring the logger since it was flaky in displaying logs locally without this. + log.SetOutput(os.Stderr) + log.SetFlags(log.LstdFlags) + log.SetPrefix("LOG: ") + + vindexName := flag.String("vindex", "xxhash", "name of the vindex") + shardsCSV := flag.String("shards", "", "comma-separated list of shard ranges") + totalShards := flag.Int("total_shards", 0, "total number of uniformly distributed shards") + valueType := flag.String("value_type", "int", "type of the value (int, uint, or string)") + flag.Parse() + + if *totalShards > 0 { + if *shardsCSV != "" { + log.Fatalf("cannot specify both total_shards and shards") + } + shardArr, err := key.GenerateShardRanges(*totalShards) + if err != nil { + log.Fatalf("failed to generate shard ranges: %v", err) + } + *shardsCSV = strings.Join(shardArr, ",") + } + if *shardsCSV == "" { + log.Fatal("shards or total_shards must be specified") + } + scanner := bufio.NewScanner(os.Stdin) + outputs, err := processValues(scanner, shardsCSV, *vindexName, *valueType) + if err != nil { + log.Fatalf("failed to process values: %v", err) + } + printOutput(outputs) +} diff --git a/tools/map-shard-for-value/map-shard-for-value.md b/tools/map-shard-for-value/map-shard-for-value.md new file mode 100644 index 00000000000..17daf7f5fe5 --- /dev/null +++ b/tools/map-shard-for-value/map-shard-for-value.md @@ -0,0 +1,47 @@ +## Map Shard for Value Tool + +### Overview + +The `map-shard-for-value` tool maps a given value to a specific shard. This tool helps in determining +which shard a particular value belongs to, based on the vindex algorithm and shard ranges. + +### Features +- + +- Allows specifying the vindex type (e.g., `hash`, `xxhash`). +- Allows specifying the shard list of (for uniformly distributed shard ranges) the total number of shards to generate. +- Designed as a _filter_: Reads input values from `stdin` and outputs the corresponding shard information, so it can be + used to map values from a file or another program. + +### Usage + +```sh +make build +``` + +```sh +echo "1\n-1\n99" | ./map-shard-for-value --total_shards=4 --vindex=xxhash +value,keyspaceID,shard +1,d46405367612b4b7,c0- +-1,d8e2a6a7c8c7623d,c0- +99,200533312244abca,-40 + +echo "1\n-1\n99" | ./map-shard-for-value --vindex=hash --shards="-80,80-" +value,keyspaceID,shard +1,166b40b44aba4bd6,-80 +-1,355550b2150e2451,-80 +99,2c40ad56f4593c47,-80 +``` + +#### Flags + +- `--vindex`: Specifies the name of the vindex to use (e.g., `hash`, `xxhash`) (default `xxhash`) + +One (and only one) of these is required: + +- `--shards`: Comma-separated list of shard ranges +- `--total_shards`: Total number of shards, only if shards are uniformly distributed + +Optional: +- `--value_type`: Type of the value to map, one of int, uint, string (default `int`) + diff --git a/tools/map-shard-for-value/map-shard-for-value_test.go b/tools/map-shard-for-value/map-shard-for-value_test.go new file mode 100644 index 00000000000..ca014818bb9 --- /dev/null +++ b/tools/map-shard-for-value/map-shard-for-value_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestProcess(t *testing.T) { + type testCase struct { + name string + shardsCSV string + vindexType string + values []int + valueType string + expected []output + } + testCases := []testCase{ + { + name: "hash,2 shards", + shardsCSV: "-80,80-", + vindexType: "hash", + values: []int{1, 99}, + valueType: "int", + expected: []output{ + { + Value: "1", + KeyspaceID: "166b40b44aba4bd6", + Shard: "-80", + }, + { + Value: "99", + KeyspaceID: "2c40ad56f4593c47", + Shard: "-80", + }, + }, + }, + { + name: "xxhash,4 shards", + shardsCSV: "-40,40-80,80-c0,c0-", + vindexType: "xxhash", + values: []int{1, 99}, + valueType: "int", + expected: []output{ + { + Value: "1", + KeyspaceID: "d46405367612b4b7", + Shard: "c0-", + }, + { + Value: "99", + KeyspaceID: "200533312244abca", + Shard: "-40", + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var input strings.Builder + for _, num := range tc.values { + fmt.Fprintf(&input, "%d\n", num) + } + reader := strings.NewReader(input.String()) + scanner := bufio.NewScanner(reader) + got, err := processValues(scanner, &tc.shardsCSV, tc.vindexType, tc.valueType) + require.NoError(t, err) + require.EqualValues(t, tc.expected, got) + }) + } +} From 68b25b30f210511eda149fe96871704ec800ab18 Mon Sep 17 00:00:00 2001 From: Noble Mittal <62551163+beingnoble03@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:02:23 +0530 Subject: [PATCH 15/28] VTAdmin: Support for schema migrations view/create (#17134) Signed-off-by: Noble Mittal --- go/vt/proto/vtadmin/vtadmin.pb.go | 35 ++- go/vt/proto/vtadmin/vtadmin_vtproto.pb.go | 88 ++++++ go/vt/vtadmin/api.go | 26 ++ go/vt/vtadmin/http/schema_migrations.go | 15 +- proto/vtadmin.proto | 6 +- web/vtadmin/src/api/http.ts | 38 +++ web/vtadmin/src/components/App.tsx | 12 + web/vtadmin/src/components/NavRail.tsx | 3 + .../createWorkflow => dialog}/ErrorDialog.tsx | 4 +- .../components/routes/SchemaMigrations.tsx | 195 +++++++++++++ .../CreateSchemaMigration.module.scss | 30 ++ .../CreateSchemaMigration.tsx | 270 ++++++++++++++++++ .../createWorkflow/CreateMaterialize.tsx | 2 +- .../createWorkflow/CreateMoveTables.tsx | 2 +- .../routes/createWorkflow/CreateReshard.tsx | 2 +- web/vtadmin/src/hooks/api.ts | 24 ++ web/vtadmin/src/proto/vtadmin.d.ts | 12 + web/vtadmin/src/proto/vtadmin.js | 48 +++- web/vtadmin/src/util/schemaMigrations.ts | 31 ++ 19 files changed, 825 insertions(+), 18 deletions(-) rename web/vtadmin/src/components/{routes/createWorkflow => dialog}/ErrorDialog.tsx (94%) create mode 100644 web/vtadmin/src/components/routes/SchemaMigrations.tsx create mode 100644 web/vtadmin/src/components/routes/createSchemaMigration/CreateSchemaMigration.module.scss create mode 100644 web/vtadmin/src/components/routes/createSchemaMigration/CreateSchemaMigration.tsx create mode 100644 web/vtadmin/src/util/schemaMigrations.ts diff --git a/go/vt/proto/vtadmin/vtadmin.pb.go b/go/vt/proto/vtadmin/vtadmin.pb.go index e85385ec409..8b6a6997c8d 100644 --- a/go/vt/proto/vtadmin/vtadmin.pb.go +++ b/go/vt/proto/vtadmin/vtadmin.pb.go @@ -1203,8 +1203,12 @@ type ApplySchemaRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ClusterId string `protobuf:"bytes,1,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` - Request *vtctldata.ApplySchemaRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + ClusterId string `protobuf:"bytes,1,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + // Request.Sql will be overriden by this Sql field. + Sql string `protobuf:"bytes,2,opt,name=sql,proto3" json:"sql,omitempty"` + // Request.CallerId will be overriden by this CallerId field. + CallerId string `protobuf:"bytes,3,opt,name=caller_id,json=callerId,proto3" json:"caller_id,omitempty"` + Request *vtctldata.ApplySchemaRequest `protobuf:"bytes,4,opt,name=request,proto3" json:"request,omitempty"` } func (x *ApplySchemaRequest) Reset() { @@ -1244,6 +1248,20 @@ func (x *ApplySchemaRequest) GetClusterId() string { return "" } +func (x *ApplySchemaRequest) GetSql() string { + if x != nil { + return x.Sql + } + return "" +} + +func (x *ApplySchemaRequest) GetCallerId() string { + if x != nil { + return x.CallerId + } + return "" +} + func (x *ApplySchemaRequest) GetRequest() *vtctldata.ApplySchemaRequest { if x != nil { return x.Request @@ -7731,11 +7749,14 @@ var file_vtadmin_proto_rawDesc = []byte{ 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0x6c, 0x0a, 0x12, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, + 0x22, 0x9b, 0x01, 0x0a, 0x12, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x71, 0x6c, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x61, 0x6c, 0x6c, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x61, 0x6c, + 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x80, 0x01, 0x0a, 0x1c, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x4d, diff --git a/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go b/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go index bc0746b7b8a..82cca2cea06 100644 --- a/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go +++ b/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go @@ -454,6 +454,8 @@ func (m *ApplySchemaRequest) CloneVT() *ApplySchemaRequest { } r := new(ApplySchemaRequest) r.ClusterId = m.ClusterId + r.Sql = m.Sql + r.CallerId = m.CallerId r.Request = m.Request.CloneVT() if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) @@ -4038,6 +4040,20 @@ func (m *ApplySchemaRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= size i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) i-- + dAtA[i] = 0x22 + } + if len(m.CallerId) > 0 { + i -= len(m.CallerId) + copy(dAtA[i:], m.CallerId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.CallerId))) + i-- + dAtA[i] = 0x1a + } + if len(m.Sql) > 0 { + i -= len(m.Sql) + copy(dAtA[i:], m.Sql) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Sql))) + i-- dAtA[i] = 0x12 } if len(m.ClusterId) > 0 { @@ -10321,6 +10337,14 @@ func (m *ApplySchemaRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + l = len(m.Sql) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.CallerId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } if m.Request != nil { l = m.Request.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -15873,6 +15897,70 @@ func (m *ApplySchemaRequest) UnmarshalVT(dAtA []byte) error { m.ClusterId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sql", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sql = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CallerId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CallerId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Request", wireType) } diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index cef8816504a..4f91459d9ed 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -59,6 +59,7 @@ import ( "vitess.io/vitess/go/vt/vtadmin/rbac" "vitess.io/vitess/go/vt/vtadmin/sort" "vitess.io/vitess/go/vt/vtadmin/vtadminproto" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" "vitess.io/vitess/go/vt/vtctl/workflow" "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/vt/vterrors" @@ -488,6 +489,31 @@ func (api *API) ApplySchema(ctx context.Context, req *vtadminpb.ApplySchemaReque return nil, err } + // Parser with default options. New() itself initializes with default MySQL version. + parser, err := sqlparser.New(sqlparser.Options{ + TruncateUILen: 512, + TruncateErrLen: 0, + }) + if err != nil { + return nil, err + } + + // Split the sql statement received from request. + sqlParts, err := parser.SplitStatementToPieces(req.Sql) + if err != nil { + return nil, err + } + + req.Request.Sql = sqlParts + + // Set the callerID if not empty. + if req.CallerId != "" { + req.Request.CallerId = &vtrpcpb.CallerID{Principal: req.CallerId} + } + + // Set the default wait replicas timeout. + req.Request.WaitReplicasTimeout = protoutil.DurationToProto(grpcvtctldserver.DefaultWaitReplicasTimeout) + return c.ApplySchema(ctx, req.Request) } diff --git a/go/vt/vtadmin/http/schema_migrations.go b/go/vt/vtadmin/http/schema_migrations.go index e0207989648..3da6026fe9f 100644 --- a/go/vt/vtadmin/http/schema_migrations.go +++ b/go/vt/vtadmin/http/schema_migrations.go @@ -34,19 +34,26 @@ func ApplySchema(ctx context.Context, r Request, api *API) *JSONResponse { decoder := json.NewDecoder(r.Body) defer r.Body.Close() - var req vtctldatapb.ApplySchemaRequest - if err := decoder.Decode(&req); err != nil { + var body struct { + Sql string `json:"sql"` + CallerId string `json:"caller_id"` + Request vtctldatapb.ApplySchemaRequest `json:"request"` + } + + if err := decoder.Decode(&body); err != nil { return NewJSONResponse(nil, &errors.BadRequest{ Err: err, }) } vars := mux.Vars(r.Request) - req.Keyspace = vars["keyspace"] + body.Request.Keyspace = vars["keyspace"] resp, err := api.server.ApplySchema(ctx, &vtadminpb.ApplySchemaRequest{ ClusterId: vars["cluster_id"], - Request: &req, + Sql: body.Sql, + CallerId: body.CallerId, + Request: &body.Request, }) return NewJSONResponse(resp, err) diff --git a/proto/vtadmin.proto b/proto/vtadmin.proto index 78f086ec345..963d1fa5779 100644 --- a/proto/vtadmin.proto +++ b/proto/vtadmin.proto @@ -388,7 +388,11 @@ message WorkflowSwitchTrafficRequest { message ApplySchemaRequest { string cluster_id = 1; - vtctldata.ApplySchemaRequest request = 2; + // Request.Sql will be overriden by this Sql field. + string sql = 2; + // Request.CallerId will be overriden by this CallerId field. + string caller_id = 3; + vtctldata.ApplySchemaRequest request = 4; } message CancelSchemaMigrationRequest { diff --git a/web/vtadmin/src/api/http.ts b/web/vtadmin/src/api/http.ts index 3f75330d240..674df961ef0 100644 --- a/web/vtadmin/src/api/http.ts +++ b/web/vtadmin/src/api/http.ts @@ -1068,3 +1068,41 @@ export const showVDiff = async ({ clusterID, request }: ShowVDiffParams) => { return vtadmin.VDiffShowResponse.create(result); }; + +export const fetchSchemaMigrations = async (request: vtadmin.IGetSchemaMigrationsRequest) => { + const { result } = await vtfetch(`/api/migrations/`, { + body: JSON.stringify(request), + method: 'post', + }); + + const err = vtadmin.GetSchemaMigrationsResponse.verify(result); + if (err) throw Error(err); + + return vtadmin.GetSchemaMigrationsResponse.create(result); +}; + +export interface ApplySchemaParams { + clusterID: string; + keyspace: string; + callerID: string; + sql: string; + request: vtctldata.IApplySchemaRequest; +} + +export const applySchema = async ({ clusterID, keyspace, callerID, sql, request }: ApplySchemaParams) => { + const body = { + sql, + caller_id: callerID, + request, + }; + + const { result } = await vtfetch(`/api/migration/${clusterID}/${keyspace}`, { + body: JSON.stringify(body), + method: 'post', + }); + + const err = vtctldata.ApplySchemaResponse.verify(result); + if (err) throw Error(err); + + return vtctldata.ApplySchemaResponse.create(result); +}; diff --git a/web/vtadmin/src/components/App.tsx b/web/vtadmin/src/components/App.tsx index ef27a35dc95..3bb41ea35f0 100644 --- a/web/vtadmin/src/components/App.tsx +++ b/web/vtadmin/src/components/App.tsx @@ -45,6 +45,8 @@ import { Transactions } from './routes/Transactions'; import { Transaction } from './routes/transaction/Transaction'; import { CreateReshard } from './routes/createWorkflow/CreateReshard'; import { CreateMaterialize } from './routes/createWorkflow/CreateMaterialize'; +import { SchemaMigrations } from './routes/SchemaMigrations'; +import { CreateSchemaMigration } from './routes/createSchemaMigration/CreateSchemaMigration'; export const App = () => { return ( @@ -140,6 +142,16 @@ export const App = () => { + + + + + {!isReadOnlyMode() && ( + + + + )} + diff --git a/web/vtadmin/src/components/NavRail.tsx b/web/vtadmin/src/components/NavRail.tsx index 9f9e1bf1681..b30cd165684 100644 --- a/web/vtadmin/src/components/NavRail.tsx +++ b/web/vtadmin/src/components/NavRail.tsx @@ -65,6 +65,9 @@ export const NavRail = () => {
    +
  • + +
    • diff --git a/web/vtadmin/src/components/routes/createWorkflow/ErrorDialog.tsx b/web/vtadmin/src/components/dialog/ErrorDialog.tsx similarity index 94% rename from web/vtadmin/src/components/routes/createWorkflow/ErrorDialog.tsx rename to web/vtadmin/src/components/dialog/ErrorDialog.tsx index 25ac5dedb0b..087876e4cd2 100644 --- a/web/vtadmin/src/components/routes/createWorkflow/ErrorDialog.tsx +++ b/web/vtadmin/src/components/dialog/ErrorDialog.tsx @@ -14,8 +14,8 @@ * limitations under the License. */ import React from 'react'; -import Dialog from '../../dialog/Dialog'; -import { Icon, Icons } from '../../Icon'; +import Dialog from './Dialog'; +import { Icon, Icons } from '../Icon'; export interface ErrorDialogProps { errorTitle?: string; diff --git a/web/vtadmin/src/components/routes/SchemaMigrations.tsx b/web/vtadmin/src/components/routes/SchemaMigrations.tsx new file mode 100644 index 00000000000..1761d26de49 --- /dev/null +++ b/web/vtadmin/src/components/routes/SchemaMigrations.tsx @@ -0,0 +1,195 @@ +/** + * Copyright 2024 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { useEffect, useState } from 'react'; +import { useKeyspaces, useSchemaMigrations } from '../../hooks/api'; +import { DataCell } from '../dataTable/DataCell'; +import { DataTable } from '../dataTable/DataTable'; +import { ContentContainer } from '../layout/ContentContainer'; +import { WorkspaceHeader } from '../layout/WorkspaceHeader'; +import { WorkspaceTitle } from '../layout/WorkspaceTitle'; +import { QueryLoadingPlaceholder } from '../placeholders/QueryLoadingPlaceholder'; +import { useDocumentTitle } from '../../hooks/useDocumentTitle'; +import { vtadmin } from '../../proto/vtadmin'; +import { Select } from '../inputs/Select'; +import { ShardLink } from '../links/ShardLink'; +import { formatDateTime } from '../../util/time'; +import { ReadOnlyGate } from '../ReadOnlyGate'; +import { formatSchemaMigrationStatus } from '../../util/schemaMigrations'; +import { Link } from 'react-router-dom'; +import { TabletLink } from '../links/TabletLink'; +import { formatAlias } from '../../util/tablets'; +import { useURLQuery } from '../../hooks/useURLQuery'; + +const COLUMNS = ['UUID', 'Status', 'DDL Action', 'Timestamps', 'Stage', 'Progress']; + +export const SchemaMigrations = () => { + useDocumentTitle('Schema Migrations'); + + const { query, replaceQuery } = useURLQuery(); + const urlKeyspace = query['keyspace']; + const urlCluster = query['cluster']; + + const keyspacesQuery = useKeyspaces(); + const { data: keyspaces = [], ...ksQuery } = keyspacesQuery; + + const [selectedKeyspace, setSelectedKeypsace] = useState(); + + const request: vtadmin.IGetSchemaMigrationsRequest = { + cluster_requests: [ + { + cluster_id: selectedKeyspace && selectedKeyspace.cluster?.id, + request: { + keyspace: selectedKeyspace && selectedKeyspace.keyspace?.name, + }, + }, + ], + }; + + const schemaMigrationsQuery = useSchemaMigrations(request, { + enabled: !!selectedKeyspace, + }); + + const schemaMigrations = schemaMigrationsQuery.data ? schemaMigrationsQuery.data.schema_migrations : []; + + const handleKeyspaceChange = (ks: vtadmin.Keyspace | null | undefined) => { + setSelectedKeypsace(ks); + + if (ks) { + replaceQuery({ keyspace: ks.keyspace?.name, cluster: ks.cluster?.id }); + } else { + replaceQuery({ keyspace: undefined, cluster: undefined }); + } + }; + + useEffect(() => { + if (urlKeyspace && urlCluster) { + const keyspace = keyspaces.find( + (ks) => ks.cluster?.id === String(urlCluster) && ks.keyspace?.name === String(urlKeyspace) + ); + + if (keyspace) { + setSelectedKeypsace(keyspace); + } else if (!ksQuery.isLoading) { + replaceQuery({ keyspace: undefined, cluster: undefined }); + } + } else { + setSelectedKeypsace(undefined); + } + }, [urlKeyspace, urlCluster, keyspaces, ksQuery.isLoading, replaceQuery]); + + const renderRows = (rows: vtadmin.ISchemaMigration[]) => { + return rows.map((row) => { + const migrationInfo = row.schema_migration; + + if (!migrationInfo) return <>; + + return ( + + +
    {migrationInfo.uuid}
    +
    + Tablet{' '} + + {formatAlias(migrationInfo.tablet)} + +
    +
    + Shard{' '} + + {`${migrationInfo.keyspace}/${migrationInfo.shard}`} + +
    +     + +
    {formatSchemaMigrationStatus(migrationInfo)}
    +     + {migrationInfo.ddl_action ? migrationInfo.ddl_action : '-'} + + {migrationInfo.added_at && ( +
    + Added + {formatDateTime(migrationInfo.added_at?.seconds)} +
    + )} + {migrationInfo.requested_at && ( +
    + Requested + {formatDateTime(migrationInfo.requested_at?.seconds)} +
    + )} + {migrationInfo.started_at && ( +
    + Started + {formatDateTime(migrationInfo.started_at?.seconds)} +
    + )} + {migrationInfo.completed_at && ( +
    + Completed + {formatDateTime(migrationInfo.completed_at?.seconds)} +
    + )} +     + {migrationInfo.stage ? migrationInfo.stage : '-'} + {migrationInfo.progress ? `${migrationInfo.progress}%` : '-'} + + ); + }); + }; + + return ( +
    + +
    + Schema Migrations + +
    + + Create Schema Migration Request + +
    +     +
    +     + + +
    + ks?.keyspace?.name || ''} + items={clusterKeyspaces} + label="Keyspace" + onChange={(ks) => setFormData({ ...formData, keyspace: ks?.keyspace?.name || '' })} + placeholder={keyspacesQuery.isLoading ? 'Loading keyspaces...' : 'Select a keyspace'} + renderItem={(ks) => `${ks?.keyspace?.name}`} + selectedItem={selectedKeyspace} + /> + +
    + +