From 1f2de683b890985605d11a606a8804117dc93d59 Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 11:32:05 +0300 Subject: [PATCH 1/2] [release-16.0] OnlineDDL: fix nil 'completed_timestamp' for cancelled migrations (#13928) (#13936) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com> Co-authored-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../scheduler/onlineddl_scheduler_test.go | 3 ++ .../onlineddl/vrepl/onlineddl_vrepl_test.go | 13 ++++--- go/test/endtoend/onlineddl/vtgate_util.go | 38 ++++++++++++++++++- go/test/endtoend/onlineddl/vttablet_util.go | 3 +- go/vt/vttablet/onlineddl/executor.go | 23 ++++++++--- go/vt/vttablet/onlineddl/schema.go | 5 ++- 6 files changed, 69 insertions(+), 16 deletions(-) diff --git a/go/test/endtoend/onlineddl/scheduler/onlineddl_scheduler_test.go b/go/test/endtoend/onlineddl/scheduler/onlineddl_scheduler_test.go index 2911fbb80b2..2b068df643d 100644 --- a/go/test/endtoend/onlineddl/scheduler/onlineddl_scheduler_test.go +++ b/go/test/endtoend/onlineddl/scheduler/onlineddl_scheduler_test.go @@ -265,6 +265,9 @@ func TestSchemaChange(t *testing.T) { t.Run("summary: validate sequential migration IDs", func(t *testing.T) { onlineddl.ValidateSequentialMigrationIDs(t, &vtParams, shards) }) + t.Run("summary: validate completed_timestamp", func(t *testing.T) { + onlineddl.ValidateCompletedTimestamp(t, &vtParams) + }) } func testScheduler(t *testing.T) { diff --git a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go index ec6eed14f17..10902baf1d4 100644 --- a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go +++ b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go @@ -213,7 +213,7 @@ func TestMain(m *testing.M) { if err != nil { fmt.Printf("%v\n", err) os.Exit(1) - } else { + } else { // nolint:revive os.Exit(exitcode) } @@ -537,11 +537,11 @@ func TestSchemaChange(t *testing.T) { onlineddl.CheckMigrationStatus(t, &vtParams, shards, uuid, schema.OnlineDDLStatusRunning) }) t.Run("wait for vreplication to run on shard -80", func(t *testing.T) { - vreplStatus := onlineddl.WaitForVReplicationStatus(t, &vtParams, currentPrimaryTablet, uuid, normalMigrationWait, "Copying", "Running") + vreplStatus := onlineddl.WaitForVReplicationStatus(t, currentPrimaryTablet, uuid, normalMigrationWait, "Copying", "Running") require.Contains(t, []string{"Copying", "Running"}, vreplStatus) }) t.Run("wait for vreplication to run on shard 80-", func(t *testing.T) { - vreplStatus := onlineddl.WaitForVReplicationStatus(t, &vtParams, shards[1].Vttablets[0], uuid, normalMigrationWait, "Copying", "Running") + vreplStatus := onlineddl.WaitForVReplicationStatus(t, shards[1].Vttablets[0], uuid, normalMigrationWait, "Copying", "Running") require.Contains(t, []string{"Copying", "Running"}, vreplStatus) }) t.Run("check status again", func(t *testing.T) { @@ -646,11 +646,11 @@ func TestSchemaChange(t *testing.T) { onlineddl.CheckMigrationStatus(t, &vtParams, shards, uuid, schema.OnlineDDLStatusRunning) }) t.Run("wait for vreplication to run on shard -80", func(t *testing.T) { - vreplStatus := onlineddl.WaitForVReplicationStatus(t, &vtParams, currentPrimaryTablet, uuid, normalMigrationWait, "Copying", "Running") + vreplStatus := onlineddl.WaitForVReplicationStatus(t, currentPrimaryTablet, uuid, normalMigrationWait, "Copying", "Running") require.Contains(t, []string{"Copying", "Running"}, vreplStatus) }) t.Run("wait for vreplication to run on shard 80-", func(t *testing.T) { - vreplStatus := onlineddl.WaitForVReplicationStatus(t, &vtParams, shards[1].Vttablets[0], uuid, normalMigrationWait, "Copying", "Running") + vreplStatus := onlineddl.WaitForVReplicationStatus(t, shards[1].Vttablets[0], uuid, normalMigrationWait, "Copying", "Running") require.Contains(t, []string{"Copying", "Running"}, vreplStatus) }) t.Run("check status again", func(t *testing.T) { @@ -888,6 +888,9 @@ func TestSchemaChange(t *testing.T) { t.Run("summary: validate sequential migration IDs", func(t *testing.T) { onlineddl.ValidateSequentialMigrationIDs(t, &vtParams, shards) }) + t.Run("summary: validate completed_timestamp", func(t *testing.T) { + onlineddl.ValidateCompletedTimestamp(t, &vtParams) + }) } func insertRow(t *testing.T) { diff --git a/go/test/endtoend/onlineddl/vtgate_util.go b/go/test/endtoend/onlineddl/vtgate_util.go index 75c35061da9..5052065082b 100644 --- a/go/test/endtoend/onlineddl/vtgate_util.go +++ b/go/test/endtoend/onlineddl/vtgate_util.go @@ -39,6 +39,14 @@ import ( "github.com/stretchr/testify/require" ) +var ( + testsStartupTime time.Time +) + +func init() { + testsStartupTime = time.Now() +} + // VtgateExecQuery runs a query on VTGate using given query params func VtgateExecQuery(t *testing.T, vtParams *mysql.ConnParams, query string, expectError string) *sqltypes.Result { t.Helper() @@ -354,7 +362,7 @@ func WaitForThrottlerStatusEnabled(t *testing.T, tablet *cluster.Vttablet, timeo jsonPath := "IsEnabled" url := fmt.Sprintf("http://localhost:%d/throttler/status", tablet.HTTPPort) - ctx, cancel := context.WithTimeout(context.Background(), throttlerConfigTimeout) + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() ticker := time.NewTicker(time.Second) @@ -428,3 +436,31 @@ func ValidateSequentialMigrationIDs(t *testing.T, vtParams *mysql.ConnParams, sh assert.Equalf(t, count, shardMax[shard]-shardMin[shard]+1, "mismatch: shared=%v, count=%v, min=%v, max=%v", shard, count, shardMin[shard], shardMax[shard]) } } + +// ValidateCompletedTimestamp ensures that any migration in `cancelled`, `completed`, `failed` statuses +// has a non-nil and valid `completed_timestamp` value. +func ValidateCompletedTimestamp(t *testing.T, vtParams *mysql.ConnParams) { + require.False(t, testsStartupTime.IsZero()) + r := VtgateExecQuery(t, vtParams, "show vitess_migrations", "") + + completedTimestampNumValidations := 0 + for _, row := range r.Named().Rows { + migrationStatus := row.AsString("migration_status", "") + require.NotEmpty(t, migrationStatus) + switch migrationStatus { + case string(schema.OnlineDDLStatusComplete), + string(schema.OnlineDDLStatusFailed), + string(schema.OnlineDDLStatusCancelled): + { + assert.False(t, row["completed_timestamp"].IsNull()) + // Also make sure the timestamp is "real", and that it is recent. + timestamp := row.AsString("completed_timestamp", "") + completedTime, err := time.Parse(sqltypes.TimestampFormat, timestamp) + assert.NoError(t, err) + assert.Greater(t, completedTime.Unix(), testsStartupTime.Unix()) + completedTimestampNumValidations++ + } + } + } + assert.NotZero(t, completedTimestampNumValidations) +} diff --git a/go/test/endtoend/onlineddl/vttablet_util.go b/go/test/endtoend/onlineddl/vttablet_util.go index 3d4ded89dac..b4669490f63 100644 --- a/go/test/endtoend/onlineddl/vttablet_util.go +++ b/go/test/endtoend/onlineddl/vttablet_util.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" @@ -31,7 +30,7 @@ import ( ) // WaitForVReplicationStatus waits for a vreplication stream to be in one of given states, or timeout -func WaitForVReplicationStatus(t *testing.T, vtParams *mysql.ConnParams, tablet *cluster.Vttablet, uuid string, timeout time.Duration, expectStatuses ...string) (status string) { +func WaitForVReplicationStatus(t *testing.T, tablet *cluster.Vttablet, uuid string, timeout time.Duration, expectStatuses ...string) (status string) { query, err := sqlparser.ParseAndBind("select state from _vt.vreplication where workflow=%a", sqltypes.StringBindVariable(uuid), diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index 413b5f012ee..62e22cf3fda 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -99,6 +99,13 @@ var vexecInsertTemplates = []string{ 'val1', 'val2', 'val3', 'val4', 'val5', 'val6', 'val7', 'val8', 'val9', FROM_UNIXTIME(0), 'vala', 'valb' )`, } +var ( + // fixCompletedTimestampDone fixes a nil `completed_tiemstamp` columns, see + // https://github.com/vitessio/vitess/issues/13927 + // The fix is in release-18.0 + // TODO: remove in release-19.0 + fixCompletedTimestampDone bool +) var emptyResult = &sqltypes.Result{} var acceptableDropTableIfExistsErrorCodes = []int{mysql.ERCantFindFile, mysql.ERNoSuchTable} @@ -3852,13 +3859,17 @@ func (e *Executor) gcArtifacts(ctx context.Context) error { e.migrationMutex.Lock() defer e.migrationMutex.Unlock() - if _, err := e.execQuery(ctx, sqlFixCompletedTimestamp); err != nil { - // This query fixes a bug where stale migrations were marked as 'failed' without updating 'completed_timestamp' - // see https://github.com/vitessio/vitess/issues/8499 - // Running this query retroactively sets completed_timestamp - // This 'if' clause can be removed in version v13 - return err + // v18 fix (backported to 16). Remove in v19 + if !fixCompletedTimestampDone { + if _, err := e.execQuery(ctx, sqlFixCompletedTimestamp); err != nil { + // This query fixes a bug where stale migrations were marked as 'cancelled' or 'failed' without updating 'completed_timestamp' + // Running this query retroactively sets completed_timestamp + // This fix is created in v18 and can be removed in v19 + return err + } + fixCompletedTimestampDone = true } + query, err := sqlparser.ParseAndBind(sqlSelectUncollectedArtifacts, sqltypes.Int64BindVariable(int64((retainOnlineDDLTables).Seconds())), ) diff --git a/go/vt/vttablet/onlineddl/schema.go b/go/vt/vttablet/onlineddl/schema.go index adf550dabfe..c4a6d96953b 100644 --- a/go/vt/vttablet/onlineddl/schema.go +++ b/go/vt/vttablet/onlineddl/schema.go @@ -66,7 +66,8 @@ const ( migration_uuid=%a ` sqlUpdateMigrationStatusFailedOrCancelled = `UPDATE _vt.schema_migrations - SET migration_status=IF(cancelled_timestamp IS NULL, 'failed', 'cancelled') + SET migration_status=IF(cancelled_timestamp IS NULL, 'failed', 'cancelled'), + completed_timestamp=NOW(6) WHERE migration_uuid=%a ` @@ -349,7 +350,7 @@ const ( SET completed_timestamp=NOW(6) WHERE - migration_status='failed' + migration_status IN ('cancelled', 'failed') AND cleanup_timestamp IS NULL AND completed_timestamp IS NULL ` From 7012754a4696465515aa971747831eb0deca4eb6 Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:12:11 -0400 Subject: [PATCH 2/2] [release-16.0] Use Debian Bullseye in Bootstrap (#13757) (#13921) Co-authored-by: Florent Poinsard <35779988+frouioui@users.noreply.github.com> Co-authored-by: Florent Poinsard --- Makefile | 5 ++++- docker/base/Dockerfile | 2 +- docker/base/Dockerfile.mysql57 | 2 +- docker/base/Dockerfile.percona57 | 2 +- docker/base/Dockerfile.percona80 | 2 +- docker/bootstrap/CHANGELOG.md | 6 +++++- docker/bootstrap/Dockerfile.common | 2 +- docker/bootstrap/Dockerfile.mysql80 | 2 +- docker/bootstrap/Dockerfile.percona57 | 7 +++---- docker/bootstrap/Dockerfile.percona80 | 4 ++-- docker/lite/Dockerfile.mysql57 | 2 +- docker/lite/Dockerfile.mysql80 | 2 +- docker/lite/Dockerfile.percona57 | 2 +- docker/lite/Dockerfile.percona80 | 2 +- docker/lite/Dockerfile.testing | 2 +- docker/lite/Dockerfile.ubi7.mysql57 | 3 ++- docker/lite/Dockerfile.ubi7.mysql80 | 7 +++++-- docker/lite/Dockerfile.ubi7.percona57 | 2 +- docker/lite/Dockerfile.ubi7.percona80 | 4 +++- docker/lite/Dockerfile.ubi8.arm64.mysql80 | 7 +++++-- docker/lite/Dockerfile.ubi8.mysql80 | 7 +++++-- docker/local/Dockerfile | 2 +- docker/vttestserver/Dockerfile.mysql57 | 4 ++-- docker/vttestserver/Dockerfile.mysql80 | 4 ++-- examples/compose/vtcompose/vtcompose.go | 6 +++--- test.go | 2 +- test/templates/dockerfile.tpl | 2 +- 27 files changed, 56 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index a61df12ffa9..b34e388064b 100644 --- a/Makefile +++ b/Makefile @@ -275,7 +275,7 @@ $(PROTO_GO_OUTS): minimaltools install_protoc-gen-go proto/*.proto # This rule builds the bootstrap images for all flavors. DOCKER_IMAGES_FOR_TEST = mysql57 mysql80 percona57 percona80 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST) -BOOTSTRAP_VERSION=14.5 +BOOTSTRAP_VERSION=14.6 ensure_bootstrap_version: find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \; sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go @@ -328,6 +328,9 @@ $(DOCKER_LITE_TARGETS): docker_lite_%: docker_lite_all: docker_lite $(DOCKER_LITE_TARGETS) +docker_lite_push: + for i in $(DOCKER_LITE_SUFFIX); do echo "pushing lite image: $$i"; docker push vitess/lite:$$i || exit 1; done + docker_local: ${call build_docker_image,docker/local/Dockerfile,vitess/local} diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 2223d7d1fd6..35bf17f1aac 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" FROM "${image}" diff --git a/docker/base/Dockerfile.mysql57 b/docker/base/Dockerfile.mysql57 index b0c9af936a4..a5c09a33f8e 100644 --- a/docker/base/Dockerfile.mysql57 +++ b/docker/base/Dockerfile.mysql57 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" FROM "${image}" diff --git a/docker/base/Dockerfile.percona57 b/docker/base/Dockerfile.percona57 index c54ba510b3e..2a367f76684 100644 --- a/docker/base/Dockerfile.percona57 +++ b/docker/base/Dockerfile.percona57 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-percona57" FROM "${image}" diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 index 83b256c6737..7d599f54e29 100644 --- a/docker/base/Dockerfile.percona80 +++ b/docker/base/Dockerfile.percona80 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-percona80" FROM "${image}" diff --git a/docker/bootstrap/CHANGELOG.md b/docker/bootstrap/CHANGELOG.md index 36ad6e8dd76..dbd611fbb5f 100644 --- a/docker/bootstrap/CHANGELOG.md +++ b/docker/bootstrap/CHANGELOG.md @@ -72,4 +72,8 @@ List of changes between bootstrap image versions. ## [14.5] - 2023-06-07 ### Changes -- Update build to golang 1.20.5 \ No newline at end of file +- Update build to golang 1.20.5 + +## [14.6] - 2023-09-06 +### Changes +- Use Debian Bullseye as base bootstrap image in order to continue upgrading the Golang version \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 5c6e4da5930..ba3b6b0dc1e 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 golang:1.20.5-buster +FROM --platform=linux/amd64 golang:1.20.5-bullseye # Install Vitess build dependencies RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index e064c638d99..dbb16bdbebc 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -8,7 +8,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyser for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 467B942D3A79BD29 && break; done && \ add-apt-repository 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' && \ for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ - echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list && \ + echo 'deb http://repo.percona.com/apt bullseye main' > /etc/apt/sources.list.d/percona.list && \ { \ echo debconf debconf/frontend select Noninteractive; \ echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \ diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 2d8beb5e95d..febe09fd8bf 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -5,16 +5,15 @@ FROM --platform=linux/amd64 "${image}" # Install Percona 5.7 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ - add-apt-repository 'deb http://repo.percona.com/apt buster main' && \ + add-apt-repository 'deb http://repo.percona.com/apt bullseye main' && \ { \ echo debconf debconf/frontend select Noninteractive; \ echo percona-server-server-5.7 percona-server-server/root_password password 'unused'; \ echo percona-server-server-5.7 percona-server-server/root_password_again password 'unused'; \ } | debconf-set-selections && \ apt-get update && \ - apt-get install -y --no-install-recommends \ - percona-server-server-5.7 \ - libperconaserverclient20-dev percona-xtrabackup-24 && \ + apt-get install -y --no-install-recommends percona-server-server-5.7 && \ + apt-get install -y --no-install-recommends libperconaserverclient20-dev percona-xtrabackup-24 && \ rm -rf /var/lib/apt/lists/* # Bootstrap Vitess diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index 5dadc32cd0a..446ec554612 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -5,7 +5,7 @@ FROM --platform=linux/amd64 "${image}" # Install Percona 8.0 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done \ - && echo 'deb http://repo.percona.com/ps-80/apt buster main' > /etc/apt/sources.list.d/percona.list && \ + && echo 'deb http://repo.percona.com/ps-80/apt bullseye main' > /etc/apt/sources.list.d/percona.list && \ { \ echo debconf debconf/frontend select Noninteractive; \ echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \ @@ -21,7 +21,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.c rsync \ libev4 \ # && rm -f /etc/apt/sources.list.d/percona.list \ - && echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list \ + && echo 'deb http://repo.percona.com/apt bullseye main' > /etc/apt/sources.list.d/percona.list \ # { \ # echo debconf debconf/frontend select Noninteractive; \ # echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \ diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 21227e342e0..9bb6905339a 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" FROM "${image}" AS builder diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index 9d5f1980865..07dc9a3afb3 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" FROM "${image}" AS builder diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index 178b0f702df..ff784c76e25 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-percona57" FROM "${image}" AS builder diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index f4d6e0a0e71..48abed4585f 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-percona80" FROM "${image}" AS builder diff --git a/docker/lite/Dockerfile.testing b/docker/lite/Dockerfile.testing index a86eea7e786..db4468b297d 100644 --- a/docker/lite/Dockerfile.testing +++ b/docker/lite/Dockerfile.testing @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" FROM "${image}" AS builder diff --git a/docker/lite/Dockerfile.ubi7.mysql57 b/docker/lite/Dockerfile.ubi7.mysql57 index 405263d974a..c14d0e71cf6 100644 --- a/docker/lite/Dockerfile.ubi7.mysql57 +++ b/docker/lite/Dockerfile.ubi7.mysql57 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" FROM "${image}" AS builder @@ -36,6 +36,7 @@ RUN make install PREFIX=/vt/install FROM registry.access.redhat.com/ubi7/ubi:latest # Install keys and dependencies +RUN rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && yum install -y --setopt=alwaysprompt=no gnupg \ && ( gpg --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 4D1BB29D63D98E422B2113B19334A25F8507EFA5 6341AB2753D78A78A7C27BB124C6A8A7F4A80EB5 A4A9406876FCBD3C456770C88C718D3B5072E1F5 ) \ diff --git a/docker/lite/Dockerfile.ubi7.mysql80 b/docker/lite/Dockerfile.ubi7.mysql80 index 3003ffdbe03..1d4dc9b08a7 100644 --- a/docker/lite/Dockerfile.ubi7.mysql80 +++ b/docker/lite/Dockerfile.ubi7.mysql80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" FROM "${image}" AS builder @@ -36,6 +36,7 @@ RUN make install PREFIX=/vt/install FROM registry.access.redhat.com/ubi7/ubi:latest # Install keys and dependencies +RUN rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && yum install -y --setopt=alwaysprompt=no gnupg \ && ( gpg --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 4D1BB29D63D98E422B2113B19334A25F8507EFA5 6341AB2753D78A78A7C27BB124C6A8A7F4A80EB5 A4A9406876FCBD3C456770C88C718D3B5072E1F5 ) \ @@ -51,6 +52,8 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm RUN echo H4sICH852V8CA2ZvbwC1jr0OgkAQhPt7CgrbY7W6xOQaDaEgRqKxMMTiOFYg/F2WI9G39xCttKGg2UxmJrNfokWqeryxVjUo99ja45kLj3s757IxGqiWhbVmC9CURB352rW63u8oh0mCAHdWY1uRLoDlJtcF6kpuRlnhU97LGt0CoNVgqhLINNxFcIoPPIxDHgVX/v3OsFVpjZlcM5ZoMZhMWex/ES9TMIPyM7UYKj4sqT+kwdufAToNLcP5AvRgmV7zAQAA | base64 -d | gzip -dc > /etc/yum.repos.d/CentOS-Base.repo \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --enablerepo c7base --enablerepo c7updates --enablerepo c7extras install libev gperftools-libs numactl-libs sysstat strace +RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +RUN yum install -y zstd RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs bzip2 ca-certificates gnupg libaio libcurl \ jemalloc gperftools-libs procps-ng rsync wget openssl hostname curl tzdata make \ @@ -60,7 +63,7 @@ RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ mysql-community-client mysql-community-server \ # Have to use hacks to ignore conflicts on /etc/my.cnf install && mkdir -p /tmp/1 \ - && yum install -y --setopt=alwaysprompt=no --downloadonly --downloaddir=/tmp/1 --enablerepo mysql80-community --disablerepo mysql57-community percona-xtrabackup-80 percona-toolkit \ + && yum install -y --skip-broken --setopt=alwaysprompt=no --downloadonly --downloaddir=/tmp/1 --enablerepo mysql80-community --disablerepo mysql57-community percona-xtrabackup-80 percona-toolkit \ && rpm -Uvh --replacefiles /tmp/1/*rpm \ && rm -rf /tmp/1 \ && yum clean all \ diff --git a/docker/lite/Dockerfile.ubi7.percona57 b/docker/lite/Dockerfile.ubi7.percona57 index b485aa660f1..8ad74e41734 100644 --- a/docker/lite/Dockerfile.ubi7.percona57 +++ b/docker/lite/Dockerfile.ubi7.percona57 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-percona57" FROM "${image}" AS builder diff --git a/docker/lite/Dockerfile.ubi7.percona80 b/docker/lite/Dockerfile.ubi7.percona80 index 20cc21e4319..48784bdff0a 100644 --- a/docker/lite/Dockerfile.ubi7.percona80 +++ b/docker/lite/Dockerfile.ubi7.percona80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-percona80" FROM "${image}" AS builder @@ -47,6 +47,8 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rpmkeys --checksig /tmp/perconarepo.rpm \ && rpm -Uvh /tmp/perconarepo.rpm \ && rm -f /tmp/perconarepo.rpm +RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +RUN yum install -y zstd RUN echo H4sICH852V8CA2ZvbwC1jr0OgkAQhPt7CgrbY7W6xOQaDaEgRqKxMMTiOFYg/F2WI9G39xCttKGg2UxmJrNfokWqeryxVjUo99ja45kLj3s757IxGqiWhbVmC9CURB352rW63u8oh0mCAHdWY1uRLoDlJtcF6kpuRlnhU97LGt0CoNVgqhLINNxFcIoPPIxDHgVX/v3OsFVpjZlcM5ZoMZhMWex/ES9TMIPyM7UYKj4sqT+kwdufAToNLcP5AvRgmV7zAQAA | base64 -d | gzip -dc > /etc/yum.repos.d/CentOS-Base.repo \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --enablerepo c7base --enablerepo c7updates --enablerepo c7extras install libev gperftools-libs numactl-libs sysstat strace RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ diff --git a/docker/lite/Dockerfile.ubi8.arm64.mysql80 b/docker/lite/Dockerfile.ubi8.arm64.mysql80 index a8f993a7279..2df3b01ba5f 100644 --- a/docker/lite/Dockerfile.ubi8.arm64.mysql80 +++ b/docker/lite/Dockerfile.ubi8.arm64.mysql80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" FROM "${image}" AS builder @@ -36,6 +36,7 @@ RUN make cross-install PREFIX=/vt/install GOOS=linux GOARCH=arm64 FROM registry.access.redhat.com/ubi8/ubi:latest # Install keys and dependencies +RUN rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && yum install -y --setopt=alwaysprompt=no gnupg \ && ( gpg --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 3A79BD29 A4A9406876FCBD3C456770C88C718D3B5072E1F5 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 ) \ @@ -52,10 +53,12 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rpmkeys --checksig /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ && rpm -Uvh /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm +RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +RUN yum install -y zstd RUN echo H4sICIDAHmICA2ZvbwDVkDFLxEAQhfv9FVfY7o4RhCBsoXJcIXKHwUIOi7m5MVk2yS6zG0//vYlRULTU4rrHvOHN+2ZL5Q4TP6oeO7bX3Od1pcuFXlyNUzVZg7S2yTmmCwDsgzjuDSUyB5SDI2+QzOChcyJBEnwkPOPQZijNuTkrigKmsHUFJ1MeCjUQEqg61tQweVtM0vOrfXItj1eAM0H0DiR2erTgbnOrV5uVvlk+6M+Kinvctby3p0ptqRziHjOnnxz3s/FnKJcxVlkYu/+k4Zcs+AvM8n3+jWW8MBc2NO6FZILUMEsoYQ76UvWI/vAGB/SOZZsCAAA= | base64 -d | gzip -dc > /etc/yum.repos.d/CentOS-Base.repo \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --enablerepo c8base --enablerepo c8updates --enablerepo c8extras libev numactl-libs sysstat strace \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs https://download-ib01.fedoraproject.org/pub/epel/8/Everything/aarch64/Packages/g/gperftools-libs-2.7-9.el8.aarch64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/aarch64/Packages/j/jemalloc-5.2.1-2.el8.aarch64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/aarch64/Packages/l/libunwind-1.3.1-3.el8.aarch64.rpm -RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ +RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --nobest \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs bzip2 ca-certificates gnupg libaio libcurl \ procps-ng rsync wget openssl hostname curl tzdata make \ # Can't use alwaysprompt=no here, since we need to pick up deps diff --git a/docker/lite/Dockerfile.ubi8.mysql80 b/docker/lite/Dockerfile.ubi8.mysql80 index f5b2dbbf146..59189f89587 100644 --- a/docker/lite/Dockerfile.ubi8.mysql80 +++ b/docker/lite/Dockerfile.ubi8.mysql80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" FROM "${image}" AS builder @@ -36,6 +36,7 @@ RUN make install PREFIX=/vt/install FROM registry.access.redhat.com/ubi8/ubi:latest # Install keys and dependencies +RUN rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && yum install -y --setopt=alwaysprompt=no gnupg \ && ( gpg --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 3A79BD29 A4A9406876FCBD3C456770C88C718D3B5072E1F5 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 ) \ @@ -51,10 +52,12 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rpmkeys --checksig /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ && rpm -Uvh /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm +RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +RUN yum install -y zstd RUN echo H4sICIDAHmICA2ZvbwDVkDFLxEAQhfv9FVfY7o4RhCBsoXJcIXKHwUIOi7m5MVk2yS6zG0//vYlRULTU4rrHvOHN+2ZL5Q4TP6oeO7bX3Od1pcuFXlyNUzVZg7S2yTmmCwDsgzjuDSUyB5SDI2+QzOChcyJBEnwkPOPQZijNuTkrigKmsHUFJ1MeCjUQEqg61tQweVtM0vOrfXItj1eAM0H0DiR2erTgbnOrV5uVvlk+6M+Kinvctby3p0ptqRziHjOnnxz3s/FnKJcxVlkYu/+k4Zcs+AvM8n3+jWW8MBc2NO6FZILUMEsoYQ76UvWI/vAGB/SOZZsCAAA= | base64 -d | gzip -dc > /etc/yum.repos.d/CentOS-Base.repo \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --enablerepo c8base --enablerepo c8updates --enablerepo c8extras libev numactl-libs sysstat strace \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/g/gperftools-libs-2.7-9.el8.x86_64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/j/jemalloc-5.2.1-2.el8.x86_64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/l/libunwind-1.3.1-3.el8.x86_64.rpm -RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ +RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --nobest \ && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs bzip2 ca-certificates gnupg libaio libcurl \ procps-ng rsync wget openssl hostname curl tzdata make \ # Can't use alwaysprompt=no here, since we need to pick up deps diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile index 6d4b96cc3ec..33769d9e888 100644 --- a/docker/local/Dockerfile +++ b/docker/local/Dockerfile @@ -1,4 +1,4 @@ -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-common" FROM "${image}" diff --git a/docker/vttestserver/Dockerfile.mysql57 b/docker/vttestserver/Dockerfile.mysql57 index 5f553816a15..228c1e72d93 100644 --- a/docker/vttestserver/Dockerfile.mysql57 +++ b/docker/vttestserver/Dockerfile.mysql57 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" FROM "${image}" AS builder @@ -33,7 +33,7 @@ USER vitess RUN make install-testing PREFIX=/vt/install # Start over and build the final image. -FROM debian:buster-slim +FROM debian:bullseye-slim # Install dependencies COPY docker/lite/install_dependencies.sh /vt/dist/install_dependencies.sh diff --git a/docker/vttestserver/Dockerfile.mysql80 b/docker/vttestserver/Dockerfile.mysql80 index e5ad3098226..303d0e82311 100644 --- a/docker/vttestserver/Dockerfile.mysql80 +++ b/docker/vttestserver/Dockerfile.mysql80 @@ -17,7 +17,7 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" FROM "${image}" AS builder @@ -33,7 +33,7 @@ USER vitess RUN make install-testing PREFIX=/vt/install # Start over and build the final image. -FROM debian:buster-slim +FROM debian:bullseye-slim # Install dependencies COPY docker/lite/install_dependencies.sh /vt/dist/install_dependencies.sh diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index a13742ea2c0..8ef6dcec387 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -196,9 +196,9 @@ func main() { // Check if it is an external_db if _, ok := externalDbInfoMap[k]; ok { - //This is no longer necessary, but we'll keep it for reference - //https://github.com/vitessio/vitess/pull/4868, https://github.com/vitessio/vitess/pull/5010 - //vSchemaFile = applyJsonInMemoryPatch(vSchemaFile,`[{"op": "add","path": "/tables/*", "value": {}}]`) + // This is no longer necessary, but we'll keep it for reference + // https://github.com/vitessio/vitess/pull/4868, https://github.com/vitessio/vitess/pull/5010 + // vSchemaFile = applyJsonInMemoryPatch(vSchemaFile,`[{"op": "add","path": "/tables/*", "value": {}}]`) } else { var primaryTableColumns map[string]string vSchemaFile, primaryTableColumns = addTablesVschemaPatch(vSchemaFile, keyspaceData.schemaFileNames) diff --git a/test.go b/test.go index d62441629be..8cc1723f0f5 100755 --- a/test.go +++ b/test.go @@ -77,7 +77,7 @@ For example: // Flags var ( flavor = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) - bootstrapVersion = flag.String("bootstrap-version", "14.5", "the version identifier to use for the docker images") + bootstrapVersion = flag.String("bootstrap-version", "14.6", "the version identifier to use for the docker images") runCount = flag.Int("runs", 1, "run each test this many times") retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") logPass = flag.Bool("log-pass", false, "log test output even if it passes") diff --git a/test/templates/dockerfile.tpl b/test/templates/dockerfile.tpl index f5d8af80cab..bc494050f7d 100644 --- a/test/templates/dockerfile.tpl +++ b/test/templates/dockerfile.tpl @@ -1,4 +1,4 @@ -ARG bootstrap_version=14.5 +ARG bootstrap_version=14.6 ARG image="vitess/bootstrap:${bootstrap_version}-{{.Platform}}" FROM "${image}"