Merge remote-tracking branch 'upstream/main' into fix-multiple-strings

Signed-off-by: Manan Gupta <[email protected]>

changelog/19.0/19.0.0/summary.md

@@ -6,7 +6,7 @@
   - **[Dropping Support for MySQL 5.7](#drop-support-mysql57)**
   - **[Deprecations and Deletions](#deprecations-and-deletions)**
     - [VTTablet Flags](#vttablet-flags)
-    - [MySQL binary in vitess/lite Docker image](#mysql-binary-in-lite-image)
+    - [Docker Image vitess/lite](#deprecation-vitess-lite-mysqld)
     - [Explain Statement Format](#explain-stmt-format)
   - **[Breaking Changes](#breaking-changes)**
      - [ExecuteFetchAsDBA rejects multi-statement SQL](#execute-fetch-as-dba-reject-multi)
@@ -47,21 +47,39 @@ Vitess will however, continue to support importing from MySQL 5.7 into Vitess ev
 `--vreplication_healthcheck_topology_refresh`, `--vreplication_healthcheck_retry_delay`, and `--vreplication_healthcheck_timeout`.
 - The `--vreplication_tablet_type` flag is now deprecated and ignored.
 
-#### <a id="mysql-binary-in-lite-image"/>MySQL binary in vitess/lite Docker image
+#### <a id="deprecation-vitess-lite-mysqld"/>Docker Image vitess/lite
 
 The `mysqld` binary is now deprecated in the `vitess/lite` Docker image and will be removed in a future release.
+This means that the MySQL/Percona version specific image tags for the `vitess/lite` image are deprecated.
 
-If you are currently using `vitess/lite` as your `mysqld` image in your vitess-operator deployment we invite you to use an official MySQL image such as `mysql:8.0.30`.
+Below is a full list of available tags for `v19.0.0` and their deprecation status:
+
+| Image                           | Deprecated | 
+|---------------------------------|------------|
+| `vitess/lite:v19.0.0`           | NO         |
+| `vitess/lite:v19.0.0-mysql57`   | YES        |
+| `vitess/lite:v19.0.0-mysql80`   | YES        |
+| `vitess/lite:v19.0.0-percona57` | YES        |
+| `vitess/lite:v19.0.0-percona80` | YES        |
+
+If you are currently using `vitess/lite` as your `mysqld` image in your vitess-operator deployment we invite you to use an official MySQL image, such as `mysql:8.0.30`.
 
 Below is an example of a kubernetes yaml file before and after upgrading to an official MySQL image:
 
 ```yaml
-# before
+# before:
+
+# the image used here includes MySQL 8.0.30 and its binaries
+
     mysqld:
-      mysql80Compatible: vitess/lite:19.0.0
+      mysql80Compatible: vitess/lite:v19.0.0-mysql80
 ```
 ```yaml
-# after
+# after:
+
+# if we still want to use MySQL 8.0.30, we now have to use the
+# official MySQL image with the 8.0.30 tag as shown below 
+
     mysqld:
       mysql80Compatible: mysql:8.0.30 # or even mysql:8.0.34 for instance
 ```

go/cmd/vtbackup/cli/vtbackup.go

@@ -204,7 +204,7 @@ func init() {
 	Main.Flags().StringVar(&initKeyspace, "init_keyspace", initKeyspace, "(init parameter) keyspace to use for this tablet")
 	Main.Flags().StringVar(&initShard, "init_shard", initShard, "(init parameter) shard to use for this tablet")
 	Main.Flags().IntVar(&concurrency, "concurrency", concurrency, "(init restore parameter) how many concurrent files to restore at once")
-	Main.Flags().StringVar(&incrementalFromPos, "incremental_from_pos", incrementalFromPos, "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position")
+	Main.Flags().StringVar(&incrementalFromPos, "incremental_from_pos", incrementalFromPos, "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.")
 
 	// mysqlctld-like flags
 	Main.Flags().IntVar(&mysqlPort, "mysql_port", mysqlPort, "mysql port")

go/cmd/vtctldclient/command/backups.go

@@ -35,15 +35,15 @@ import (
 var (
 	// Backup makes a Backup gRPC call to a vtctld.
 	Backup = &cobra.Command{
-		Use:                   "Backup [--concurrency <concurrency>] [--allow-primary] [--incremental-from-pos=<pos>|auto] [--upgrade-safe] <tablet_alias>",
+		Use:                   "Backup [--concurrency <concurrency>] [--allow-primary] [--incremental-from-pos=<pos>|<backup-name>|auto] [--upgrade-safe] <tablet_alias>",
 		Short:                 "Uses the BackupStorage service on the given tablet to create and store a new backup.",
 		DisableFlagsInUseLine: true,
 		Args:                  cobra.ExactArgs(1),
 		RunE:                  commandBackup,
 	}
 	// BackupShard makes a BackupShard gRPC call to a vtctld.
 	BackupShard = &cobra.Command{
-		Use:   "BackupShard [--concurrency <concurrency>] [--allow-primary] [--incremental-from-pos=<pos>|auto] [--upgrade-safe] <keyspace/shard>",
+		Use:   "BackupShard [--concurrency <concurrency>] [--allow-primary] [--incremental-from-pos=<pos>|<backup-name>|auto] [--upgrade-safe] <keyspace/shard>",
 		Short: "Finds the most up-to-date REPLICA, RDONLY, or SPARE tablet in the given shard and uses the BackupStorage service on that tablet to create and store a new backup.",
 		Long: `Finds the most up-to-date REPLICA, RDONLY, or SPARE tablet in the given shard and uses the BackupStorage service on that tablet to create and store a new backup.
 
@@ -281,14 +281,14 @@ func commandRestoreFromBackup(cmd *cobra.Command, args []string) error {
 func init() {
 	Backup.Flags().BoolVar(&backupOptions.AllowPrimary, "allow-primary", false, "Allow the primary of a shard to be used for the backup. WARNING: If using the builtin backup engine, this will shutdown mysqld on the primary and stop writes for the duration of the backup.")
 	Backup.Flags().Int32Var(&backupOptions.Concurrency, "concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously.")
-	Backup.Flags().StringVar(&backupOptions.IncrementalFromPos, "incremental-from-pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position")
+	Backup.Flags().StringVar(&backupOptions.IncrementalFromPos, "incremental-from-pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.")
 
 	Backup.Flags().BoolVar(&backupOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
 	Root.AddCommand(Backup)
 
 	BackupShard.Flags().BoolVar(&backupShardOptions.AllowPrimary, "allow-primary", false, "Allow the primary of a shard to be used for the backup. WARNING: If using the builtin backup engine, this will shutdown mysqld on the primary and stop writes for the duration of the backup.")
 	BackupShard.Flags().Int32Var(&backupShardOptions.Concurrency, "concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously.")
-	BackupShard.Flags().StringVar(&backupShardOptions.IncrementalFromPos, "incremental-from-pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position")
+	BackupShard.Flags().StringVar(&backupShardOptions.IncrementalFromPos, "incremental-from-pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.")
 	BackupShard.Flags().BoolVar(&backupOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
 	Root.AddCommand(BackupShard)
 

go/flags/endtoend/vtbackup.txt

@@ -137,7 +137,7 @@ Flags:
       --grpc_max_message_size int                                   Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216)
       --grpc_prometheus                                             Enable gRPC monitoring with Prometheus.
   -h, --help                                                        help for vtbackup
-      --incremental_from_pos string                                 Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position
+      --incremental_from_pos string                                 Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.
       --init_db_name_override string                                (init parameter) override the name of the db used by vttablet
       --init_db_sql_file string                                     path to .sql file to run after mysql_install_db
       --init_keyspace string                                        (init parameter) keyspace to use for this tablet

go/mysql/conn.go

@@ -1525,15 +1525,13 @@ type PacketOK struct {
 	sessionStateData string
 }
 
-func (c *Conn) parseOKPacket(in []byte) (*PacketOK, error) {
+func (c *Conn) parseOKPacket(packetOK *PacketOK, in []byte) error {
 	data := &coder{
 		data: in,
 		pos:  1, // We already read the type.
 	}
-	packetOK := &PacketOK{}
-
-	fail := func(format string, args ...any) (*PacketOK, error) {
-		return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, format, args...)
+	fail := func(format string, args ...any) error {
+		return vterrors.Errorf(vtrpcpb.Code_INTERNAL, format, args...)
 	}
 
 	// Affected rows.
@@ -1578,7 +1576,7 @@ func (c *Conn) parseOKPacket(in []byte) (*PacketOK, error) {
 			if !ok || length == 0 {
 				// In case we have no more data or a zero length string, there's no additional information so
 				// we can return the packet.
-				return packetOK, nil
+				return nil
 			}
 
 			// Alright, now we need to read each sub packet from the session state change.
@@ -1615,7 +1613,7 @@ func (c *Conn) parseOKPacket(in []byte) (*PacketOK, error) {
 		}
 	}
 
-	return packetOK, nil
+	return nil
 }
 
 // isErrorPacket determines whether or not the packet is an error packet. Mostly here for

go/mysql/conn_test.go

@@ -246,7 +246,8 @@ func TestBasicPackets(t *testing.T) {
 	require.NotEmpty(data)
 	assert.EqualValues(data[0], OKPacket, "OKPacket")
 
-	packetOk, err := cConn.parseOKPacket(data)
+	var packetOk PacketOK
+	err = cConn.parseOKPacket(&packetOk, data)
 	require.NoError(err)
 	assert.EqualValues(12, packetOk.affectedRows)
 	assert.EqualValues(34, packetOk.lastInsertID)
@@ -272,7 +273,7 @@ func TestBasicPackets(t *testing.T) {
 	require.NotEmpty(data)
 	assert.EqualValues(data[0], OKPacket, "OKPacket")
 
-	packetOk, err = cConn.parseOKPacket(data)
+	err = cConn.parseOKPacket(&packetOk, data)
 	require.NoError(err)
 	assert.EqualValues(23, packetOk.affectedRows)
 	assert.EqualValues(45, packetOk.lastInsertID)
@@ -295,7 +296,7 @@ func TestBasicPackets(t *testing.T) {
 	require.NotEmpty(data)
 	assert.True(cConn.isEOFPacket(data), "expected EOF")
 
-	packetOk, err = cConn.parseOKPacket(data)
+	err = cConn.parseOKPacket(&packetOk, data)
 	require.NoError(err)
 	assert.EqualValues(12, packetOk.affectedRows)
 	assert.EqualValues(34, packetOk.lastInsertID)
@@ -690,7 +691,8 @@ func TestOkPackets(t *testing.T) {
 			cConn.Capabilities = testCase.cc
 			sConn.Capabilities = testCase.cc
 			// parse the packet
-			packetOk, err := cConn.parseOKPacket(data)
+			var packetOk PacketOK
+			err := cConn.parseOKPacket(&packetOk, data)
 			if testCase.expectedErr != "" {
 				require.Error(t, err)
 				require.Equal(t, testCase.expectedErr, err.Error())
@@ -699,7 +701,7 @@ func TestOkPackets(t *testing.T) {
 			require.NoError(t, err, "failed to parse OK packet")
 
 			// write the ok packet from server
-			err = sConn.writeOKPacket(packetOk)
+			err = sConn.writeOKPacket(&packetOk)
 			require.NoError(t, err, "failed to write OK packet")
 
 			// receive the ok packet on client

go/mysql/datetime/datetime.go

@@ -610,7 +610,7 @@ func (dt *DateTime) addInterval(itv *Interval) bool {
 		dt.Time.minute = uint8((dur % time.Hour) / time.Minute)
 		dt.Time.hour = uint16(dur / time.Hour)
 
-		daynum := mysqlDayNumber(dt.Date.Year(), dt.Date.Month(), 1) + int(days)
+		daynum := MysqlDayNumber(dt.Date.Year(), dt.Date.Month(), 1) + int(days)
 		if daynum < 0 || daynum > maxDay {
 			return false
 		}
@@ -619,7 +619,7 @@ func (dt *DateTime) addInterval(itv *Interval) bool {
 		return true
 
 	case itv.unit.HasDayParts():
-		daynum := mysqlDayNumber(dt.Date.Year(), dt.Date.Month(), dt.Date.Day())
+		daynum := MysqlDayNumber(dt.Date.Year(), dt.Date.Month(), dt.Date.Day())
 		daynum += itv.day
 		dt.Date.year, dt.Date.month, dt.Date.day = mysqlDateFromDayNumber(daynum)
 		return true

go/mysql/datetime/mydate.go

@@ -16,7 +16,7 @@ limitations under the License.
 
 package datetime
 
-// mysqlDayNumber converts a date into an absolute day number.
+// MysqlDayNumber converts a date into an absolute day number.
 // This is an algorithm that has been reverse engineered from MySQL;
 // the tables used as a reference can be found in `testdata/year_to_daynr.json`.
 // It is worth noting that this absolute day number does not match the
@@ -29,7 +29,7 @@ package datetime
 // This API should only be used when performing datetime calculations (addition
 // and subtraction), so that the results match MySQL's. All other date handling
 // operations must use our helpers based on Go's standard library.
-func mysqlDayNumber(year, month, day int) int {
+func MysqlDayNumber(year, month, day int) int {
 	if year == 0 && month == 0 {
 		return 0
 	}
@@ -49,8 +49,8 @@ func mysqlDayNumber(year, month, day int) int {
 // mysqlDateFromDayNumber converts an absolute day number into a date (a year, month, day triplet).
 // This is an algorithm that has been reverse engineered from MySQL;
 // the tables used as a reference can be found in `testdata/daynr_to_date.json`.
-// See the warning from mysqlDayNumber: the day number used as an argument to
-// this function must come from mysqlDayNumber or the results won't be correct.
+// See the warning from MysqlDayNumber: the day number used as an argument to
+// this function must come from MysqlDayNumber or the results won't be correct.
 // This API should only be used when performing datetime calculations (addition
 // and subtraction), so that the results match MySQL's. All other date handling
 // operations must use our helpers based on Go's standard library.

go/mysql/datetime/mydate_test.go

@@ -35,7 +35,7 @@ func TestDayNumber(t *testing.T) {
 	require.NoError(t, err)
 
 	for year, daynr := range expected {
-		assert.Equal(t, daynr, mysqlDayNumber(year, 1, 1))
+		assert.Equal(t, daynr, MysqlDayNumber(year, 1, 1))
 	}
 }
 
@@ -54,6 +54,6 @@ func TestDayNumberFields(t *testing.T) {
 		assert.Equal(t, tc[2], int(m))
 		assert.Equal(t, tc[3], int(d))
 
-		assert.Equalf(t, tc[0], mysqlDayNumber(tc[1], tc[2], tc[3]), "date %d-%d-%d", tc[1], tc[2], tc[3])
+		assert.Equalf(t, tc[0], MysqlDayNumber(tc[1], tc[2], tc[3]), "date %d-%d-%d", tc[1], tc[2], tc[3])
 	}
 }

go/mysql/query.go

@@ -354,8 +354,9 @@ func (c *Conn) ExecuteFetchWithWarningCount(query string, maxrows int, wantfield
 
 // ReadQueryResult gets the result from the last written query.
 func (c *Conn) ReadQueryResult(maxrows int, wantfields bool) (*sqltypes.Result, bool, uint16, error) {
+	var packetOk PacketOK
 	// Get the result.
-	colNumber, packetOk, err := c.readComQueryResponse()
+	colNumber, err := c.readComQueryResponse(&packetOk)
 	if err != nil {
 		return nil, false, 0, err
 	}
@@ -441,8 +442,7 @@ func (c *Conn) ReadQueryResult(maxrows int, wantfields bool) (*sqltypes.Result,
 				more = (statusFlags & ServerMoreResultsExists) != 0
 				result.StatusFlags = statusFlags
 			} else {
-				packetOk, err := c.parseOKPacket(data)
-				if err != nil {
+				if err := c.parseOKPacket(&packetOk, data); err != nil {
 					return nil, false, 0, err
 				}
 				warnings = packetOk.warnings
@@ -497,35 +497,34 @@ func (c *Conn) drainResults() error {
 	}
 }
 
-func (c *Conn) readComQueryResponse() (int, *PacketOK, error) {
+func (c *Conn) readComQueryResponse(packetOk *PacketOK) (int, error) {
 	data, err := c.readEphemeralPacket()
 	if err != nil {
-		return 0, nil, sqlerror.NewSQLError(sqlerror.CRServerLost, sqlerror.SSUnknownSQLState, "%v", err)
+		return 0, sqlerror.NewSQLError(sqlerror.CRServerLost, sqlerror.SSUnknownSQLState, "%v", err)
 	}
 	defer c.recycleReadPacket()
 	if len(data) == 0 {
-		return 0, nil, sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "invalid empty COM_QUERY response packet")
+		return 0, sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "invalid empty COM_QUERY response packet")
 	}
 
 	switch data[0] {
 	case OKPacket:
-		packetOk, err := c.parseOKPacket(data)
-		return 0, packetOk, err
+		return 0, c.parseOKPacket(packetOk, data)
 	case ErrPacket:
 		// Error
-		return 0, nil, ParseErrorPacket(data)
+		return 0, ParseErrorPacket(data)
 	case 0xfb:
 		// Local infile
-		return 0, nil, vterrors.Errorf(vtrpc.Code_UNIMPLEMENTED, "not implemented")
+		return 0, vterrors.Errorf(vtrpc.Code_UNIMPLEMENTED, "not implemented")
 	}
 	n, pos, ok := readLenEncInt(data, 0)
 	if !ok {
-		return 0, nil, sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "cannot get column number")
+		return 0, sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "cannot get column number")
 	}
 	if pos != len(data) {
-		return 0, nil, sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "extra data in COM_QUERY response")
+		return 0, sqlerror.NewSQLError(sqlerror.CRMalformedPacket, sqlerror.SSUnknownSQLState, "extra data in COM_QUERY response")
 	}
-	return int(n), &PacketOK{}, nil
+	return int(n), nil
 }
 
 //

go/mysql/streaming_query.go

@@ -49,7 +49,8 @@ func (c *Conn) ExecuteStreamFetch(query string) (err error) {
 	}
 
 	// Get the result.
-	colNumber, _, err := c.readComQueryResponse()
+	var packetOk PacketOK
+	colNumber, err := c.readComQueryResponse(&packetOk)
 	if err != nil {
 		return err
 	}

go/test/endtoend/backup/vtctlbackup/backup_utils.go

@@ -1255,13 +1255,9 @@ func waitForNumBackups(t *testing.T, expectNumBackups int) []string {
 	}
 }
 
-func testReplicaIncrementalBackup(t *testing.T, replica *cluster.Vttablet, incrementalFromPos replication.Position, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) {
+func testReplicaIncrementalBackup(t *testing.T, replica *cluster.Vttablet, incrementalFromPos string, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) {
 	numBackups := len(waitForNumBackups(t, -1))
-	incrementalFromPosArg := "auto"
-	if !incrementalFromPos.IsZero() {
-		incrementalFromPosArg = replication.EncodePosition(incrementalFromPos)
-	}
-	output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("Backup", "--incremental-from-pos", incrementalFromPosArg, replica.Alias)
+	output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("Backup", "--incremental-from-pos", incrementalFromPos, replica.Alias)
 	if expectError != "" {
 		require.Errorf(t, err, "expected: %v", expectError)
 		require.Contains(t, output, expectError)
@@ -1278,7 +1274,7 @@ func testReplicaIncrementalBackup(t *testing.T, replica *cluster.Vttablet, incre
 	return readManifestFile(t, backupLocation), backupName
 }
 
-func TestReplicaIncrementalBackup(t *testing.T, replicaIndex int, incrementalFromPos replication.Position, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) {
+func TestReplicaIncrementalBackup(t *testing.T, replicaIndex int, incrementalFromPos string, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) {
 	replica := getReplica(t, replicaIndex)
 	return testReplicaIncrementalBackup(t, replica, incrementalFromPos, expectError)
 }