Merge remote-tracking branch 'origin/main' into changeTabletTagsRPC

Signed-off-by: Tim Vaillancourt <[email protected]>

Makefile

@@ -286,7 +286,7 @@ $(PROTO_GO_OUTS): minimaltools install_protoc-gen-go proto/*.proto
 # This rule builds the bootstrap images for all flavors.
 DOCKER_IMAGES_FOR_TEST = mysql80 percona80
 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST)
-BOOTSTRAP_VERSION=36
+BOOTSTRAP_VERSION=37
 ensure_bootstrap_version:
 	find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \;
 	sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go

build.env

@@ -17,7 +17,7 @@
 source ./tools/shell_functions.inc
 
 go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions."
-goversion_min 1.23.1 || echo "Go version reported: `go version`. Version 1.23.1+ recommended. See https://vitess.io/contributing/build-from-source for install instructions."
+goversion_min 1.23.2 || echo "Go version reported: `go version`. Version 1.23.2+ recommended. See https://vitess.io/contributing/build-from-source for install instructions."
 
 mkdir -p dist
 mkdir -p bin

docker/bootstrap/CHANGELOG.md

@@ -141,4 +141,8 @@ List of changes between bootstrap image versions.
 
 ## [36] - 2024-09-05
 ### Changes
-- Update build to golang 1.23.1
+- Update build to golang 1.23.1
+
+## [37] - 2024-10-04
+### Changes
+- Update build to golang 1.23.2

docker/bootstrap/Dockerfile.common

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 golang:1.23.1-bullseye
+FROM --platform=linux/amd64 golang:1.23.2-bullseye
 
 # Install Vitess build dependencies
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \

docker/lite/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=linux/amd64 golang:1.23.1-bullseye AS builder
+FROM --platform=linux/amd64 golang:1.23.2-bullseye AS builder
 
 # Allows docker builds to set the BUILD_NUMBER
 ARG BUILD_NUMBER

docker/lite/Dockerfile.percona80

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=linux/amd64 golang:1.23.1-bullseye AS builder
+FROM --platform=linux/amd64 golang:1.23.2-bullseye AS builder
 
 # Allows docker builds to set the BUILD_NUMBER
 ARG BUILD_NUMBER

docker/vttestserver/Dockerfile.mysql80

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=linux/amd64 golang:1.23.1-bullseye AS builder
+FROM --platform=linux/amd64 golang:1.23.2-bullseye AS builder
 
 # Allows docker builds to set the BUILD_NUMBER
 ARG BUILD_NUMBER

go.mod

@@ -1,6 +1,6 @@
 module vitess.io/vitess
 
-go 1.23.1
+go 1.23.2
 
 require (
 	cloud.google.com/go/storage v1.43.0

go/cmd/vtctldclient/command/vreplication/common/utils.go

@@ -287,7 +287,7 @@ func AddCommonSwitchTrafficFlags(cmd *cobra.Command, initializeTargetSequences b
 	cmd.Flags().BoolVar(&SwitchTrafficOptions.DryRun, "dry-run", false, "Print the actions that would be taken and report any known errors that would have occurred.")
 	cmd.Flags().BoolVar(&SwitchTrafficOptions.Force, "force", false, "Force the traffic switch even if some potentially non-critical actions cannot be performed; for example the tablet refresh fails on some tablets in the keyspace. WARNING: this should be used with extreme caution and only in emergency situations!")
 	if initializeTargetSequences {
-		cmd.Flags().BoolVar(&SwitchTrafficOptions.InitializeTargetSequences, "initialize-target-sequences", false, "When moving tables from an unsharded keyspace to a sharded keyspace, initialize any sequences that are being used on the target when switching writes.")
+		cmd.Flags().BoolVar(&SwitchTrafficOptions.InitializeTargetSequences, "initialize-target-sequences", false, "When moving tables from an unsharded keyspace to a sharded keyspace, initialize any sequences that are being used on the target when switching writes. If the sequence table is not found, and the sequence table reference was fully qualified OR a value was specified for --global-keyspace, then we will attempt to create the sequence table in that keyspace.")
 	}
 }
 

go/cmd/vtctldclient/command/vreplication/movetables/create.go

@@ -40,6 +40,8 @@ var (
 		NoRoutingRules      bool
 		AtomicCopy          bool
 		WorkflowOptions     vtctldatapb.WorkflowOptions
+		// This maps to a WorkflowOptions.ShardedAutoIncrementHandling ENUM value.
+		ShardedAutoIncrementHandlingStr string
 	}{}
 
 	// create makes a MoveTablesCreate gRPC call to a vtctld.
@@ -87,6 +89,20 @@ var (
 				return fmt.Errorf("cannot specify both --tenant-id (i.e. a multi-tenant migration) and --source-shards (i.e. a shard-by-shard migration)")
 			}
 
+			// createOptions.ShardedAutoIncrementHandlingStr is the CLI flag value
+			// provided and we need to map that to the ENUM value for
+			// createOptions.WorkflowOptions.ShardedAutoIncrementHandling which
+			// gets saved in the _vt.vreplication record's options column.
+			createOptions.ShardedAutoIncrementHandlingStr = strings.ToUpper(createOptions.ShardedAutoIncrementHandlingStr)
+			val, ok := vtctldatapb.ShardedAutoIncrementHandling_value[createOptions.ShardedAutoIncrementHandlingStr]
+			if !ok {
+				return fmt.Errorf("invalid value provided for --sharded-auto-increment-handling, valid values are: %s", shardedAutoIncHandlingStrOptions)
+			}
+			createOptions.WorkflowOptions.ShardedAutoIncrementHandling = vtctldatapb.ShardedAutoIncrementHandling(val)
+			if val == int32(vtctldatapb.ShardedAutoIncrementHandling_REPLACE) && createOptions.WorkflowOptions.GlobalKeyspace == "" {
+				fmt.Println("WARNING: no global-keyspace value provided so all sequence table references not fully qualified must be created manually before switching traffic")
+			}
+
 			return nil
 		},
 		RunE: commandCreate,

go/cmd/vtctldclient/command/vreplication/movetables/movetables.go

@@ -17,10 +17,15 @@ limitations under the License.
 package movetables
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/spf13/cobra"
 
 	"vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
 	"vitess.io/vitess/go/vt/topo/topoproto"
+
+	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
 )
 
 var (
@@ -32,6 +37,7 @@ var (
 		Aliases:               []string{"movetables"},
 		Args:                  cobra.ExactArgs(1),
 	}
+	shardedAutoIncHandlingStrOptions string
 )
 
 func registerCommands(root *cobra.Command) {
@@ -49,8 +55,16 @@ func registerCommands(root *cobra.Command) {
 	create.Flags().BoolVar(&createOptions.NoRoutingRules, "no-routing-rules", false, "(Advanced) Do not create routing rules while creating the workflow. See the reference documentation for limitations if you use this flag.")
 	create.Flags().BoolVar(&createOptions.AtomicCopy, "atomic-copy", false, "(EXPERIMENTAL) A single copy phase is run for all tables from the source. Use this, for example, if your source keyspace has tables which use foreign key constraints.")
 	create.Flags().StringVar(&createOptions.WorkflowOptions.TenantId, "tenant-id", "", "(EXPERIMENTAL: Multi-tenant migrations only) The tenant ID to use for the MoveTables workflow into a multi-tenant keyspace.")
-	create.Flags().BoolVar(&createOptions.WorkflowOptions.StripShardedAutoIncrement, "remove-sharded-auto-increment", true, "If moving the table(s) to a sharded keyspace, remove any auto_increment clauses when copying the schema to the target as sharded keyspaces should rely on either user/application generated values or Vitess sequences to ensure uniqueness.")
 	create.Flags().StringSliceVar(&createOptions.WorkflowOptions.Shards, "shards", nil, "(EXPERIMENTAL: Multi-tenant migrations only) Specify that vreplication streams should only be created on this subset of target shards. Warning: you should first ensure that all rows on the source route to the specified subset of target shards using your VIndex of choice or you could lose data during the migration.")
+	create.Flags().StringVar(&createOptions.WorkflowOptions.GlobalKeyspace, "global-keyspace", "", "If specified, then attempt to create any global resources here such as sequence tables needed to replace auto_increment table clauses that are removed due to --sharded-auto-increment-handling=REPLACE. The value must be an unsharded keyspace that already exists.")
+	create.Flags().StringVar(&createOptions.ShardedAutoIncrementHandlingStr, "sharded-auto-increment-handling", vtctldatapb.ShardedAutoIncrementHandling_REMOVE.String(),
+		fmt.Sprintf("If moving the table(s) to a sharded keyspace, remove any MySQL auto_increment clauses when copying the schema to the target as sharded keyspaces should rely on either user/application generated values or Vitess sequences to ensure uniqueness. If REPLACE is specified then they are automatically replaced by Vitess sequence definitions. (options are: %s)",
+			shardedAutoIncHandlingStrOptions))
+	// This flag was deprecated in v21 so can be removed in v22+.
+	create.Flags().StringVar(&createOptions.ShardedAutoIncrementHandlingStr, "remove-sharded-auto-increment", vtctldatapb.ShardedAutoIncrementHandling_REMOVE.String(),
+		fmt.Sprintf("If moving the table(s) to a sharded keyspace, remove any MySQL auto_increment clauses when copying the schema to the target as sharded keyspaces should rely on either user/application generated values or Vitess sequences to ensure uniqueness. If REPLACE is specified then they are automatically replaced by Vitess sequence definitions. (options are: %s)",
+			shardedAutoIncHandlingStrOptions))
+	create.Flags().MarkDeprecated("remove-sharded-auto-increment", "please use --sharded-auto-increment-handling instead.")
 	base.AddCommand(create)
 
 	opts := &common.SubCommandsOpts{
@@ -100,4 +114,17 @@ func registerCommands(root *cobra.Command) {
 
 func init() {
 	common.RegisterCommandHandler("MoveTables", registerCommands)
+
+	sb := strings.Builder{}
+	strvals := make([]string, len(vtctldatapb.ShardedAutoIncrementHandling_name))
+	for enumval, strval := range vtctldatapb.ShardedAutoIncrementHandling_name {
+		strvals[enumval] = strval
+	}
+	for i, v := range strvals {
+		if i > 0 {
+			sb.WriteByte(',')
+		}
+		sb.WriteString(v)
+	}
+	shardedAutoIncHandlingStrOptions = sb.String()
 }

go/test/endtoend/vreplication/vreplication_vtctldclient_cli_test.go

@@ -110,8 +110,8 @@ func testMoveTablesFlags1(t *testing.T, mt *iMoveTables, sourceKeyspace, targetK
 	createFlags := []string{"--auto-start=false", "--defer-secondary-keys=false", "--stop-after-copy",
 		"--no-routing-rules", "--on-ddl", "STOP", "--exclude-tables", "customer2",
 		"--tablet-types", "primary,rdonly", "--tablet-types-in-preference-order=true",
-		"--all-cells",
-		"--config-overrides", mapToCSV(overrides),
+		"--all-cells", "--config-overrides", mapToCSV(overrides),
+		"--sharded-auto-increment-handling=REPLACE", fmt.Sprintf("--global-keyspace=%s", sourceKeyspace),
 	}
 	completeFlags := []string{"--keep-routing-rules", "--keep-data"}
 	switchFlags := []string{}
@@ -591,6 +591,10 @@ func validateMoveTablesWorkflow(t *testing.T, workflows []*vtctldatapb.Workflow)
 	require.Equalf(t, 1, len(bls.Filter.Rules), "Rules are %+v", bls.Filter.Rules) // only customer, customer2 should be excluded
 	require.Equal(t, binlogdatapb.OnDDLAction_STOP, bls.OnDdl)
 	require.True(t, bls.StopAfterCopy)
+
+	// Validate the sharded-auto-increment-handling related value handling.
+	require.Equal(t, vtctldatapb.ShardedAutoIncrementHandling_REPLACE, vtctldatapb.ShardedAutoIncrementHandling(wf.Options.ShardedAutoIncrementHandling))
+	require.Equal(t, wf.Source.Keyspace, wf.Options.GlobalKeyspace)
 }
 
 // Test that routing rules can be applied using the vtctldclient CLI for all types of routing rules.