Merge remote-tracking branch 'origin/main' into vtctldclient_bootstra…

…p_cluster

Signed-off-by: Matt Lord <[email protected]>

.github/workflows/docker_build_base.yml

@@ -177,7 +177,7 @@ jobs:
       fail-fast: true
       matrix:
         debian: [ bullseye, bookworm ]
-        component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, logrotate, logtail ]
+        component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, logrotate, logtail, vtbackup, vtexplain ]
 
     steps:
       - name: Check out code

.github/workflows/docker_build_vttestserver.yml

@@ -0,0 +1,65 @@
+name: Docker Build vttestserver
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+
+concurrency:
+  group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build vttestserver')
+  cancel-in-progress: true
+
+permissions: read-all
+
+jobs:
+  build_and_push:
+    name: Build and push vitess/vttestserver Docker images
+    runs-on: gh-hosted-runners-16cores-1
+    if: github.repository == 'vitessio/vitess'
+
+    strategy:
+      fail-fast: true
+      matrix:
+        branch: [ mysql57, mysql80 ]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set Dockerfile path
+        run: |
+          echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV
+
+      - name: Build and push on main
+        if: github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ env.DOCKERFILE }}
+          push: true
+          tags: vitess/vttestserver:${{ matrix.branch }}
+
+      - name: Get the Git tag
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+      - name: Set Docker tag name
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV
+
+      - name: Build and push on tags
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ env.DOCKERFILE }}
+          push: true
+          tags: ${{ env.DOCKER_TAG }}

go/cmd/vtctldclient/command/vreplication/common/utils.go

@@ -26,6 +26,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"vitess.io/vitess/go/cmd/vtctldclient/cli"
+	"vitess.io/vitess/go/vt/topo"
 	"vitess.io/vitess/go/vt/topo/topoproto"
 	"vitess.io/vitess/go/vt/vtctl/vtctldclient"
 	"vitess.io/vitess/go/vt/vtctl/workflow"
@@ -56,6 +57,7 @@ var (
 
 	CreateOptions = struct {
 		Cells                        []string
+		AllCells                     bool
 		TabletTypes                  []topodatapb.TabletType
 		TabletTypesInPreferenceOrder bool
 		OnDDL                        string
@@ -98,12 +100,28 @@ func GetCommandCtx() context.Context {
 	return commandCtx
 }
 
-func ParseCells(cmd *cobra.Command) {
-	if cmd.Flags().Lookup("cells").Changed { // Validate the provided value(s)
+func ParseCells(cmd *cobra.Command) error {
+	cf := cmd.Flags().Lookup("cells")
+	af := cmd.Flags().Lookup("all-cells")
+	if cf != nil && cf.Changed && af != nil && af.Changed {
+		return fmt.Errorf("cannot specify both --cells and --all-cells")
+	}
+	if cf.Changed { // Validate the provided value(s)
 		for i, cell := range CreateOptions.Cells { // Which only means trimming whitespace
 			CreateOptions.Cells[i] = strings.TrimSpace(cell)
 		}
 	}
+	if CreateOptions.AllCells { // Use all current cells
+		ctx, cancel := context.WithTimeout(commandCtx, topo.RemoteOperationTimeout)
+		defer cancel()
+		resp, err := client.GetCellInfoNames(ctx, &vtctldatapb.GetCellInfoNamesRequest{})
+		if err != nil {
+			return fmt.Errorf("failed to get current cells: %v", err)
+		}
+		CreateOptions.Cells = make([]string, len(resp.Names))
+		copy(CreateOptions.Cells, resp.Names)
+	}
+	return nil
 }
 
 func ParseTabletTypes(cmd *cobra.Command) error {
@@ -130,7 +148,9 @@ func ParseAndValidateCreateOptions(cmd *cobra.Command) error {
 	if err := validateOnDDL(cmd); err != nil {
 		return err
 	}
-	ParseCells(cmd)
+	if err := ParseCells(cmd); err != nil {
+		return err
+	}
 	if err := ParseTabletTypes(cmd); err != nil {
 		return err
 	}
@@ -192,6 +212,7 @@ func AddCommonFlags(cmd *cobra.Command) {
 
 func AddCommonCreateFlags(cmd *cobra.Command) {
 	cmd.Flags().StringSliceVarP(&CreateOptions.Cells, "cells", "c", nil, "Cells and/or CellAliases to copy table data from.")
+	cmd.Flags().BoolVarP(&CreateOptions.AllCells, "all-cells", "a", false, "Copy table data from any existing cell.")
 	cmd.Flags().Var((*topoproto.TabletTypeListFlag)(&CreateOptions.TabletTypes), "tablet-types", "Source tablet types to replicate table data from (e.g. PRIMARY,REPLICA,RDONLY).")
 	cmd.Flags().BoolVar(&CreateOptions.TabletTypesInPreferenceOrder, "tablet-types-in-preference-order", true, "When performing source tablet selection, look for candidates in the type order as they are listed in the tablet-types flag.")
 	cmd.Flags().StringVar(&CreateOptions.OnDDL, "on-ddl", onDDLDefault, "What to do when DDL is encountered in the VReplication stream. Possible values are IGNORE, STOP, EXEC, and EXEC_IGNORE.")

go/cmd/vtctldclient/command/vreplication/common/utils_test.go

@@ -14,19 +14,38 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package common
+package common_test
 
 import (
+	"context"
 	"testing"
+	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/cmd/vtctldclient/command"
+	"vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
+	"vitess.io/vitess/go/vt/topo"
+	"vitess.io/vitess/go/vt/topo/memorytopo"
+	"vitess.io/vitess/go/vt/vtctl/grpcvtctldserver"
+	"vitess.io/vitess/go/vt/vtctl/localvtctldclient"
+	"vitess.io/vitess/go/vt/vtctl/vtctldclient"
+	"vitess.io/vitess/go/vt/vttablet/tmclient"
 )
 
 func TestParseAndValidateCreateOptions(t *testing.T) {
+	common.SetCommandCtx(context.Background())
+	ctx, cancel := context.WithTimeout(common.GetCommandCtx(), 60*time.Second)
+	defer cancel()
+	cells := []string{"zone1", "zone2", "zone3"}
+	SetupLocalVtctldClient(t, ctx, cells...)
+
 	tests := []struct {
-		name    string
-		setFunc func(*cobra.Command) error
-		wantErr bool
+		name      string
+		setFunc   func(*cobra.Command) error
+		wantErr   bool
+		checkFunc func()
 	}{
 		{
 			name: "invalid tablet type",
@@ -58,25 +77,77 @@ func TestParseAndValidateCreateOptions(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "cells and all-cells",
+			setFunc: func(cmd *cobra.Command) error {
+				cellsFlag := cmd.Flags().Lookup("cells")
+				allCellsFlag := cmd.Flags().Lookup("all-cells")
+				if err := cellsFlag.Value.Set("cella"); err != nil {
+					return err
+				}
+				cellsFlag.Changed = true
+				if err := allCellsFlag.Value.Set("true"); err != nil {
+					return err
+				}
+				allCellsFlag.Changed = true
+				return nil
+			},
+			wantErr: true,
+		},
+		{
+			name: "all cells",
+			setFunc: func(cmd *cobra.Command) error {
+				allCellsFlag := cmd.Flags().Lookup("all-cells")
+				if err := allCellsFlag.Value.Set("true"); err != nil {
+					return err
+				}
+				allCellsFlag.Changed = true
+				return nil
+			},
+			wantErr: false,
+			checkFunc: func() {
+				require.Equal(t, cells, common.CreateOptions.Cells)
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cmd := &cobra.Command{}
-			AddCommonCreateFlags(cmd)
+			common.AddCommonCreateFlags(cmd)
 			test := func() error {
 				if tt.setFunc != nil {
 					if err := tt.setFunc(cmd); err != nil {
 						return err
 					}
 				}
-				if err := ParseAndValidateCreateOptions(cmd); err != nil {
+				if err := common.ParseAndValidateCreateOptions(cmd); err != nil {
 					return err
 				}
 				return nil
 			}
 			if err := test(); (err != nil) != tt.wantErr {
 				t.Errorf("ParseAndValidateCreateOptions() error = %v, wantErr %t", err, tt.wantErr)
 			}
+			if tt.checkFunc != nil {
+				tt.checkFunc()
+			}
 		})
 	}
 }
+
+// SetupLocalVtctldClient sets up a local or internal VtctldServer and
+// VtctldClient for tests. It uses a memorytopo instance which contains
+// the cells provided.
+func SetupLocalVtctldClient(t *testing.T, ctx context.Context, cells ...string) {
+	ts, factory := memorytopo.NewServerAndFactory(ctx, cells...)
+	topo.RegisterFactory("test", factory)
+	tmclient.RegisterTabletManagerClientFactory("grpc", func() tmclient.TabletManagerClient {
+		return nil
+	})
+	vtctld := grpcvtctldserver.NewVtctldServer(ts)
+	localvtctldclient.SetServer(vtctld)
+	command.VtctldClientProtocol = "local"
+	client, err := vtctldclient.New(command.VtctldClientProtocol, "")
+	require.NoError(t, err, "failed to create local vtctld client which uses an internal vtctld server")
+	common.SetClient(client)
+}

go/mysql/conn.go

@@ -1708,6 +1708,10 @@ func (c *Conn) IsMarkedForClose() bool {
 	return c.closing
 }
 
+func (c *Conn) IsShuttingDown() bool {
+	return c.listener.shutdown.Load()
+}
+
 // ConnCheck ensures that this connection to the MySQL server hasn't been broken.
 // This is a fast, non-blocking check. For details on its implementation, please read
 // "Three Bugs in the Go MySQL Driver" (Vicent Marti, GitHub, 2020)
@@ -1745,8 +1749,3 @@ func (c *Conn) ConnCheck() error {
 	}
 	return nil
 }
-
-// GetTestConn returns a conn for testing purpose only.
-func GetTestConn() *Conn {
-	return newConn(testConn{})
-}

go/mysql/conn_fake.go

@@ -81,3 +81,14 @@ func (m mockAddress) String() string {
 }
 
 var _ net.Addr = (*mockAddress)(nil)
+
+// GetTestConn returns a conn for testing purpose only.
+func GetTestConn() *Conn {
+	return newConn(testConn{})
+}
+
+// GetTestServerConn is only meant to be used for testing.
+// It creates a server connection using a testConn and the provided listener.
+func GetTestServerConn(listener *Listener) *Conn {
+	return newServerConn(testConn{}, listener)
+}

go/vt/discovery/tablet_picker.go

@@ -135,6 +135,8 @@ type TabletPicker struct {
 	inOrder       bool
 	cellPref      TabletPickerCellPreference
 	localCellInfo localCellInfo
+	// This map is keyed on the results of TabletAlias.String().
+	ignoreTablets map[string]struct{}
 }
 
 // NewTabletPicker returns a TabletPicker.
@@ -144,6 +146,7 @@ func NewTabletPicker(
 	cells []string,
 	localCell, keyspace, shard, tabletTypesStr string,
 	options TabletPickerOptions,
+	ignoreTablets ...*topodatapb.TabletAlias,
 ) (*TabletPicker, error) {
 	// Keep inOrder parsing here for backward compatability until TabletPickerTabletOrder is fully adopted.
 	if tabletTypesStr == "" {
@@ -219,7 +222,7 @@ func NewTabletPicker(
 		}
 	}
 
-	return &TabletPicker{
+	tp := &TabletPicker{
 		ts:            ts,
 		cells:         dedupeCells(cells),
 		localCellInfo: localCellInfo{localCell: localCell, cellsInAlias: aliasCellMap},
@@ -228,7 +231,15 @@ func NewTabletPicker(
 		tabletTypes:   tabletTypes,
 		inOrder:       inOrder,
 		cellPref:      cellPref,
-	}, nil
+		ignoreTablets: make(map[string]struct{}, len(ignoreTablets)),
+	}
+
+	for _, ignoreTablet := range ignoreTablets {
+		tp.ignoreTablets[ignoreTablet.String()] = struct{}{}
+	}
+
+	return tp, nil
+
 }
 
 // dedupeCells is used to remove duplicates in the cell list in case it is passed in
@@ -358,7 +369,9 @@ func (tp *TabletPicker) GetMatchingTablets(ctx context.Context) []*topo.TabletIn
 			log.Errorf("Error getting shard %s/%s: %v", tp.keyspace, tp.shard, err)
 			return nil
 		}
-		aliases = append(aliases, si.PrimaryAlias)
+		if _, ignore := tp.ignoreTablets[si.PrimaryAlias.String()]; !ignore {
+			aliases = append(aliases, si.PrimaryAlias)
+		}
 	} else {
 		actualCells := make([]string, 0)
 		for _, cell := range tp.cells {
@@ -394,7 +407,9 @@ func (tp *TabletPicker) GetMatchingTablets(ctx context.Context) []*topo.TabletIn
 				continue
 			}
 			for _, node := range sri.Nodes {
-				aliases = append(aliases, node.TabletAlias)
+				if _, ignore := tp.ignoreTablets[node.TabletAlias.String()]; !ignore {
+					aliases = append(aliases, node.TabletAlias)
+				}
 			}
 		}
 	}

go/vt/discovery/tablet_picker_test.go

@@ -391,6 +391,29 @@ func TestPickUsingCellAsAlias(t *testing.T) {
 	}
 }
 
+func TestPickWithIgnoreList(t *testing.T) {
+	ctx := utils.LeakCheckContext(t)
+
+	te := newPickerTestEnv(t, ctx, []string{"cell1", "cell2"})
+
+	want := addTablet(ctx, te, 101, topodatapb.TabletType_REPLICA, "cell1", true, true)
+	defer deleteTablet(t, te, want)
+
+	dontWant := addTablet(ctx, te, 102, topodatapb.TabletType_REPLICA, "cell1", true, true)
+	defer deleteTablet(t, te, dontWant)
+
+	// Specify the alias as the cell.
+	tp, err := NewTabletPicker(ctx, te.topoServ, []string{"cella"}, "cell1", te.keyspace, te.shard, "replica", TabletPickerOptions{}, dontWant.GetAlias())
+	require.NoError(t, err)
+
+	// Try it many times to be sure we don't ever pick from the ignore list.
+	for i := 0; i < 100; i++ {
+		tablet, err := tp.PickForStreaming(ctx)
+		require.NoError(t, err)
+		require.False(t, proto.Equal(dontWant, tablet), "Picked the tablet we shouldn't have: %v", dontWant)
+	}
+}
+
 func TestPickUsingCellAliasOnlySpecified(t *testing.T) {
 	ctx := utils.LeakCheckContextTimeout(t, 200*time.Millisecond)