grpc: Always pass through context for dialer

The change in vitessio#15780 made me
audit the rest of the places where we dial with GRPC today where we
don't pass in a context yet. This is something we should do, since the
Dial can block otherwise indefinitely even on context cancel in
the caller.

This fixes those cases and even cleans up a TODO that mentioned this
needed to be done in a specific spot.

Once we move to grpc.NewClient in a more piecemeal fashion, we can
probably clean up / undo a lot of these cases but until then we should
do this properly with a context passed in.

Signed-off-by: Dirkjan Bussink <[email protected]>

go/cmd/vtctldclient/command/root.go

@@ -218,7 +218,7 @@ func getClientForCommand(cmd *cobra.Command) (vtctldclient.VtctldClient, error)
 		server = ""
 	}
 
-	return vtctldclient.New(VtctldClientProtocol, server)
+	return vtctldclient.New(cmd.Context(), VtctldClientProtocol, server)
 }
 
 func init() {

go/cmd/vtctldclient/command/vreplication/common/utils_test.go

@@ -148,7 +148,7 @@ func SetupLocalVtctldClient(t *testing.T, ctx context.Context, cells ...string)
 	vtctld := grpcvtctldserver.NewVtctldServer(vtenv.NewTestEnv(), ts)
 	localvtctldclient.SetServer(vtctld)
 	command.VtctldClientProtocol = "local"
-	client, err := vtctldclient.New(command.VtctldClientProtocol, "")
+	client, err := vtctldclient.New(ctx, command.VtctldClientProtocol, "")
 	require.NoError(t, err, "failed to create local vtctld client which uses an internal vtctld server")
 	common.SetClient(client)
 }

go/cmd/vtctldclient/command/vreplication/vdiff/vdiff_env_test.go

@@ -89,7 +89,7 @@ func newTestVDiffEnv(t testing.TB, ctx context.Context, sourceShards, targetShar
 
 	// Generate a unique dialer name.
 	dialerName := fmt.Sprintf("VDiffTest-%s-%d", t.Name(), rand.IntN(1000000000))
-	tabletconn.RegisterDialer(dialerName, func(tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) {
+	tabletconn.RegisterDialer(dialerName, func(ctx context.Context, tablet *topodatapb.Tablet, failFast grpcclient.FailFast) (queryservice.QueryService, error) {
 		env.mu.Lock()
 		defer env.mu.Unlock()
 		if qs, ok := env.tablets[int(tablet.Alias.Uid)]; ok {

go/cmd/vttestserver/cli/main_test.go

@@ -238,6 +238,9 @@ func TestCanGetKeyspaces(t *testing.T) {
 	conf := config
 	defer resetConfig(conf)
 
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
 	clusterInstance, err := startCluster()
 	assert.NoError(t, err)
 	defer clusterInstance.TearDown()
@@ -248,13 +251,16 @@ func TestCanGetKeyspaces(t *testing.T) {
 		}
 	}()
 
-	assertGetKeyspaces(t, clusterInstance)
+	assertGetKeyspaces(ctx, t, clusterInstance)
 }
 
 func TestExternalTopoServerConsul(t *testing.T) {
 	conf := config
 	defer resetConfig(conf)
 
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
 	// Start a single consul in the background.
 	cmd, serverAddr := startConsul(t)
 	defer func() {
@@ -273,7 +279,7 @@ func TestExternalTopoServerConsul(t *testing.T) {
 	assert.NoError(t, err)
 	defer cluster.TearDown()
 
-	assertGetKeyspaces(t, cluster)
+	assertGetKeyspaces(ctx, t, cluster)
 }
 
 func TestMtlsAuth(t *testing.T) {
@@ -445,12 +451,12 @@ func randomPort() int {
 	return int(v + 10000)
 }
 
-func assertGetKeyspaces(t *testing.T, cluster vttest.LocalCluster) {
-	client, err := vtctlclient.New(fmt.Sprintf("localhost:%v", cluster.GrpcPort()))
+func assertGetKeyspaces(ctx context.Context, t *testing.T, cluster vttest.LocalCluster) {
+	client, err := vtctlclient.New(ctx, fmt.Sprintf("localhost:%v", cluster.GrpcPort()))
 	assert.NoError(t, err)
 	defer client.Close()
 	stream, err := client.ExecuteVtctlCommand(
-		context.Background(),
+		ctx,
 		[]string{
 			"GetKeyspaces",
 			"--server",

go/test/endtoend/cluster/cluster_process.go

@@ -897,7 +897,7 @@ func (cluster *LocalProcessCluster) ExecOnTablet(ctx context.Context, vttablet *
 		return nil, err
 	}
 
-	conn, err := tabletconn.GetDialer()(tablet, grpcclient.FailFast(false))
+	conn, err := tabletconn.GetDialer()(ctx, tablet, grpcclient.FailFast(false))
 	if err != nil {
 		return nil, err
 	}
@@ -940,7 +940,7 @@ func (cluster *LocalProcessCluster) StreamTabletHealth(ctx context.Context, vtta
 		return nil, err
 	}
 
-	conn, err := tabletconn.GetDialer()(tablet, grpcclient.FailFast(false))
+	conn, err := tabletconn.GetDialer()(ctx, tablet, grpcclient.FailFast(false))
 	if err != nil {
 		return nil, err
 	}
@@ -975,7 +975,7 @@ func (cluster *LocalProcessCluster) StreamTabletHealthUntil(ctx context.Context,
 		return err
 	}
 
-	conn, err := tabletconn.GetDialer()(tablet, grpcclient.FailFast(false))
+	conn, err := tabletconn.GetDialer()(ctx, tablet, grpcclient.FailFast(false))
 	if err != nil {
 		return err
 	}

go/test/endtoend/encryption/encryptedtransport/encrypted_transport_test.go

@@ -177,7 +177,7 @@ func TestSecureTransport(t *testing.T) {
 	setCreds(t, "vtgate-client-1", "vtgate-server")
 	ctx := context.Background()
 	request := getRequest("select * from vt_insert_test")
-	vc, err := getVitessClient(grpcAddress)
+	vc, err := getVitessClient(ctx, grpcAddress)
 	require.NoError(t, err)
 
 	qr, err := vc.Execute(ctx, request)
@@ -188,7 +188,7 @@ func TestSecureTransport(t *testing.T) {
 	// 'vtgate client 2' is not authorized to access vt_insert_test
 	setCreds(t, "vtgate-client-2", "vtgate-server")
 	request = getRequest("select * from vt_insert_test")
-	vc, err = getVitessClient(grpcAddress)
+	vc, err = getVitessClient(ctx, grpcAddress)
 	require.NoError(t, err)
 	qr, err = vc.Execute(ctx, request)
 	require.NoError(t, err)
@@ -217,7 +217,7 @@ func useEffectiveCallerID(ctx context.Context, t *testing.T) {
 	setSSLInfoEmpty()
 
 	// get vitess client
-	vc, err := getVitessClient(grpcAddress)
+	vc, err := getVitessClient(ctx, grpcAddress)
 	require.NoError(t, err)
 
 	// test with empty effective caller Id
@@ -266,7 +266,7 @@ func useEffectiveGroups(ctx context.Context, t *testing.T) {
 	setSSLInfoEmpty()
 
 	// get vitess client
-	vc, err := getVitessClient(grpcAddress)
+	vc, err := getVitessClient(ctx, grpcAddress)
 	require.NoError(t, err)
 
 	// test with empty effective caller Id
@@ -452,12 +452,12 @@ func tabletConnExtraArgs(name string) []string {
 	return args
 }
 
-func getVitessClient(addr string) (vtgateservicepb.VitessClient, error) {
+func getVitessClient(ctx context.Context, addr string) (vtgateservicepb.VitessClient, error) {
 	opt, err := grpcclient.SecureDialOption(grpcCert, grpcKey, grpcCa, "", grpcName)
 	if err != nil {
 		return nil, err
 	}
-	cc, err := grpcclient.Dial(addr, grpcclient.FailFast(false), opt)
+	cc, err := grpcclient.DialContext(ctx, addr, grpcclient.FailFast(false), opt)
 	if err != nil {
 		return nil, err
 	}

go/test/endtoend/mysqlctld/mysqlctld_test.go

@@ -164,15 +164,15 @@ func TestAutoDetect(t *testing.T) {
 }
 
 func TestVersionString(t *testing.T) {
-	client, err := mysqlctlclient.New("unix", primaryTablet.MysqlctldProcess.SocketFile)
+	client, err := mysqlctlclient.New(context.Background(), "unix", primaryTablet.MysqlctldProcess.SocketFile)
 	require.NoError(t, err)
 	version, err := client.VersionString(context.Background())
 	require.NoError(t, err)
 	require.NotEmpty(t, version)
 }
 
 func TestReadBinlogFilesTimestamps(t *testing.T) {
-	client, err := mysqlctlclient.New("unix", primaryTablet.MysqlctldProcess.SocketFile)
+	client, err := mysqlctlclient.New(context.Background(), "unix", primaryTablet.MysqlctldProcess.SocketFile)
 	require.NoError(t, err)
 	_, err = client.ReadBinlogFilesTimestamps(context.Background(), &mysqlctl.ReadBinlogFilesTimestampsRequest{})
 	require.ErrorContains(t, err, "empty binlog list in ReadBinlogFilesTimestampsRequest")

go/test/endtoend/reparent/prssettingspool/main_test.go

@@ -104,13 +104,13 @@ func TestSettingsPoolWithTXAndPRS(t *testing.T) {
 	// prs should happen without any error.
 	text, err := rutils.Prs(t, clusterInstance, tablets[1])
 	require.NoError(t, err, text)
-	rutils.WaitForTabletToBeServing(t, clusterInstance, tablets[0], 1*time.Minute)
+	rutils.WaitForTabletToBeServing(ctx, t, clusterInstance, tablets[0], 1*time.Minute)
 
 	defer func() {
 		// reset state
 		text, err = rutils.Prs(t, clusterInstance, tablets[0])
 		require.NoError(t, err, text)
-		rutils.WaitForTabletToBeServing(t, clusterInstance, tablets[1], 1*time.Minute)
+		rutils.WaitForTabletToBeServing(ctx, t, clusterInstance, tablets[1], 1*time.Minute)
 	}()
 
 	// no error should occur and it should go to the right tablet.
@@ -134,12 +134,12 @@ func TestSettingsPoolWithoutTXAndPRS(t *testing.T) {
 	// prs should happen without any error.
 	text, err := rutils.Prs(t, clusterInstance, tablets[1])
 	require.NoError(t, err, text)
-	rutils.WaitForTabletToBeServing(t, clusterInstance, tablets[0], 1*time.Minute)
+	rutils.WaitForTabletToBeServing(ctx, t, clusterInstance, tablets[0], 1*time.Minute)
 	defer func() {
 		// reset state
 		text, err = rutils.Prs(t, clusterInstance, tablets[0])
 		require.NoError(t, err, text)
-		rutils.WaitForTabletToBeServing(t, clusterInstance, tablets[1], 1*time.Minute)
+		rutils.WaitForTabletToBeServing(ctx, t, clusterInstance, tablets[1], 1*time.Minute)
 	}()
 
 	// no error should occur and it should go to the right tablet.

go/test/endtoend/reparent/utils/utils.go

@@ -728,11 +728,11 @@ func CheckReplicationStatus(ctx context.Context, t *testing.T, tablet *cluster.V
 	}
 }
 
-func WaitForTabletToBeServing(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tablet *cluster.Vttablet, timeout time.Duration) {
+func WaitForTabletToBeServing(ctx context.Context, t *testing.T, clusterInstance *cluster.LocalProcessCluster, tablet *cluster.Vttablet, timeout time.Duration) {
 	vTablet, err := clusterInstance.VtctldClientProcess.GetTablet(tablet.Alias)
 	require.NoError(t, err)
 
-	tConn, err := tabletconn.GetDialer()(vTablet, false)
+	tConn, err := tabletconn.GetDialer()(ctx, vTablet, false)
 	require.NoError(t, err)
 
 	newCtx, cancel := context.WithTimeout(context.Background(), timeout)

go/vt/binlog/binlogplayer/binlog_player.go

@@ -329,7 +329,7 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error {
 		return fmt.Errorf("no binlog player client factory named %v", binlogPlayerProtocol)
 	}
 	blplClient := clientFactory()
-	err = blplClient.Dial(blp.tablet)
+	err = blplClient.Dial(ctx, blp.tablet)
 	if err != nil {
 		err := fmt.Errorf("error dialing binlog server: %v", err)
 		log.Error(err)

go/vt/binlog/binlogplayer/client.go

@@ -53,7 +53,7 @@ type BinlogTransactionStream interface {
 // Client is the interface all clients must satisfy
 type Client interface {
 	// Dial a server
-	Dial(tablet *topodatapb.Tablet) error
+	Dial(ctx context.Context, tablet *topodatapb.Tablet) error
 
 	// Close the connection
 	Close()

go/vt/binlog/binlogplayer/framework_test.go

@@ -46,7 +46,7 @@ func newFakeBinlogClient() *fakeBinlogClient {
 	return globalFBC
 }
 
-func (fbc *fakeBinlogClient) Dial(tablet *topodatapb.Tablet) error {
+func (fbc *fakeBinlogClient) Dial(ctx context.Context, tablet *topodatapb.Tablet) error {
 	fbc.lastTablet = tablet
 	return nil
 }

go/vt/binlog/binlogplayertest/player.go

@@ -17,13 +17,12 @@ limitations under the License.
 package binlogplayertest
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"strings"
 	"testing"
 
-	"context"
-
 	"google.golang.org/protobuf/proto"
 
 	"vitess.io/vitess/go/vt/binlog/binlogplayer"
@@ -227,8 +226,8 @@ func (fake *FakeBinlogStreamer) HandlePanic(err *error) {
 }
 
 // Run runs the test suite
-func Run(t *testing.T, bpc binlogplayer.Client, tablet *topodatapb.Tablet, fake *FakeBinlogStreamer) {
-	if err := bpc.Dial(tablet); err != nil {
+func Run(ctx context.Context, t *testing.T, bpc binlogplayer.Client, tablet *topodatapb.Tablet, fake *FakeBinlogStreamer) {
+	if err := bpc.Dial(ctx, tablet); err != nil {
 		t.Fatalf("Dial failed: %v", err)
 	}
 

go/vt/binlog/grpcbinlogplayer/player.go

@@ -52,14 +52,14 @@ type client struct {
 	c  binlogservicepb.UpdateStreamClient
 }
 
-func (client *client) Dial(tablet *topodatapb.Tablet) error {
+func (client *client) Dial(ctx context.Context, tablet *topodatapb.Tablet) error {
 	addr := netutil.JoinHostPort(tablet.Hostname, tablet.PortMap["grpc"])
 	var err error
 	opt, err := grpcclient.SecureDialOption(cert, key, ca, crl, name)
 	if err != nil {
 		return err
 	}
-	client.cc, err = grpcclient.Dial(addr, grpcclient.FailFast(true), opt)
+	client.cc, err = grpcclient.DialContext(ctx, addr, grpcclient.FailFast(true), opt)
 	if err != nil {
 		return err
 	}

go/vt/binlog/grpcbinlogplayer/player_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package grpcbinlogplayer
 
 import (
+	"context"
 	"net"
 	"testing"
 
@@ -48,9 +49,11 @@ func TestGRPCBinlogStreamer(t *testing.T) {
 
 	// Create a GRPC client to talk to the fake tablet
 	c := &client{}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	// and send it to the test suite
-	binlogplayertest.Run(t, c, &topodatapb.Tablet{
+	binlogplayertest.Run(ctx, t, c, &topodatapb.Tablet{
 		Hostname: host,
 		PortMap: map[string]int32{
 			"grpc": int32(port),

go/vt/discovery/fake_healthcheck.go

@@ -229,7 +229,7 @@ func (fhc *FakeHealthCheck) ReplaceTablet(old, new *topodatapb.Tablet) {
 }
 
 // TabletConnection returns the TabletConn of the given tablet.
-func (fhc *FakeHealthCheck) TabletConnection(alias *topodatapb.TabletAlias, target *querypb.Target) (queryservice.QueryService, error) {
+func (fhc *FakeHealthCheck) TabletConnection(ctx context.Context, alias *topodatapb.TabletAlias, target *querypb.Target) (queryservice.QueryService, error) {
 	aliasStr := topoproto.TabletAliasString(alias)
 	fhc.mu.RLock()
 	defer fhc.mu.RUnlock()

go/vt/discovery/healthcheck.go

@@ -214,7 +214,7 @@ type HealthCheck interface {
 	WaitForAllServingTablets(ctx context.Context, targets []*query.Target) error
 
 	// TabletConnection returns the TabletConn of the given tablet.
-	TabletConnection(alias *topodata.TabletAlias, target *query.Target) (queryservice.QueryService, error)
+	TabletConnection(ctx context.Context, alias *topodata.TabletAlias, target *query.Target) (queryservice.QueryService, error)
 
 	// RegisterStats registers the connection counts stats
 	RegisterStats()
@@ -828,15 +828,15 @@ func (hc *HealthCheckImpl) GetTabletHealth(kst KeyspaceShardTabletType, alias *t
 }
 
 // TabletConnection returns the Connection to a given tablet.
-func (hc *HealthCheckImpl) TabletConnection(alias *topodata.TabletAlias, target *query.Target) (queryservice.QueryService, error) {
+func (hc *HealthCheckImpl) TabletConnection(ctx context.Context, alias *topodata.TabletAlias, target *query.Target) (queryservice.QueryService, error) {
 	hc.mu.Lock()
 	thc := hc.healthByAlias[tabletAliasString(topoproto.TabletAliasString(alias))]
 	hc.mu.Unlock()
 	if thc == nil || thc.Conn == nil {
 		// TODO: test that throws this error
 		return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "tablet: %v is either down or nonexistent", alias)
 	}
-	return thc.Connection(hc), nil
+	return thc.Connection(ctx, hc), nil
 }
 
 // getAliasByCell should only be called while holding hc.mu

go/vt/discovery/healthcheck_test.go

@@ -1283,7 +1283,7 @@ func TestDebugURLFormatting(t *testing.T) {
 	require.Contains(t, wr.String(), expectedURL, "output missing formatted URL")
 }
 
-func tabletDialer(tablet *topodatapb.Tablet, _ grpcclient.FailFast) (queryservice.QueryService, error) {
+func tabletDialer(ctx context.Context, tablet *topodatapb.Tablet, _ grpcclient.FailFast) (queryservice.QueryService, error) {
 	connMapMu.Lock()
 	defer connMapMu.Unlock()
 

go/vt/discovery/tablet_health_check.go

@@ -128,7 +128,7 @@ func (thc *tabletHealthCheck) setServingState(serving bool, reason string) {
 
 // stream streams healthcheck responses to callback.
 func (thc *tabletHealthCheck) stream(ctx context.Context, hc *HealthCheckImpl, callback func(*query.StreamHealthResponse) error) error {
-	conn := thc.Connection(hc)
+	conn := thc.Connection(ctx, hc)
 	if conn == nil {
 		// This signals the caller to retry
 		return nil
@@ -141,10 +141,10 @@ func (thc *tabletHealthCheck) stream(ctx context.Context, hc *HealthCheckImpl, c
 	return err
 }
 
-func (thc *tabletHealthCheck) Connection(hc *HealthCheckImpl) queryservice.QueryService {
+func (thc *tabletHealthCheck) Connection(ctx context.Context, hc *HealthCheckImpl) queryservice.QueryService {
 	thc.connMu.Lock()
 	defer thc.connMu.Unlock()
-	return thc.connectionLocked(hc)
+	return thc.connectionLocked(ctx, hc)
 }
 
 func healthCheckDialerFactory(hc *HealthCheckImpl) func(ctx context.Context, addr string) (net.Conn, error) {
@@ -162,14 +162,14 @@ func healthCheckDialerFactory(hc *HealthCheckImpl) func(ctx context.Context, add
 	}
 }
 
-func (thc *tabletHealthCheck) connectionLocked(hc *HealthCheckImpl) queryservice.QueryService {
+func (thc *tabletHealthCheck) connectionLocked(ctx context.Context, hc *HealthCheckImpl) queryservice.QueryService {
 	if thc.Conn == nil {
 		withDialerContextOnce.Do(func() {
 			grpcclient.RegisterGRPCDialOptions(func(opts []grpc.DialOption) ([]grpc.DialOption, error) {
 				return append(opts, grpc.WithContextDialer(healthCheckDialerFactory(hc))), nil
 			})
 		})
-		conn, err := tabletconn.GetDialer()(thc.Tablet, grpcclient.FailFast(true))
+		conn, err := tabletconn.GetDialer()(ctx, thc.Tablet, grpcclient.FailFast(true))
 		if err != nil {
 			thc.LastError = err
 			return nil