PRS and ERS don't promote replicas taking backups

go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go

@@ -188,6 +188,8 @@ type TabletManagerClient struct {
 		EventJitter   time.Duration
 		ErrorAfter    time.Duration
 	}
+	// Backing Up - keyed by tablet alias.
+	BackingUp map[string]bool
 	// keyed by tablet alias.
 	ChangeTagsResult map[string]struct {
 		Response *tabletmanagerdatapb.ChangeTagsResponse
@@ -1052,6 +1054,9 @@ func (fake *TabletManagerClient) ReplicationStatus(ctx context.Context, tablet *
 	}
 
 	if result, ok := fake.ReplicationStatusResults[key]; ok {
+		if _, ok = fake.BackingUp[key]; ok {
+			result.Position.BackingUp = fake.BackingUp[key]
+		}
 		return result.Position, result.Error
 	}
 

go/vt/vtctl/reparentutil/emergency_reparenter.go

@@ -249,7 +249,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve
 	// 2. Remove the tablets with the Must_not promote rule
 	// 3. Remove cross-cell tablets if PreventCrossCellPromotion is specified
 	// Our final primary candidate MUST belong to this list of valid candidates
-	validCandidateTablets, err = erp.filterValidCandidates(validCandidateTablets, stoppedReplicationSnapshot.reachableTablets, prevPrimary, opts)
+	validCandidateTablets, err = erp.filterValidCandidates(validCandidateTablets, stoppedReplicationSnapshot.reachableTablets, stoppedReplicationSnapshot.backingUpTablets, prevPrimary, opts)
 	if err != nil {
 		return err
 	}
@@ -729,7 +729,7 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(
 }
 
 // filterValidCandidates filters valid tablets, keeping only the ones which can successfully be promoted without any constraint failures and can make forward progress on being promoted
-func (erp *EmergencyReparenter) filterValidCandidates(validTablets []*topodatapb.Tablet, tabletsReachable []*topodatapb.Tablet, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) {
+func (erp *EmergencyReparenter) filterValidCandidates(validTablets []*topodatapb.Tablet, tabletsReachable []*topodatapb.Tablet, tabletsBackingUp map[string]bool, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) {
 	var restrictedValidTablets []*topodatapb.Tablet
 	for _, tablet := range validTablets {
 		tabletAliasStr := topoproto.TabletAliasString(tablet.Alias)
@@ -757,6 +757,15 @@ func (erp *EmergencyReparenter) filterValidCandidates(validTablets []*topodatapb
 			}
 			continue
 		}
+		// Remove candidates that are running a backup.
+		backingUp, ok := tabletsBackingUp[tabletAliasStr]
+		if ok && backingUp {
+			erp.logger.Infof("Removing %s from list of valid candidates for promotion because it is running a backup", tabletAliasStr)
+			if opts.NewPrimaryAlias != nil && topoproto.TabletAliasEqual(opts.NewPrimaryAlias, tablet.Alias) {
+				return nil, vterrors.Errorf(vtrpc.Code_ABORTED, "proposed primary %s will not be able to make forward progress on being promoted", topoproto.TabletAliasString(opts.NewPrimaryAlias))
+			}
+			continue
+		}
 		restrictedValidTablets = append(restrictedValidTablets, tablet)
 	}
 	return restrictedValidTablets, nil

go/vt/vtctl/reparentutil/emergency_reparenter_test.go

@@ -4460,11 +4460,22 @@ func TestEmergencyReparenter_filterValidCandidates(t *testing.T) {
 		}
 	)
 	allTablets := []*topodatapb.Tablet{primaryTablet, replicaTablet, rdonlyTablet, replicaCrossCellTablet, rdonlyCrossCellTablet}
+	noTabletsBackingUp := map[string]bool{
+		topoproto.TabletAliasString(primaryTablet.Alias): false, topoproto.TabletAliasString(replicaTablet.Alias): false,
+		topoproto.TabletAliasString(rdonlyTablet.Alias): false, topoproto.TabletAliasString(replicaCrossCellTablet.Alias): false,
+		topoproto.TabletAliasString(rdonlyCrossCellTablet.Alias): false,
+	}
+	replicaBackingUp := map[string]bool{
+		topoproto.TabletAliasString(primaryTablet.Alias): false, topoproto.TabletAliasString(replicaTablet.Alias): true,
+		topoproto.TabletAliasString(rdonlyTablet.Alias): false, topoproto.TabletAliasString(replicaCrossCellTablet.Alias): false,
+		topoproto.TabletAliasString(rdonlyCrossCellTablet.Alias): false,
+	}
 	tests := []struct {
 		name             string
 		durability       string
 		validTablets     []*topodatapb.Tablet
 		tabletsReachable []*topodatapb.Tablet
+		tabletsBackingUp map[string]bool
 		prevPrimary      *topodatapb.Tablet
 		opts             EmergencyReparentOptions
 		filteredTablets  []*topodatapb.Tablet
@@ -4475,12 +4486,21 @@ func TestEmergencyReparenter_filterValidCandidates(t *testing.T) {
 			durability:       "none",
 			validTablets:     allTablets,
 			tabletsReachable: allTablets,
+			tabletsBackingUp: noTabletsBackingUp,
 			filteredTablets:  []*topodatapb.Tablet{primaryTablet, replicaTablet, replicaCrossCellTablet},
+		}, {
+			name:             "filter backing up",
+			durability:       "none",
+			validTablets:     allTablets,
+			tabletsReachable: allTablets,
+			tabletsBackingUp: replicaBackingUp,
+			filteredTablets:  []*topodatapb.Tablet{primaryTablet, replicaCrossCellTablet},
 		}, {
 			name:             "filter cross cell",
 			durability:       "none",
 			validTablets:     allTablets,
 			tabletsReachable: allTablets,
+			tabletsBackingUp: noTabletsBackingUp,
 			prevPrimary: &topodatapb.Tablet{
 				Alias: &topodatapb.TabletAlias{
 					Cell: "zone-1",
@@ -4495,6 +4515,7 @@ func TestEmergencyReparenter_filterValidCandidates(t *testing.T) {
 			durability:       "cross_cell",
 			validTablets:     []*topodatapb.Tablet{primaryTablet, replicaTablet},
 			tabletsReachable: []*topodatapb.Tablet{primaryTablet, replicaTablet, rdonlyTablet, rdonlyCrossCellTablet},
+			tabletsBackingUp: noTabletsBackingUp,
 			filteredTablets:  nil,
 		}, {
 			name:       "filter mixed",
@@ -4509,12 +4530,14 @@ func TestEmergencyReparenter_filterValidCandidates(t *testing.T) {
 			},
 			validTablets:     allTablets,
 			tabletsReachable: allTablets,
+			tabletsBackingUp: noTabletsBackingUp,
 			filteredTablets:  []*topodatapb.Tablet{replicaCrossCellTablet},
 		}, {
 			name:             "error - requested primary must not",
 			durability:       "none",
 			validTablets:     allTablets,
 			tabletsReachable: allTablets,
+			tabletsBackingUp: noTabletsBackingUp,
 			opts: EmergencyReparentOptions{
 				NewPrimaryAlias: rdonlyTablet.Alias,
 			},
@@ -4535,6 +4558,7 @@ func TestEmergencyReparenter_filterValidCandidates(t *testing.T) {
 			durability:       "cross_cell",
 			validTablets:     allTablets,
 			tabletsReachable: []*topodatapb.Tablet{primaryTablet, replicaTablet, rdonlyTablet, rdonlyCrossCellTablet},
+			tabletsBackingUp: noTabletsBackingUp,
 			opts: EmergencyReparentOptions{
 				NewPrimaryAlias: primaryTablet.Alias,
 			},
@@ -4548,7 +4572,7 @@ func TestEmergencyReparenter_filterValidCandidates(t *testing.T) {
 			tt.opts.durability = durability
 			logger := logutil.NewMemoryLogger()
 			erp := NewEmergencyReparenter(nil, nil, logger)
-			tabletList, err := erp.filterValidCandidates(tt.validTablets, tt.tabletsReachable, tt.prevPrimary, tt.opts)
+			tabletList, err := erp.filterValidCandidates(tt.validTablets, tt.tabletsReachable, tt.tabletsBackingUp, tt.prevPrimary, tt.opts)
 			if tt.errShouldContain != "" {
 				require.Error(t, err)
 				require.Contains(t, err.Error(), tt.errShouldContain)