From 82ada80717d9147edc0267ac9073fb651235a305 Mon Sep 17 00:00:00 2001 From: Phan Le Date: Sat, 18 Jan 2025 20:24:33 -0800 Subject: [PATCH] fix: bug cannot use the disk which has failed replica We should not exclude the disk which has failed replica in 2 cases: 1. The caller is trying to reuse the failed replica, this disk is a valid candidate 2. The failed replica is no longer reusable, this disk is a valid candidate longhorn-10210 Signed-off-by: Phan Le --- scheduler/replica_scheduler.go | 12 ++++++++++-- scheduler/replica_scheduler_test.go | 4 +++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/scheduler/replica_scheduler.go b/scheduler/replica_scheduler.go index f71537aef2..824d6a76e6 100644 --- a/scheduler/replica_scheduler.go +++ b/scheduler/replica_scheduler.go @@ -265,7 +265,7 @@ func (rcs *ReplicaScheduler) getDiskCandidates(nodeInfo map[string]*longhorn.Nod } multiError.Append(errors) } - diskCandidates = filterDisksWithMatchingReplicas(diskCandidates, replicas, diskSoftAntiAffinity) + diskCandidates = filterDisksWithMatchingReplicas(diskCandidates, replicas, diskSoftAntiAffinity, ignoreFailedReplicas) return diskCandidates, multiError } @@ -467,9 +467,17 @@ func (rcs *ReplicaScheduler) filterNodeDisksForReplica(node *longhorn.Node, disk // filterDiskWithMatchingReplicas returns disk that have no matching replicas when diskSoftAntiAffinity is false. // Otherwise, it returns the input disks map. func filterDisksWithMatchingReplicas(disks map[string]*Disk, replicas map[string]*longhorn.Replica, - diskSoftAntiAffinity bool) map[string]*Disk { + diskSoftAntiAffinity, ignoreFailedReplicas bool) map[string]*Disk { replicasCountPerDisk := map[string]int{} for _, r := range replicas { + if r.Spec.FailedAt != "" { + if ignoreFailedReplicas { + continue + } + if !IsPotentiallyReusableReplica(r) { + continue // This replica can never be used again, so it does not count in scheduling decisions. + } + } replicasCountPerDisk[r.Spec.DiskID]++ } diff --git a/scheduler/replica_scheduler_test.go b/scheduler/replica_scheduler_test.go index a8ede45f50..5170fe156a 100644 --- a/scheduler/replica_scheduler_test.go +++ b/scheduler/replica_scheduler_test.go @@ -1392,6 +1392,7 @@ func (s *TestSuite) TestFilterDisksWithMatchingReplicas(c *C) { inputDiskUUIDs []string inputReplicas map[string]*longhorn.Replica diskSoftAntiAffinity bool + ignoreFailedReplicas bool expectDiskUUIDs []string } @@ -1454,6 +1455,7 @@ func (s *TestSuite) TestFilterDisksWithMatchingReplicas(c *C) { replica4.Name: replica4, } tc.diskSoftAntiAffinity = true + tc.ignoreFailedReplicas = false tc.expectDiskUUIDs = []string{diskUUID5} // Only disk5 has no matching replica. tests["only schedule to disk without matching replica"] = tc @@ -1463,7 +1465,7 @@ func (s *TestSuite) TestFilterDisksWithMatchingReplicas(c *C) { for _, UUID := range tc.inputDiskUUIDs { inputDisks[UUID] = &Disk{} } - outputDiskUUIDs := filterDisksWithMatchingReplicas(inputDisks, tc.inputReplicas, tc.diskSoftAntiAffinity) + outputDiskUUIDs := filterDisksWithMatchingReplicas(inputDisks, tc.inputReplicas, tc.diskSoftAntiAffinity, tc.ignoreFailedReplicas) c.Assert(len(outputDiskUUIDs), Equals, len(tc.expectDiskUUIDs)) for _, UUID := range tc.expectDiskUUIDs { _, ok := outputDiskUUIDs[UUID]