Skip to content

Commit

Permalink
Merge pull request #10706 from typeid/10705
Browse files Browse the repository at this point in the history
🐛 Speed up ignoring terminating Pods when draining unreachable Nodes
  • Loading branch information
k8s-ci-robot authored Jun 17, 2024
2 parents 9411031 + debbc3a commit 5b6043e
Showing 1 changed file with 12 additions and 2 deletions.
14 changes: 12 additions & 2 deletions internal/controllers/machine/machine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -670,8 +670,18 @@ func (r *Reconciler) drainNode(ctx context.Context, cluster *clusterv1.Cluster,
}

if noderefutil.IsNodeUnreachable(node) {
// When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them.
drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes
// Kubelet is unreachable, pods will never disappear.

// SkipWaitForDeleteTimeoutSeconds ensures the drain completes
// even if pod objects are not deleted.
drainer.SkipWaitForDeleteTimeoutSeconds = 1

// kube-apiserver sets the `deletionTimestamp` to a future date computed using the grace period.
// We are effectively waiting for GracePeriodSeconds + SkipWaitForDeleteTimeoutSeconds.
// Override the grace period of pods to reduce the time needed to skip them.
drainer.GracePeriodSeconds = 1

log.V(5).Info("Node is unreachable, draining will ignore gracePeriod. PDBs are still honored.")
}

if err := kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil {
Expand Down

0 comments on commit 5b6043e

Please sign in to comment.