From 8dea2dce83b8a24326ae2e3c59074cd0bd4f7be2 Mon Sep 17 00:00:00 2001 From: Jwalant Modi Date: Thu, 26 Sep 2024 13:40:33 +0530 Subject: [PATCH 1/2] Added reasons to logs for unscheduled pods --- internal/controller/cluster/pod.go | 2 +- pkg/utils/pod.go | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/controller/cluster/pod.go b/internal/controller/cluster/pod.go index 2b3c93315..5c2deb308 100644 --- a/internal/controller/cluster/pod.go +++ b/internal/controller/cluster/pod.go @@ -854,7 +854,7 @@ func (r *SingleClusterReconciler) getIgnorablePods(racksToDelete []asdbv1.Rack, pod := &podList.Items[podIdx] if !utils.IsPodRunningAndReady(pod) { - if utils.IsPodReasonUnschedulable(pod) { + if isPodUnschedulable, _ := utils.IsPodReasonUnschedulable(pod); isPodUnschedulable { pendingPod = append(pendingPod, pod.Name) continue } diff --git a/pkg/utils/pod.go b/pkg/utils/pod.go index 5addec1fc..1504d7531 100644 --- a/pkg/utils/pod.go +++ b/pkg/utils/pod.go @@ -30,11 +30,14 @@ func CheckPodFailed(pod *corev1.Pod) error { // if the value of ".status.phase" is "Failed", the pod is trivially in a failure state if pod.Status.Phase == corev1.PodFailed { - return fmt.Errorf("pod %s has failed status", pod.Name) + return fmt.Errorf("pod %s has failed status with reason: %s and message: %s", + pod.Name, pod.Status.Reason, pod.Status.Message) } - if pod.Status.Phase == corev1.PodPending && IsPodReasonUnschedulable(pod) { - return fmt.Errorf("pod %s is in unschedulable state", pod.Name) + if pod.Status.Phase == corev1.PodPending { + if isPodUnschedulable, err := IsPodReasonUnschedulable(pod); isPodUnschedulable { + return fmt.Errorf("pod %s is in unschedulable state and reason is %v", pod.Name, err) + } } // grab the status of every container in the pod (including its init containers) @@ -73,7 +76,8 @@ func CheckPodImageFailed(pod *corev1.Pod) error { // if the value of ".status.phase" is "Failed", the pod is trivially in a failure state if pod.Status.Phase == corev1.PodFailed { - return fmt.Errorf("pod has failed status") + return fmt.Errorf("pod %s has failed status with reason: %s and message: %s", + pod.Name, pod.Status.Reason, pod.Status.Message) } // grab the status of every container in the pod (including its init containers) @@ -198,13 +202,13 @@ func isPodError(reason string) bool { return strings.HasSuffix(reason, "Error") } -func IsPodReasonUnschedulable(pod *corev1.Pod) bool { +func IsPodReasonUnschedulable(pod *corev1.Pod) (bool, error) { for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodScheduled && (condition.Reason == corev1.PodReasonUnschedulable || condition.Reason == corev1.PodReasonSchedulerError) { - return true + return true, fmt.Errorf("%s", condition.Message) } } - return false + return false, nil } From 5034747fff6c05e08277e393077a27887de5343d Mon Sep 17 00:00:00 2001 From: Jwalant Modi Date: Fri, 27 Sep 2024 12:25:04 +0530 Subject: [PATCH 2/2] Added reasons to logs for unscheduled pods --- pkg/utils/pod.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/utils/pod.go b/pkg/utils/pod.go index 1504d7531..a5b9368a4 100644 --- a/pkg/utils/pod.go +++ b/pkg/utils/pod.go @@ -35,8 +35,8 @@ func CheckPodFailed(pod *corev1.Pod) error { } if pod.Status.Phase == corev1.PodPending { - if isPodUnschedulable, err := IsPodReasonUnschedulable(pod); isPodUnschedulable { - return fmt.Errorf("pod %s is in unschedulable state and reason is %v", pod.Name, err) + if isPodUnschedulable, reason := IsPodReasonUnschedulable(pod); isPodUnschedulable { + return fmt.Errorf("pod %s is in unschedulable state and reason is %s", pod.Name, reason) } } @@ -202,13 +202,13 @@ func isPodError(reason string) bool { return strings.HasSuffix(reason, "Error") } -func IsPodReasonUnschedulable(pod *corev1.Pod) (bool, error) { +func IsPodReasonUnschedulable(pod *corev1.Pod) (isPodUnschedulable bool, reason string) { for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodScheduled && (condition.Reason == corev1.PodReasonUnschedulable || condition.Reason == corev1.PodReasonSchedulerError) { - return true, fmt.Errorf("%s", condition.Message) + return true, condition.Message } } - return false, nil + return false, "" }