Sidecar terminator ignore the exit code of the sidecar container

Signed-off-by: liuzhenwei <[email protected]>

add ut

Signed-off-by: liuzhenwei <[email protected]>

add some comments and simplified some code

Signed-off-by: liuzhenwei <[email protected]>

remove unnecessary pod status operations

Signed-off-by: liuzhenwei <[email protected]>

change pod to terminal phase before create crr

Signed-off-by: liuzhenwei <[email protected]>

pkg/controller/sidecarterminator/sidecar_terminator_controller.go

@@ -22,31 +22,36 @@ import (
 	"strings"
 	"time"
 
-	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
-	"github.com/openkruise/kruise/pkg/features"
-	"github.com/openkruise/kruise/pkg/util"
-	utilclient "github.com/openkruise/kruise/pkg/util/client"
-	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
-	"github.com/openkruise/kruise/pkg/util/ratelimiter"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	"github.com/openkruise/kruise/pkg/features"
+	"github.com/openkruise/kruise/pkg/util"
+	utilclient "github.com/openkruise/kruise/pkg/util/client"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
+	"github.com/openkruise/kruise/pkg/util/ratelimiter"
 )
 
 func init() {
 	flag.IntVar(&concurrentReconciles, "sidecarterminator-workers", concurrentReconciles, "Max concurrent workers for SidecarTerminator controller.")
 }
 
 var (
-	concurrentReconciles = 3
+	concurrentReconciles                         = 3
+	SidecarTerminated    corev1.PodConditionType = "SidecarTerminated"
 )
 
 /**
@@ -71,6 +76,7 @@ func newReconciler(mgr manager.Manager) reconcile.Reconciler {
 		Client:   cli,
 		recorder: recorder,
 		scheme:   mgr.GetScheme(),
+		clock:    clock.RealClock{},
 	}
 }
 
@@ -101,6 +107,7 @@ type ReconcileSidecarTerminator struct {
 	client.Client
 	recorder record.EventRecorder
 	scheme   *runtime.Scheme
+	clock    clock.Clock
 }
 
 // Reconcile get the pod whose sidecar containers should be stopped, and stop them.
@@ -131,8 +138,8 @@ func (r *ReconcileSidecarTerminator) doReconcile(pod *corev1.Pod) (reconcile.Res
 		return reconcile.Result{}, nil
 	}
 
-	if containersCompleted(pod, getSidecar(pod)) {
-		klog.V(3).Infof("SidecarTerminator -- all sidecars of pod(%v/%v) have been completed, no need to process", pod.Namespace, pod.Name)
+	if containersSucceeded(pod, getSidecar(pod)) {
+		klog.V(3).Infof("SidecarTerminator -- all sidecars of pod(%v/%v) have been succeeded, no need to process", pod.Namespace, pod.Name)
 		return reconcile.Result{}, nil
 	}
 
@@ -141,7 +148,8 @@ func (r *ReconcileSidecarTerminator) doReconcile(pod *corev1.Pod) (reconcile.Res
 		return reconcile.Result{}, nil
 	}
 
-	sidecarNeedToExecuteKillContainer, sidecarNeedToExecuteInPlaceUpdate, err := r.groupSidecars(pod)
+	sidecarNeedToExecuteKillContainer, sidecarNeedToExecuteInPlaceUpdate, sidecarNeedToSyncStatus, err := r.groupSidecars(pod)
+
 	if err != nil {
 		return reconcile.Result{}, err
 	}
@@ -150,27 +158,88 @@ func (r *ReconcileSidecarTerminator) doReconcile(pod *corev1.Pod) (reconcile.Res
 		return reconcile.Result{}, err
 	}
 
+	if err := r.terminateJobPod(pod, sidecarNeedToSyncStatus); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	if err := r.executeKillContainerAction(pod, sidecarNeedToExecuteKillContainer); err != nil {
 		return reconcile.Result{}, err
 	}
 
 	return reconcile.Result{}, nil
 }
 
-func (r *ReconcileSidecarTerminator) groupSidecars(pod *corev1.Pod) (sets.String, sets.String, error) {
+// terminateJobPod terminate the job pod and skip the state of the sidecar containers
+// This method should only be called after the executeKillContainerAction is called
+func (r *ReconcileSidecarTerminator) terminateJobPod(pod *corev1.Pod, sidecars sets.String) error {
+	if sidecars.Len() == 0 {
+		return nil
+	}
+	if isJobPodCompleted(pod) {
+		klog.V(3).Infof("the job pod is completed -- pod (%s/%s) ,no need to process", pod.Namespace, pod.Name)
+		return nil
+	}
+
+	// after the pod is terminated by the sidecar terminator, kubelet will kill the containers that are not in the terminal phase
+	// 1. sidecar container terminate with non-zero exit code
+	// 2. sidecar container is not in a terminal phase (still running or waiting)
+	if !containersSucceeded(pod, sidecars) {
+		klog.V(3).Infof("all of the main containers are completed, will terminate the job pod %s/%s", pod.Namespace, pod.Name)
+		// terminate the pod, ignore the status of the sidecar containers.
+		// kubelet will not override pod phase after we updated (in kubelet,pods are not allowed to transition out of terminal phases).
+		if containersSucceeded(pod, getMain(pod)) {
+			pod.Status.Phase = corev1.PodSucceeded
+			for i, condition := range pod.Status.Conditions {
+				if condition.Type == corev1.PodReady || condition.Type == corev1.ContainersReady {
+					pod.Status.Conditions[i].Reason = "PodCompleted"
+					pod.Status.Conditions[i].Status = corev1.ConditionTrue
+				}
+			}
+		} else {
+			pod.Status.Phase = corev1.PodFailed
+			for i, condition := range pod.Status.Conditions {
+				if condition.Type == corev1.PodReady || condition.Type == corev1.ContainersReady {
+					pod.Status.Conditions[i].Reason = "PodFailed"
+					pod.Status.Conditions[i].Status = corev1.ConditionFalse
+				}
+			}
+		}
+
+		// condition
+		_, condition := podutil.GetPodCondition(&pod.Status, SidecarTerminated)
+		if condition == nil {
+			pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
+				Type:               SidecarTerminated,
+				Status:             corev1.ConditionTrue,
+				LastTransitionTime: metav1.Now(),
+				Message:            "Terminated by Sidecar Terminator",
+			})
+		} else {
+			condition.LastTransitionTime = metav1.Now()
+		}
+		klog.V(3).Infof("terminate the job pod %s/%s phase=%s", pod.Namespace, pod.Name, pod.Status.Phase)
+		return r.Status().Update(context.TODO(), pod)
+	}
+
+	return nil
+}
+
+func (r *ReconcileSidecarTerminator) groupSidecars(pod *corev1.Pod) (sets.String, sets.String, sets.String, error) {
 	runningOnVK, err := IsPodRunningOnVirtualKubelet(pod, r.Client)
 	if err != nil {
-		return nil, nil, client.IgnoreNotFound(err)
+		return nil, nil, nil, client.IgnoreNotFound(err)
 	}
 
 	inPlaceUpdate := sets.NewString()
 	killContainer := sets.NewString()
+	syncStatusContainer := sets.NewString()
 	for i := range pod.Spec.Containers {
 		container := &pod.Spec.Containers[i]
 		for j := range container.Env {
 			if !runningOnVK && container.Env[j].Name == appsv1alpha1.KruiseTerminateSidecarEnv &&
 				strings.EqualFold(container.Env[j].Value, "true") {
 				killContainer.Insert(container.Name)
+				syncStatusContainer.Insert(container.Name)
 				break
 			}
 			if container.Env[j].Name == appsv1alpha1.KruiseTerminateSidecarWithImageEnv &&
@@ -179,7 +248,7 @@ func (r *ReconcileSidecarTerminator) groupSidecars(pod *corev1.Pod) (sets.String
 			}
 		}
 	}
-	return killContainer, inPlaceUpdate, nil
+	return killContainer, inPlaceUpdate, syncStatusContainer, nil
 }
 
 func containersCompleted(pod *corev1.Pod, containers sets.String) bool {
@@ -210,3 +279,14 @@ func containersSucceeded(pod *corev1.Pod, containers sets.String) bool {
 	}
 	return true
 }
+
+func isJobPodCompleted(pod *corev1.Pod) bool {
+	mainContainers := getMain(pod)
+	if !containersCompleted(pod, mainContainers) {
+		return false
+	}
+	if containersSucceeded(pod, mainContainers) {
+		return pod.Status.Phase == corev1.PodSucceeded
+	}
+	return pod.Status.Phase == corev1.PodFailed
+}