Skip to content

Commit

Permalink
[PLAY-751] fix canary deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
ghkadim committed Feb 7, 2024
1 parent 89f394a commit 2883009
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 5 deletions.
17 changes: 13 additions & 4 deletions internal/controllers/app_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ const (
maxWaitTimeDuration = time.Duration(120) * time.Second
)

var (
errNotAllPodsRunning = errors.New("not all pods are running")
errNotAllPodsHealthy = errors.New("not all pods are in healthy state")
)

// +kubebuilder:rbac:groups=theketch.io,resources=apps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=theketch.io,resources=apps/status,verbs=get;update;patch
// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -380,9 +385,13 @@ func (r *AppReconciler) reconcile(ctx context.Context, app *ketchv1.App, logger
}

// retry until all pods for canary deployment comes to running state.
if _, err := checkPodStatus(r.Group, r.Client, app.Name, app.Spec.Deployments[1].Version); err != nil {
if podName, err := checkPodStatus(r.Group, r.Client, app.Name, app.Spec.Deployments[1].Version); err != nil {

if !timeoutExpired(app.Spec.Canary.Started, r.Now()) {
if errors.Is(err, errNotAllPodsRunning) || errors.Is(err, errNotAllPodsHealthy) {
logger.Info("waiting for canary pod", "pod", podName, "err", err)
return appReconcileResult{useTimeout: true}
}
return appReconcileResult{
err: fmt.Errorf("canary update failed: %w", err),
useTimeout: true,
Expand All @@ -401,7 +410,7 @@ func (r *AppReconciler) reconcile(ctx context.Context, app *ketchv1.App, logger
var hpaList autoscalingv2.HorizontalPodAutoscalerList
if err := r.List(ctx, &hpaList, &client.ListOptions{Namespace: app.Spec.Namespace}); err != nil {
return appReconcileResult{
err: fmt.Errorf("failed to find HPAs"),
err: fmt.Errorf("failed to find HPAs: %w", err),
}
}

Expand Down Expand Up @@ -789,12 +798,12 @@ func checkPodStatus(group string, c client.Client, appName string, depVersion ke
}

if pod.Status.Phase != v1.PodRunning {
return pod.Name, errors.New("all pods are not running")
return pod.Name, errNotAllPodsRunning
}

for _, c := range pod.Status.Conditions {
if c.Status != v1.ConditionTrue {
return pod.Name, errors.New("all pods are not in healthy state")
return pod.Name, errNotAllPodsHealthy
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion internal/controllers/app_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,7 @@ func Test_checkPodStatus(t *testing.T) {
createPod("theketch.io", "my-app", "5", v1.PodStatus{Phase: v1.PodPending}),
},
expectedPod: "my-app-5",
wantErr: `all pods are not running`,
wantErr: `not all pods are running`,
},
{
name: "pod in Pending state but group doesn't match",
Expand Down

0 comments on commit 2883009

Please sign in to comment.