Merge pull request #7 from tomp21/feat/selfheal

Feat/selfheal
tomp21 · Dec 26, 2024 · f9ef28b · f9ef28b
2 parents b9ffe80 + 2db2bbe
commit f9ef28b
Show file tree

Hide file tree

Showing 8 changed files with 212 additions and 8 deletions.
diff --git a/internal/controller/reconcilers/configmap_reconciler.go b/internal/controller/reconcilers/configmap_reconciler.go
@@ -28,6 +28,24 @@ var redisMasterConf string
 //go:embed configmaps/conf/replica.conf
 var redisReplicaConf string
 
+//go:embed configmaps/health/ping_liveness_local.sh
+var redisLivenessLocal string
+
+//go:embed configmaps/health/ping_readiness_local.sh
+var redisReadinessLocal string
+
+//go:embed configmaps/health/ping_liveness_master.sh
+var redisLivenessMaster string
+
+//go:embed configmaps/health/ping_readiness_master.sh
+var redisReadinessMaster string
+
+//go:embed configmaps/health/ping_liveness_local_and_master.sh
+var redisLivenessLocalMaster string
+
+//go:embed configmaps/health/ping_readiness_local_and_master.sh
+var redisReadinessLocalMaster string
+
 type ConfigMapReconciler struct {
 	client *client.Client
 	scheme *runtime.Scheme
@@ -49,7 +67,11 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, redis *cachev1alpha
 	if err != nil {
 		return err
 	}
-	return nil
+
+	err = r.ReconcileHealthScripts(ctx, redis)
+
+	return err
+
 }
 
 func (r *ConfigMapReconciler) ReconcileStartScripts(ctx context.Context, redis *cachev1alpha1.Redis) error {
@@ -67,10 +89,8 @@ func (r *ConfigMapReconciler) ReconcileStartScripts(ctx context.Context, redis *
 	_, err := controllerutil.CreateOrUpdate(ctx, *r.client, cmStartScripts, func() error {
 		return controllerutil.SetControllerReference(redis, cmStartScripts, r.scheme)
 	})
-	if err != nil {
-		return err
-	}
-	return nil
+
+	return err
 }
 
 func (r *ConfigMapReconciler) ReconcileConfigs(ctx context.Context, redis *cachev1alpha1.Redis) error {
@@ -90,10 +110,29 @@ func (r *ConfigMapReconciler) ReconcileConfigs(ctx context.Context, redis *cache
 	_, err := controllerutil.CreateOrUpdate(ctx, *r.client, cmConfig, func() error {
 		return controllerutil.SetControllerReference(redis, cmConfig, r.scheme)
 	})
+	return err
+}
 
-	if err != nil {
-		return err
+func (r *ConfigMapReconciler) ReconcileHealthScripts(ctx context.Context, redis *cachev1alpha1.Redis) error {
+	cmHealth := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "redis-health",
+			Namespace: redis.Namespace,
+			Labels:    util.GetLabels(redis, nil),
+		},
+		Data: map[string]string{
+			"ping_liveness_local.sh":             redisLivenessLocal,
+			"ping_readiness_local.sh":            redisReadinessLocal,
+			"ping_liveness_master.sh":            redisLivenessMaster,
+			"ping_readiness_master.sh":           redisReadinessMaster,
+			"ping_liveness_local_and_master.sh":  redisLivenessLocalMaster,
+			"ping_readiness_local_and_master.sh": redisReadinessLocalMaster,
+		},
 	}
 
-	return nil
+	_, err := controllerutil.CreateOrUpdate(ctx, *r.client, cmHealth, func() error {
+		return controllerutil.SetControllerReference(redis, cmHealth, r.scheme)
+	})
+
+	return err
 }
diff --git a/internal/controller/reconcilers/configmaps/health/ping_liveness_local.sh b/internal/controller/reconcilers/configmaps/health/ping_liveness_local.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+[[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")"
+[[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD"
+response=$(
+  timeout -s 15 $1 \
+  redis-cli \
+    -h localhost \
+    -p $REDIS_PORT \
+    ping
+)
+if [ "$?" -eq "124" ]; then
+  echo "Timed out"
+  exit 1
+fi
+responseFirstWord=$(echo $response | head -n1 | awk '{print $1;}')
+if [ "$response" != "PONG" ] && [ "$responseFirstWord" != "LOADING" ] && [ "$responseFirstWord" != "MASTERDOWN" ]; then
+  echo "$response"
+  exit 1
+fi
diff --git a/internal/controller/reconcilers/configmaps/health/ping_liveness_local_and_master.sh b/internal/controller/reconcilers/configmaps/health/ping_liveness_local_and_master.sh
@@ -0,0 +1,5 @@
+script_dir="$(dirname "$0")"
+exit_status=0
+"$script_dir/ping_liveness_local.sh" $1 || exit_status=$?
+"$script_dir/ping_liveness_master.sh" $1 || exit_status=$?
+exit $exit_status
diff --git a/internal/controller/reconcilers/configmaps/health/ping_liveness_master.sh b/internal/controller/reconcilers/configmaps/health/ping_liveness_master.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+[[ -f $REDIS_MASTER_PASSWORD_FILE ]] && export REDIS_MASTER_PASSWORD="$(< "${REDIS_MASTER_PASSWORD_FILE}")"
+[[ -n "$REDIS_MASTER_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_MASTER_PASSWORD"
+response=$(
+  timeout -s 15 $1 \
+  redis-cli \
+    -h $REDIS_MASTER_HOST \
+    -p $REDIS_PORT \
+    ping
+)
+if [ "$?" -eq "124" ]; then
+  echo "Timed out"
+  exit 1
+fi
+responseFirstWord=$(echo $response | head -n1 | awk '{print $1;}')
+if [ "$response" != "PONG" ] && [ "$responseFirstWord" != "LOADING" ]; then
+  echo "$response"
+  exit 1
+fi
diff --git a/internal/controller/reconcilers/configmaps/health/ping_readiness_local.sh b/internal/controller/reconcilers/configmaps/health/ping_readiness_local.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+[[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")"
+[[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD"
+response=$(
+  timeout -s 15 $1 \
+  redis-cli \
+    -h localhost \
+    -p $REDIS_PORT \
+    ping
+)
+if [ "$?" -eq "124" ]; then
+  echo "Timed out"
+  exit 1
+fi
+if [ "$response" != "PONG" ]; then
+  echo "$response"
+  exit 1
+fi
diff --git a/internal/controller/reconcilers/configmaps/health/ping_readiness_local_and_master.sh b/internal/controller/reconcilers/configmaps/health/ping_readiness_local_and_master.sh
@@ -0,0 +1,5 @@
+script_dir="$(dirname "$0")"
+exit_status=0
+"$script_dir/ping_readiness_local.sh" $1 || exit_status=$?
+"$script_dir/ping_readiness_master.sh" $1 || exit_status=$?
+exit $exit_status
diff --git a/internal/controller/reconcilers/configmaps/health/ping_readiness_master.sh b/internal/controller/reconcilers/configmaps/health/ping_readiness_master.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+[[ -f $REDIS_MASTER_PASSWORD_FILE ]] && export REDIS_MASTER_PASSWORD="$(< "${REDIS_MASTER_PASSWORD_FILE}")"
+[[ -n "$REDIS_MASTER_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_MASTER_PASSWORD"
+response=$(
+  timeout -s 15 $1 \
+  redis-cli \
+    -h $REDIS_MASTER_HOST \
+    -p $REDIS_PORT \
+    ping
+)
+if [ "$?" -eq "124" ]; then
+  echo "Timed out"
+  exit 1
+fi
+if [ "$response" != "PONG" ]; then
+  echo "$response"
+  exit 1
+fi
diff --git a/internal/controller/reconcilers/statefulset_reconciler.go b/internal/controller/reconcilers/statefulset_reconciler.go
@@ -215,6 +215,32 @@ func getMasterContainers(name, image string) []corev1.Container {
 					Name:      fmt.Sprintf("%s-data", name),
 					MountPath: "/data",
 				},
+				{
+					Name:      "health",
+					MountPath: "/health",
+				},
+			},
+			LivenessProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{
+							"sh",
+							"-c",
+							"/health/ping_liveness_local.sh 5",
+						},
+					},
+				},
+			},
+			ReadinessProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{
+							"sh",
+							"-c",
+							"/health/ping_readiness_local.sh 1",
+						},
+					},
+				},
 			},
 			Resources: getResources(),
 		},
@@ -240,6 +266,17 @@ func getReplicaContainers(name, image string) []corev1.Container {
 						},
 					},
 				},
+				{
+					Name: "REDIS_MASTER_PASSWORD",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: name,
+							},
+							Key: PasswordSecretKey,
+						},
+					},
+				},
 				{
 					Name:  "REDIS_REPLICATION_MODE",
 					Value: "replica",
@@ -256,6 +293,10 @@ func getReplicaContainers(name, image string) []corev1.Container {
 					Name:  "HEADLESS_SERVICE",
 					Value: fmt.Sprintf("%s-headless", name),
 				},
+				{
+					Name:  "ALLOW_EMPLTY_PASSWORD",
+					Value: "no",
+				},
 			},
 			Ports: []corev1.ContainerPort{
 				{
@@ -275,6 +316,32 @@ func getReplicaContainers(name, image string) []corev1.Container {
 					Name:      fmt.Sprintf("%s-data", name),
 					MountPath: "/data",
 				},
+				{
+					Name:      "health",
+					MountPath: "/health",
+				},
+			},
+			LivenessProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{
+							"sh",
+							"-c",
+							"/health/ping_liveness_local_and_master.sh 5",
+						},
+					},
+				},
+			},
+			ReadinessProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{
+							"sh",
+							"-c",
+							"/health/ping_readiness_local_and_master.sh 1",
+						},
+					},
+				},
 			},
 			Resources: getResources(),
 		},
@@ -329,6 +396,16 @@ func getVolumes() []corev1.Volume {
 					DefaultMode: &defaultMode,
 				},
 			},
+		}, {
+			Name: "health",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: "redis-health",
+					},
+					DefaultMode: &defaultMode,
+				},
+			},
 		},
 	}
 }