K8SPSMDB-974: add sleep infinity for mongod pods (#1315)

* K8SPSMDB-974: add `sleep infinity` for mongod pods https://jira.percona.com/browse/K8SPSMDB-974 * fix `golangci-lint` warns and replace `context.TODO()` * fix tests * fix tests * fix tests --------- Co-authored-by: Ege Güneş <[email protected]> Co-authored-by: Inel Pandzic <[email protected]>
percona · Sep 18, 2023 · 9340bef · 9340bef
1 parent f005fe4
commit 9340bef
Show file tree

Hide file tree

Showing 136 changed files with 1,031 additions and 315 deletions.
diff --git a/build/ps-entry.sh b/build/ps-entry.sh
@@ -46,15 +46,15 @@ fi
 file_env() {
 	local var="$1"
 	local fileVar="${var}_FILE"
-	local def="${2:-}"
-	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+	local def="${2-}"
+	if [ "${!var-}" ] && [ "${!fileVar-}" ]; then
 		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
 		exit 1
 	fi
 	local val="$def"
-	if [ "${!var:-}" ]; then
+	if [ "${!var-}" ]; then
 		val="${!var}"
-	elif [ "${!fileVar:-}" ]; then
+	elif [ "${!fileVar-}" ]; then
 		val="$(<"${!fileVar}")"
 	fi
 	export "$var"="$val"
@@ -245,7 +245,20 @@ _dbPath() {
 	echo "$dbPath"
 }
 
+is_manual_recovery() {
+	recovery_file='/data/db/sleep-forever'
+	if [ -f "${recovery_file}" ]; then
+		echo "The $recovery_file file is detected, node is going to infinity loop"
+		echo "If you want to exit from infinity loop you need to remove $recovery_file file"
+		while [ -f "${recovery_file}" ]; do
+			sleep 1
+		done
+	fi
+}
+
 if [ "$originalArgOne" = 'mongod' ]; then
+	is_manual_recovery
+
 	file_env 'MONGO_INITDB_ROOT_USERNAME'
 	file_env 'MONGO_INITDB_ROOT_PASSWORD'
 	# pre-check a few factors to see if it's even worth bothering with initdb

diff --git a/cmd/mongodb-healthcheck/main.go b/cmd/mongodb-healthcheck/main.go
@@ -65,6 +65,15 @@ func main() {
 		os.Exit(0)
 	}
 
+	sleepForever, err := fileExists("/data/db/sleep-forever")
+	if err != nil {
+		log.Error(err, "check if sleep-forever file exists")
+		os.Exit(1)
+	}
+	if sleepForever {
+		os.Exit(0)
+	}
+
 	cnf, err := db.NewConfig(
 		app,
 		pkg.EnvMongoDBClusterMonitorUser,
@@ -82,38 +91,22 @@ func main() {
 		os.Exit(1)
 	}
 
-	client, err := db.Dial(ctx, cnf)
-	if err != nil {
-		log.Error(err, "connection error")
-		os.Exit(1)
-	}
-
-	defer func() {
-		if err := client.Disconnect(ctx); err != nil {
-			log.Error(err, "failed to disconnect")
-			os.Exit(1)
-		}
-	}()
-
 	switch command {
-
 	case "k8s liveness":
 		log.Info("Running Kubernetes liveness check for", "component", component)
 		switch *component {
 
 		case "mongod":
-			memberState, err := healthcheck.HealthCheckMongodLiveness(client, int64(*startupDelaySeconds))
+			memberState, err := healthcheck.HealthCheckMongodLiveness(ctx, cnf, int64(*startupDelaySeconds))
 			if err != nil {
-				client.Disconnect(ctx) // nolint:golint,errcheck
 				log.Error(err, "Member failed Kubernetes liveness check")
 				os.Exit(1)
 			}
 			log.Info("Member passed Kubernetes liveness check with replication state", "state", memberState)
 
 		case "mongos":
-			err := healthcheck.HealthCheckMongosLiveness(client)
+			err := healthcheck.HealthCheckMongosLiveness(ctx, cnf)
 			if err != nil {
-				client.Disconnect(ctx) // nolint:golint,errcheck
 				log.Error(err, "Member failed Kubernetes liveness check")
 				os.Exit(1)
 			}
@@ -125,14 +118,14 @@ func main() {
 		switch *component {
 
 		case "mongod":
-			client.Disconnect(ctx) // nolint:golint,errcheck
-			log.Error(err, "readiness check for mongod is not implemented")
-			os.Exit(1)
-
+			err := healthcheck.MongodReadinessCheck(ctx, cnf.Hosts[0])
+			if err != nil {
+				log.Error(err, "Member failed Kubernetes readiness check")
+				os.Exit(1)
+			}
 		case "mongos":
-			err := healthcheck.MongosReadinessCheck(ctx, client)
+			err := healthcheck.MongosReadinessCheck(ctx, cnf)
 			if err != nil {
-				client.Disconnect(ctx) // nolint:golint,errcheck
 				log.Error(err, "Member failed Kubernetes readiness check")
 				os.Exit(1)
 			}

diff --git a/e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter-oc.yml b/e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter-oc.yml
@@ -104,12 +104,17 @@ spec:
           name: mongodb
           protocol: TCP
         readinessProbe:
+          exec:
+            command:
+              - /opt/percona/mongodb-healthcheck
+              - k8s
+              - readiness
+              - --component
+              - mongod
           failureThreshold: 8
           initialDelaySeconds: 10
           periodSeconds: 3
           successThreshold: 1
-          tcpSocket:
-            port: 27017
           timeoutSeconds: 2
         resources: {}
         securityContext:

diff --git a/e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter.yml b/e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter.yml
@@ -98,12 +98,17 @@ spec:
           name: mongodb
           protocol: TCP
         readinessProbe:
+          exec:
+            command:
+              - /opt/percona/mongodb-healthcheck
+              - k8s
+              - readiness
+              - --component
+              - mongod
           failureThreshold: 8
           initialDelaySeconds: 10
           periodSeconds: 3
           successThreshold: 1
-          tcpSocket:
-            port: 27017
           timeoutSeconds: 2
         resources: {}
         securityContext:

diff --git a/e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter-oc.yml b/e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter-oc.yml
@@ -105,12 +105,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 8
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter.yml b/e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter.yml
@@ -105,12 +105,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 8
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml b/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml
@@ -92,12 +92,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 3
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml b/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml
@@ -92,12 +92,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 3
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml b/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml
@@ -93,12 +93,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 8
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources:
             limits:

diff --git a/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml b/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml
@@ -93,12 +93,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 8
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources:
             limits:

diff --git a/e2e-tests/default-cr/compare/statefulset_minimal-cluster-cfg-oc.yml b/e2e-tests/default-cr/compare/statefulset_minimal-cluster-cfg-oc.yml
@@ -106,12 +106,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 3
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/default-cr/compare/statefulset_minimal-cluster-cfg.yml b/e2e-tests/default-cr/compare/statefulset_minimal-cluster-cfg.yml
@@ -106,12 +106,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 3
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/default-cr/compare/statefulset_minimal-cluster-rs0-oc.yml b/e2e-tests/default-cr/compare/statefulset_minimal-cluster-rs0-oc.yml
@@ -106,12 +106,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 8
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/default-cr/compare/statefulset_minimal-cluster-rs0.yml b/e2e-tests/default-cr/compare/statefulset_minimal-cluster-rs0.yml
@@ -106,12 +106,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 8
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources: {}
           securityContext:

diff --git a/e2e-tests/default-cr/compare/statefulset_my-cluster-name-cfg-oc.yml b/e2e-tests/default-cr/compare/statefulset_my-cluster-name-cfg-oc.yml
@@ -94,12 +94,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 3
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources:
             limits:

diff --git a/e2e-tests/default-cr/compare/statefulset_my-cluster-name-cfg.yml b/e2e-tests/default-cr/compare/statefulset_my-cluster-name-cfg.yml
@@ -94,12 +94,17 @@ spec:
               name: mongodb
               protocol: TCP
           readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
             failureThreshold: 3
             initialDelaySeconds: 10
             periodSeconds: 3
             successThreshold: 1
-            tcpSocket:
-              port: 27017
             timeoutSeconds: 2
           resources:
             limits: