feat(provider): add liveness healthchecks

akash-network · Mar 7, 2024 · e08e95a · e08e95a
1 parent e96755c
commit e08e95a
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 8 deletions.
diff --git a/charts/akash-provider/Chart.yaml b/charts/akash-provider/Chart.yaml
@@ -17,7 +17,7 @@ type: application
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
 # Major version bit highlights the mainnet release (e.g. mainnet4 = 4.x.x, mainnet5 = 5.x.x, ...)
-version: 9.0.6
+version: 9.0.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/akash-provider/scripts/liveness_checks.sh b/charts/akash-provider/scripts/liveness_checks.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Ensure the script fails if any part of a pipeline fails
+set -o pipefail
+
+# Provider API /status check
+if ! timeout 5s curl -o /dev/null -fsk https://127.0.0.1:8443/status; then
+  echo "api /status check failed"
+  exit 1
+fi
+
+# Provider gRPC check
+if ! timeout 5s curl -k -v --http2-prior-knowledge https://127.0.0.1:8444 2>&1 | grep -qi 'application/grpc'; then
+  echo "gRPC check failed"
+  exit 1
+fi
+
+# RPC node sync check
+current_time=$(date -u +%s)
+latest_block_time_str=$(curl -s $AKASH_NODE/status | jq -r '.result.sync_info.latest_block_time')
+latest_block_time=$(date -u -d "$latest_block_time_str" +%s)
+
+# Allow for a 60 seconds drift
+let "time_diff = current_time - latest_block_time"
+if [ "$time_diff" -gt 60 ] || [ "$time_diff" -lt -60 ]; then
+  echo "RPC node sync check failed"
+  exit 1
+fi
+
+echo "All checks passed"
+exit 0
diff --git a/charts/akash-provider/templates/configmap-boot.yaml b/charts/akash-provider/templates/configmap-boot.yaml
@@ -8,3 +8,5 @@ data:
 {{ tpl (.Files.Get "scripts/init.sh") . | indent 4 }}
   run.sh: |
 {{ tpl (.Files.Get "scripts/run.sh") . | indent 4 }}
+  liveness_checks.sh: |
+{{ tpl (.Files.Get "scripts/liveness_checks.sh") . | indent 4 }}
diff --git a/...vider/templates/configmap-bidscripts.yaml → ...provider/templates/configmap-scripts.yaml b/...vider/templates/configmap-bidscripts.yaml → ...provider/templates/configmap-scripts.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "provider.fullname" . }}-bidscripts
+  name: {{ include "provider.fullname" . }}-scripts
   namespace: {{ .Release.Namespace }}
 data:
   dummy: |
@@ -10,3 +10,5 @@ data:
   price_script.sh: |
 {{ .Values.bidpricescript | b64dec | indent 4 }}
 {{- end }}
+  liveness_checks.sh: |
+{{ tpl (.Files.Get "scripts/liveness_checks.sh") . | indent 4 }}
diff --git a/charts/akash-provider/templates/statefulset.yaml b/charts/akash-provider/templates/statefulset.yaml
@@ -19,7 +19,7 @@ spec:
     metadata:
       annotations:
         checksum/cm-boot: {{ include (print $.Template.BasePath "/configmap-boot.yaml") . | sha256sum }}
-        checksum/cm-bidscripts: {{ include (print $.Template.BasePath "/configmap-bidscripts.yaml") . | sha256sum }}
+        checksum/cm-scripts: {{ include (print $.Template.BasePath "/configmap-scripts.yaml") . | sha256sum }}
       labels:
         app: {{ include "provider.fullname" . }}
         {{- include "provider.selectorLabels" . | nindent 8 }}
@@ -176,7 +176,7 @@ spec:
               value: "{{ ternary "scale" "shellScript" (empty .Values.bidpricescript) }}"
 {{ if .Values.bidpricescript }}
             - name: AKASH_BID_PRICE_SCRIPT_PATH
-              value: "/bidscripts/price_script.sh"
+              value: "/scripts/price_script.sh"
 {{- end }}
 {{ if .Values.whitelist_url }}
             - name: WHITELIST_URL
@@ -259,12 +259,23 @@ spec:
               containerPort: 8444
               protocol: TCP
 
+          livenessProbe:
+            exec:
+              command:
+              - sh
+              - -c
+              - /scripts/liveness_checks.sh
+            initialDelaySeconds: 120
+            periodSeconds: 10
+            failureThreshold: 1
+            timeoutSeconds: 10
+
           volumeMounts:
             - name: boot
               mountPath: /boot
               readOnly: true
-            - name: bidscripts
-              mountPath: /bidscripts
+            - name: scripts
+              mountPath: /scripts
               readOnly: true
             - name: keys
               mountPath: /boot-keys
@@ -277,9 +288,9 @@ spec:
         - name: boot
           configMap:
             name: {{ include "provider.fullname" . }}-boot
-        - name: bidscripts
+        - name: scripts
           configMap:
-            name: {{ include "provider.fullname" . }}-bidscripts
+            name: {{ include "provider.fullname" . }}-scripts
             defaultMode: 0744
         - name: keys
           secret: