fix(charts): sequencer prometheus rules (#1598)

## Summary Fixes prometheus rules and metrics templates for both `sequencer` and `sequencer-relayer` charts ## Background ServiceMonitors for alerting and monitoring is missing on sequencer-relayer chart ## Changes - adds service monitor and prometheus rules templating for sequencer-relayer - fixes sequencer app labels ## Testing locally, dusk-10 ## Metrics -enables metrics and alerts on sequencer, sequencer-relayer charts ## Related Issues closes #1597
astriaorg · Sep 30, 2024 · a9f5197 · a9f5197
1 parent eced579
commit a9f5197
Show file tree

Hide file tree

Showing 9 changed files with 78 additions and 10 deletions.
diff --git a/charts/sequencer-relayer/Chart.yaml b/charts/sequencer-relayer/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.12.1
+version: 0.12.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/sequencer-relayer/templates/_helpers.tpl b/charts/sequencer-relayer/templates/_helpers.tpl
@@ -9,6 +9,28 @@ Namepsace to deploy elements into.
 {{ .Values.images.sequencerRelayer.repo }}:{{ if .Values.global.dev }}{{ .Values.images.sequencerRelayer.devTag }}{{ else }}{{ .Values.images.sequencerRelayer.tag }}{{ end }}
 {{- end }}
 
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "sequencer-relayer.name" -}}
+sequencer-relayer
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "sequencer-relayer.labels" -}}
+{{ include "sequencer-relayer.selectorLabels" . }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "sequencer-relayer.selectorLabels" -}}
+app: {{ include "sequencer-relayer.name" . }}
+name: {{ include "sequencer-relayer.name" . }}-metrics
+{{- end }}
+
 {{- define "sequencer-relayer.storage.mountPath" -}}
 /astria-sequencer-relayer
 {{- end }}

diff --git a/charts/sequencer-relayer/templates/prometheusrule.yaml b/charts/sequencer-relayer/templates/prometheusrule.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.alerting.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "sequencer-relayer.name" . }}
+{{- if .Values.alerting.prometheusRule.namespace }}
+  namespace: {{ .Values.alerting.prometheusRule.namespace | quote }}
+{{- end }}
+  labels:
+    {{- include "sequencer-relayer.labels" . | nindent 4 }}
+  {{- if .Values.alerting.prometheusRule.additionalLabels }}
+    {{- toYaml .Values.alerting.prometheusRule.additionalLabels | nindent 4 }}
+  {{- end }}
+spec:
+{{- if .Values.alerting.prometheusRule.rules }}
+  groups:
+  - name: {{ template "sequencer-relayer.name" . }}
+    rules: {{- toYaml .Values.alerting.prometheusRule.rules | nindent 4 }}
+{{- end }}
+{{- end }}
diff --git a/charts/sequencer-relayer/templates/service.yaml b/charts/sequencer-relayer/templates/service.yaml
@@ -4,6 +4,8 @@ apiVersion: v1
 metadata:
   name: sequencer-relayer-metrics
   namespace: {{ include "sequencer-relayer.namespace" . }}
+  labels:
+    {{- include "sequencer-relayer.labels" . | nindent 4 }}
 spec:
   selector:
     app: sequencer-relayer

diff --git a/charts/sequencer-relayer/templates/servicemonitor.yaml b/charts/sequencer-relayer/templates/servicemonitor.yaml
@@ -4,7 +4,7 @@ kind: ServiceMonitor
 metadata:
   name: sequencer-relayer-metrics
   labels:
-    app: sequencer-relayer
+    {{- include "sequencer-relayer.labels" . | nindent 4 }}
     {{- with .Values.serviceMonitor.additionalLabels }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
@@ -15,7 +15,7 @@ spec:
       - {{ include "sequencer-relayer.namespace" . }}
   selector:
     matchLabels:
-      app: sequencer-relayer
+      {{- include "sequencer-relayer.labels" . | nindent 6 }}
   endpoints:
     - port: metrics
       path: /

diff --git a/charts/sequencer-relayer/values.yaml b/charts/sequencer-relayer/values.yaml
@@ -55,6 +55,30 @@ serviceMonitor:
   additionalLabels:
     release: kube-prometheus-stack
 
+# Alerting Configuration
+alerting:
+  enabled: false
+  interval: ""
+  additionalLabels:
+    release: kube-prometheus-stack
+  annotations: {}
+  # scrapeTimeout: 10s
+  # path: /metrics
+  prometheusRule:
+    enabled: true
+    additionalLabels:
+      release: kube-prometheus-stack
+    namespace: monitoring
+    rules:
+    - alert: Sequencer_Relayer_Down
+      expr: up{container="sequencer-relayer"} == 0  # Insert your query Expression
+      for: 1m  # Rough number but should be enough to init warn
+      labels:
+        severity: warning
+      annotations:
+        summary: Sequencer Relayer is Down (instance {{ $labels.instance }})
+        description: "Sequencer relayer '{{ $labels.namespace }}' has disappeared from Prometheus target discovery.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
 storage:
   enabled: false
   local: true

diff --git a/charts/sequencer/Chart.lock b/charts/sequencer/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: sequencer-relayer
   repository: file://../sequencer-relayer
-  version: 0.12.1
-digest: sha256:470368e9d1cbb0456ba3a198c42f708591454cc6638c9a0d9918828d898627b0
-generated: "2024-09-10T11:55:38.765161-07:00"
+  version: 0.12.2
+digest: sha256:3ee234bbedc042030976130a12c0fe364ff171a40d341fec820c49720fef9378
+generated: "2024-09-30T11:40:37.590749-04:00"
diff --git a/charts/sequencer/Chart.yaml b/charts/sequencer/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.23.0
+version: 0.23.1
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
@@ -24,7 +24,7 @@ appVersion: "0.17.0"
 
 dependencies:
   - name: sequencer-relayer
-    version: "0.12.1"
+    version: "0.12.2"
     repository: "file://../sequencer-relayer"
     condition: sequencer-relayer.enabled
 

diff --git a/charts/sequencer/templates/prometheusrule.yaml b/charts/sequencer/templates/prometheusrule.yaml
@@ -2,7 +2,7 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: {{ template "sequencer.fullname" . }}
+  name: {{ include "sequencer.name" . }}
 {{- if .Values.alerting.prometheusRule.namespace }}
   namespace: {{ .Values.alerting.prometheusRule.namespace | quote }}
 {{- end }}
@@ -14,7 +14,7 @@ metadata:
 spec:
 {{- if .Values.alerting.prometheusRule.rules }}
   groups:
-  - name: {{ template "sequencer.fullname" . }}
+  - name: {{ template "sequencer.name" . }}
     rules: {{- toYaml .Values.alerting.prometheusRule.rules | nindent 4 }}
 {{- end }}
 {{- end }}