From f7378d6170697f9796dc9de41d38bbd80485b191 Mon Sep 17 00:00:00 2001 From: santhosh-tg Date: Fri, 7 Jun 2024 11:21:53 +0530 Subject: [PATCH] Fix monitoring alerts --- .../monitoring/alertrules/templates/custom_promrules_k8s.yml | 2 +- .../monitoring/alertrules/templates/promrulesNode.yml | 4 ++-- .../templates/prometheus/rules-1.14/general.rules.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/custom_promrules_k8s.yml b/kubernetes/helm_charts/monitoring/alertrules/templates/custom_promrules_k8s.yml index 9ac3fabd67..f6eaf4fd01 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/templates/custom_promrules_k8s.yml +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/custom_promrules_k8s.yml @@ -15,7 +15,7 @@ spec: - alert: TargetDown annotations: message: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' - expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10 + expr: 100 * (count(up == 0) BY (namespace, service) / count(up) BY (namespace, service)) > 10 for: 10m labels: severity: critical diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yml b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yml index d36e7d6823..309502a509 100644 --- a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yml +++ b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yml @@ -85,12 +85,12 @@ spec: message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}} summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}} - alert: node_exporter_down_critical - expr: up == 0 + expr: up{job="vm-node-exporter"} == 0 for: 1m labels: severity: critical annotations: - message: {{`The node exporter '{{ $labels.job }}' is down.`}} + message: {{`'The node exporter '{{ $labels.job }}' is down.'`}} summary: {{`'NODE EXPORTER SERVICE critical: NODE ''{{ $labels.host }}'''`}} - alert: node_running_out_of_disk_space_warning expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Warning }} and ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) < {{ .Values.node_disk_usage_percentage_threshold_Critical }} ) diff --git a/kubernetes/helm_charts/monitoring/prometheus-operator/templates/prometheus/rules-1.14/general.rules.yaml b/kubernetes/helm_charts/monitoring/prometheus-operator/templates/prometheus/rules-1.14/general.rules.yaml index 8139fcaed9..ce8dbd1795 100644 --- a/kubernetes/helm_charts/monitoring/prometheus-operator/templates/prometheus/rules-1.14/general.rules.yaml +++ b/kubernetes/helm_charts/monitoring/prometheus-operator/templates/prometheus/rules-1.14/general.rules.yaml @@ -36,7 +36,7 @@ spec: description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown summary: One or more targets are unreachable. - expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + expr: 100 * (count(up == 0) BY (cluster,namespace, service) / count(up) BY (cluster,namespace, service)) > 10 for: 10m {{- with .Values.defaultRules.keepFiringFor }} keep_firing_for: "{{ . }}" @@ -122,4 +122,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }}