From 478dd7c4a3c94b085865ab6c3d7f225d4266133a Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 19 Jan 2024 05:44:55 -0500 Subject: [PATCH 1/4] fix: 5xx alarm --- .../monitoring/panels/lb/error_5xx.libsonnet | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/terraform/monitoring/panels/lb/error_5xx.libsonnet b/terraform/monitoring/panels/lb/error_5xx.libsonnet index 594c047..5b75a01 100644 --- a/terraform/monitoring/panels/lb/error_5xx.libsonnet +++ b/terraform/monitoring/panels/lb/error_5xx.libsonnet @@ -29,6 +29,38 @@ local _configuration = defaults.configuration.timeseries value = threshold, ) + .setAlert( + vars.environment, + grafana.alert.new( + namespace = vars.namespace, + name = "%(env)s - 5XX alert" % { env: grafana.utils.strings.capitalize(env) }, + message = '%(env)s - Notify - 5XX alert' % { env: grafana.utils.strings.capitalize(env) }, + notifications = vars.notifications, + noDataState = 'no_data', + period = '0m', + conditions = [ + grafana.alertCondition.new( + evaluatorParams = [ 5 ], + evaluatorType = 'gt', + operatorType = 'or', + queryRefId = 'ELB', + queryTimeStart = '5m', + queryTimeEnd = 'now', + reducerType = grafana.alert_reducers.Avg + ), + grafana.alertCondition.new( + evaluatorParams = [ threshold ], + evaluatorType = 'gt', + operatorType = 'or', + queryRefId = 'Target', + queryTimeStart = '5m', + queryTimeEnd = 'now', + reducerType = grafana.alert_reducers.Avg + ), + ], + ) + ) + .addTarget(targets.cloudwatch( alias = 'ELB', datasource = ds.cloudwatch, From 3d5063255bc8ffa1a3aabcf88f4a6ebb625038f9 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 19 Jan 2024 06:06:29 -0500 Subject: [PATCH 2/4] fix: var --- terraform/monitoring/panels/lb/error_5xx.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/monitoring/panels/lb/error_5xx.libsonnet b/terraform/monitoring/panels/lb/error_5xx.libsonnet index 5b75a01..caa7d37 100644 --- a/terraform/monitoring/panels/lb/error_5xx.libsonnet +++ b/terraform/monitoring/panels/lb/error_5xx.libsonnet @@ -33,8 +33,8 @@ local _configuration = defaults.configuration.timeseries vars.environment, grafana.alert.new( namespace = vars.namespace, - name = "%(env)s - 5XX alert" % { env: grafana.utils.strings.capitalize(env) }, - message = '%(env)s - Notify - 5XX alert' % { env: grafana.utils.strings.capitalize(env) }, + name = "%(env)s - 5XX alert" % { env: grafana.utils.strings.capitalize(vars.environment) }, + message = '%(env)s - Notify - 5XX alert' % { env: grafana.utils.strings.capitalize(vars.environment) }, notifications = vars.notifications, noDataState = 'no_data', period = '0m', From 40deb44955531ae062481976327f0adc1bf899a2 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 19 Jan 2024 07:05:31 -0500 Subject: [PATCH 3/4] fix: alert syntax --- terraform/monitoring/panels/lb/error_5xx.libsonnet | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/monitoring/panels/lb/error_5xx.libsonnet b/terraform/monitoring/panels/lb/error_5xx.libsonnet index caa7d37..8753ba1 100644 --- a/terraform/monitoring/panels/lb/error_5xx.libsonnet +++ b/terraform/monitoring/panels/lb/error_5xx.libsonnet @@ -30,7 +30,6 @@ local _configuration = defaults.configuration.timeseries ) .setAlert( - vars.environment, grafana.alert.new( namespace = vars.namespace, name = "%(env)s - 5XX alert" % { env: grafana.utils.strings.capitalize(vars.environment) }, From 6d7a419047ece0d18c130a9ed2e128c9eb83da09 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 19 Jan 2024 14:58:26 -0500 Subject: [PATCH 4/4] chore: reduce threshold --- terraform/monitoring/panels/lb/error_5xx.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/monitoring/panels/lb/error_5xx.libsonnet b/terraform/monitoring/panels/lb/error_5xx.libsonnet index 8753ba1..3c9ca3b 100644 --- a/terraform/monitoring/panels/lb/error_5xx.libsonnet +++ b/terraform/monitoring/panels/lb/error_5xx.libsonnet @@ -4,7 +4,7 @@ local defaults = import '../../grafonnet-lib/defaults.libsonnet'; local panels = grafana.panels; local targets = grafana.targets; -local threshold = 100; +local threshold = 5; local _configuration = defaults.configuration.timeseries .withSoftLimit(