feat: [PAYMCLOUD-175] Add infra_opsgenie to action groups in alerts o…

…n next-core (#2580) Add infra_opsgenie to action groups in alerts This change incorporates the infra_opsgenie action group into the existing alert configurations across multiple Terraform files. The infra_opsgenie action group is added under various alert actions, and it is specifically enabled conditionally for production environments in several monitoring configurations. This ensures critical alerts are also sent to the infra_opsgenie group, enhancing our alerting capabilities. Signed-off-by: Fabio Felici <[email protected]>
pagopa · Nov 22, 2024 · 31dcabb · 31dcabb
1 parent 7bb7c24
commit 31dcabb
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 52 deletions.
diff --git a/src/next-core/03_apim.tf b/src/next-core/03_apim.tf
@@ -117,16 +117,27 @@ module "apim" {
 
   alerts_enabled = var.apim_v2_alerts_enabled
 
-  action = [
-    {
-      action_group_id    = azurerm_monitor_action_group.slack.id
-      webhook_properties = null
-    },
-    {
-      action_group_id    = azurerm_monitor_action_group.email.id
-      webhook_properties = null
-    }
-  ]
+  # takes a list and replaces any elements that are lists with a
+  # flattened sequence of the list contents.
+  # In this case, we enable OpsGenie only on prod env
+  action = flatten([
+    [
+      {
+        action_group_id    = azurerm_monitor_action_group.slack.id
+        webhook_properties = null
+      },
+      {
+        action_group_id    = azurerm_monitor_action_group.email.id
+        webhook_properties = null
+      }
+    ],
+    (var.env == "prod" ? [
+      {
+        action_group_id    = azurerm_monitor_action_group.infra_opsgenie.0.id
+        webhook_properties = null
+      }
+    ] : [])
+  ])
 
   # metrics docs
   # https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftapimanagementservice

diff --git a/src/next-core/04_appgw.tf b/src/next-core/04_appgw.tf
@@ -568,16 +568,27 @@ module "app_gw" {
 
   alerts_enabled = var.app_gateway_alerts_enabled
 
-  action = [
-    {
-      action_group_id    = azurerm_monitor_action_group.slack.id
-      webhook_properties = null
-    },
-    {
-      action_group_id    = azurerm_monitor_action_group.email.id
-      webhook_properties = null
-    }
-  ]
+  # takes a list and replaces any elements that are lists with a
+  # flattened sequence of the list contents.
+  # In this case, we enable OpsGenie only on prod env
+  action = flatten([
+    [
+      {
+        action_group_id    = azurerm_monitor_action_group.slack.id
+        webhook_properties = null
+      },
+      {
+        action_group_id    = azurerm_monitor_action_group.email.id
+        webhook_properties = null
+      }
+    ],
+    (var.env == "prod" ? [
+      {
+        action_group_id    = azurerm_monitor_action_group.infra_opsgenie.0.id
+        webhook_properties = null
+      }
+    ] : [])
+  ])
 
   # metrics docs
   # https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftnetworkapplicationgateways

diff --git a/src/next-core/04_appgw_integration.tf b/src/next-core/04_appgw_integration.tf
@@ -234,16 +234,27 @@ module "app_gw_integration" {
 
   alerts_enabled = var.integration_app_gateway_alerts_enabled
 
-  action = [
-    {
-      action_group_id    = azurerm_monitor_action_group.slack.id
-      webhook_properties = null
-    },
-    {
-      action_group_id    = azurerm_monitor_action_group.email.id
-      webhook_properties = null
-    }
-  ]
+  # takes a list and replaces any elements that are lists with a
+  # flattened sequence of the list contents.
+  # In this case, we enable OpsGenie only on prod env
+  action = flatten([
+    [
+      {
+        action_group_id    = azurerm_monitor_action_group.slack.id
+        webhook_properties = null
+      },
+      {
+        action_group_id    = azurerm_monitor_action_group.email.id
+        webhook_properties = null
+      }
+    ],
+    (var.env == "prod" ? [
+      {
+        action_group_id    = azurerm_monitor_action_group.infra_opsgenie.0.id
+        webhook_properties = null
+      }
+    ] : [])
+  ])
 
   # metrics docs
   # https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftnetworkapplicationgateways

diff --git a/src/next-core/05_messages.tf b/src/next-core/05_messages.tf
@@ -73,16 +73,28 @@ module "event_hub03" {
 
   alerts_enabled = var.ehns_alerts_enabled
   metric_alerts  = var.ehns_metric_alerts
-  action = [
-    {
-      action_group_id    = azurerm_monitor_action_group.slack.id
-      webhook_properties = null
-    },
-    {
-      action_group_id    = azurerm_monitor_action_group.email.id
-      webhook_properties = null
-    }
-  ]
+
+  # takes a list and replaces any elements that are lists with a
+  # flattened sequence of the list contents.
+  # In this case, we enable OpsGenie only on prod env
+  action = flatten([
+    [
+      {
+        action_group_id    = azurerm_monitor_action_group.slack.id
+        webhook_properties = null
+      },
+      {
+        action_group_id    = azurerm_monitor_action_group.email.id
+        webhook_properties = null
+      }
+    ],
+    (var.env == "prod" ? [
+      {
+        action_group_id    = azurerm_monitor_action_group.infra_opsgenie.0.id
+        webhook_properties = null
+      }
+    ] : [])
+  ])
 
   tags = var.tags
 }
@@ -110,16 +122,27 @@ module "event_hub04" {
 
   eventhubs = var.eventhubs_04
 
-  action = [
-    {
-      action_group_id    = azurerm_monitor_action_group.slack.id
-      webhook_properties = null
-    },
-    {
-      action_group_id    = azurerm_monitor_action_group.email.id
-      webhook_properties = null
-    }
-  ]
+  # takes a list and replaces any elements that are lists with a
+  # flattened sequence of the list contents.
+  # In this case, we enable OpsGenie only on prod env
+  action = flatten([
+    [
+      {
+        action_group_id    = azurerm_monitor_action_group.slack.id
+        webhook_properties = null
+      },
+      {
+        action_group_id    = azurerm_monitor_action_group.email.id
+        webhook_properties = null
+      }
+    ],
+    (var.env == "prod" ? [
+      {
+        action_group_id    = azurerm_monitor_action_group.infra_opsgenie.0.id
+        webhook_properties = null
+      }
+    ] : [])
+  ])
 
   tags = var.tags
 }

diff --git a/src/next-core/06_node_forwarder.tf b/src/next-core/06_node_forwarder.tf
@@ -364,6 +364,10 @@ resource "azurerm_monitor_metric_alert" "app_service_over_cpu_usage" {
   action {
     action_group_id = azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id
   }
+  action {
+    action_group_id = azurerm_monitor_action_group.infra_opsgenie.0.id
+  }
+
 
   tags = var.tags
 }
@@ -401,6 +405,9 @@ resource "azurerm_monitor_metric_alert" "app_service_over_mem_usage" {
   action {
     action_group_id = azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id
   }
+  action {
+    action_group_id = azurerm_monitor_action_group.infra_opsgenie.0.id
+  }
 
   tags = var.tags
 }
@@ -461,7 +468,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "opex_pagopa-node-forward
   location            = var.location
 
   action {
-    action_group           = [azurerm_monitor_action_group.email.id, azurerm_monitor_action_group.slack.id, azurerm_monitor_action_group.mo_email.id, azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id]
+    action_group           = [azurerm_monitor_action_group.email.id, azurerm_monitor_action_group.slack.id, azurerm_monitor_action_group.mo_email.id, azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id, azurerm_monitor_action_group.infra_opsgenie.0.id]
     email_subject          = "Email Header"
     custom_webhook_payload = "{}"
   }
@@ -495,7 +502,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "opex_pagopa-node-forward
   location            = var.location
 
   action {
-    action_group           = [azurerm_monitor_action_group.email.id, azurerm_monitor_action_group.slack.id, azurerm_monitor_action_group.mo_email.id, azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id]
+    action_group           = [azurerm_monitor_action_group.email.id, azurerm_monitor_action_group.slack.id, azurerm_monitor_action_group.mo_email.id, azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id, azurerm_monitor_action_group.infra_opsgenie.0.id]
     email_subject          = "Email Header"
     custom_webhook_payload = "{}"
   }

diff --git a/src/next-core/08_natgateway.tf b/src/next-core/08_natgateway.tf
@@ -81,6 +81,9 @@ resource "azurerm_monitor_metric_alert" "snat_connection_over_10K" {
   action {
     action_group_id = azurerm_monitor_action_group.new_conn_srv_opsgenie[0].id
   }
+  action {
+    action_group_id = azurerm_monitor_action_group.infra_opsgenie.0.id
+  }
 
   tags = var.tags
 }