Skip to content

Commit

Permalink
feat: Initial Pagerduty set up (#13)
Browse files Browse the repository at this point in the history
  • Loading branch information
lholota authored Aug 17, 2023
1 parent 980c0ee commit 42b8aed
Show file tree
Hide file tree
Showing 13 changed files with 109 additions and 8 deletions.
1 change: 1 addition & 0 deletions .github/workflows/cd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ jobs:
aad_credentials: ${{ secrets.AZUREAD_CREDENTIALS_PROD }}
terraform_token: ${{ secrets.TERRAFORM_CLOUD_TOKEN }}
cloudflare_api_token: ${{ secrets.CLOUDFLARE_API_TOKEN }}
pagerduty_api_token: ${{ secrets.PAGERDUTY_API_TOKEN_PROD }}
sops_age_key: ${{ secrets.SOPS_AGE_PRIVATE_KEY }}
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ jobs:
aad_credentials: ${{ secrets.AZUREAD_CREDENTIALS_LAB }}
terraform_token: ${{ secrets.TERRAFORM_CLOUD_TOKEN }}
cloudflare_api_token: ${{ secrets.CLOUDFLARE_API_TOKEN }}
pagerduty_api_token: ${{ secrets.PAGERDUTY_API_TOKEN_LAB }}
sops_age_key: ${{ secrets.SOPS_AGE_PRIVATE_KEY }}

plan_prod:
Expand Down Expand Up @@ -51,6 +52,7 @@ jobs:
env:
SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_PRIVATE_KEY }}
TF_VAR_cloudflare_api_token: ${{ secrets.CLOUDFLARE_API_TOKEN }}
TF_VAR_pagerduty_api_token: ${{ secrets.PAGERDUTY_API_TOKEN_PROD }}

- name: Update Pull Request
uses: actions/[email protected]
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/partial-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ on:
required: true
cloudflare_api_token:
required: true
pagerduty_api_token:
required: true
sops_age_key:
required: true

Expand Down Expand Up @@ -66,4 +68,5 @@ jobs:
run: yarn run ${{ inputs.environment }}:apply -var 'aad_credentials=${{ secrets.aad_credentials }}'
env:
SOPS_AGE_KEY: ${{ secrets.sops_age_key }}
TF_VAR_cloudflare_api_token: ${{ secrets.cloudflare_api_token }}
TF_VAR_cloudflare_api_token: ${{ secrets.cloudflare_api_token }}
TF_VAR_pagerduty_api_token: ${{ secrets.pagerduty_api_token }}
6 changes: 3 additions & 3 deletions environments/prod.tfvars
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ argocd_redirect_urls = [
]

cloudflare_ssh_hosts = [
{ hostname = "pve1" },
{ hostname = "pve2" },
{ hostname = "pve3" }
{ hostname = "pve1" },
{ hostname = "pve2" },
{ hostname = "pve3" }
]

cloudflare_apps_subdomain_suffix = ""
Expand Down
6 changes: 4 additions & 2 deletions environments/secrets.lab.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ cloudflare_account_id: ENC[AES256_GCM,data:TB/EoksURZeLBWYPmCPdd4hQ8LX3v8OTSmFCD
cloudflare_zone_id: ENC[AES256_GCM,data:2OK4LP8pKYRN2qEpO+V7C9AfLRo6FdtxcQCFXVyT+9Y=,iv:2xC9IW4GlBkdwjwqG5IxtUWLN6ortAdMFEiJvBYO3ps=,tag:e8oM2cBdQnMlyb0Xa+Aphw==,type:str]
cloudflare_team_name: ENC[AES256_GCM,data:yV022fKKPZq0,iv:sSOIu7nC2DKACVSfxTWIkiPBEo5Pe7KCQH/ogQw2Twc=,tag:HnZDDCYLPGugkLRfDgbvag==,type:str]
cloudflare_tunnel_secret: ENC[AES256_GCM,data:D5COxVfOnk9azfSHqVHbdzAWS2xR8DLjZ5y4Qj8tMkS0FhnaPTVW1XAJHQfi0+qFcoYGRMF0F4gHLT8lDCvnKhgazaLV2Eh7KoDAsEELoSC3lv7seHkHmHXTlU/1Eemxew4BJcmQG9ncp8NAt8l78PyeYoSN2OBwE1bpIk9gPmA=,iv:1zeI5Ax78Ytr746x+79GnbLkDqW2kujEIxXGck5DnLo=,tag:r46TnrW7dJ3ZYEPSkxmiFA==,type:str]
pagerduty_user_email: ENC[AES256_GCM,data:61FRGGFuVcDgmG0RM1x7J8HRTt7qI+zATw==,iv:InUE49JZu9HzwheIjA5Ud9e2zMVOt/amJHqq3RbZT7I=,tag:DYEDEhvOEnUe5J+qBVKzNg==,type:str]
pagerduty_email_domain: ENC[AES256_GCM,data:yu59x8xSRlsbrJ2gWuDpdLc8vRc5jmbifraswTMS,iv:LBN9c2kJNmSvR4tFk5uyEyFyzAFk1+jubDwqk0AoU4U=,tag:nC19b8e5XlLVxofVYc4DHg==,type:str]
sops:
kms: []
gcp_kms: []
Expand All @@ -20,8 +22,8 @@ sops:
UEFOWE5BcEFXeEFEVG11UVNWY2dnOTQKE+nqVcOCOZ8f7c0t+a8ag+83S/kMBVzk
7hC4pLFU2Uy2rWmKtEIaLIBlSCQA32dg7iNMg2P7nXNk4MM2gxxGFQ==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2023-06-06T23:17:56Z"
mac: ENC[AES256_GCM,data:E8Ay2w2GrOEt//yn37w2fCcoGHc1oySDUIfasOLIxJcOVPt5Oh2FUh5gX5VE2hmDwd3TAlVEF8zqal6JsoFhRApFBDsBXzCNV4MSRTN52I7w/7rr7x1wRnzcwEdVYBXkS/k5rE43cdBhNtDFvrzNmFGnYlZphfDkeshsrvjJ610=,iv:bNcnUHYTj7jIXfhx8jBHIra8oIeUYaPtUq/57glfWVQ=,tag:tIlzhiFnb0EEahv6KI6wbQ==,type:str]
lastmodified: "2023-08-17T08:33:58Z"
mac: ENC[AES256_GCM,data:XTBq5S2lUQnSAHFVAqL697SLHymvRnYWwF4UWLswuxgohId9aGNLUTQzdo84RaO2rVRf4zEAHKMBnDy0CfKDEc5g6q0hSLYHjyPk/jckr3OnZyei4SqeGOZk2Tl+5eJl7i8KlEANbKx9SoepzqowgemZ+AClw/KRfGNwIhefAoQ=,iv:p4V8m/hNchG8hjJvvIiYweolNORv7KtptTfMsc9++1c=,tag:PlkS0gTvu2KGY+99hDEvmA==,type:str]
pgp:
- created_at: "2023-01-22T16:20:35Z"
enc: "-----BEGIN PGP MESSAGE-----\r\n\r\nhQIMA7Pg+ndCcR5CAQ/8DlGl2wYan+AVF/GpUeVs82PL0jmYzZt1/64fqeDZ0IbD\r\n4yzbhhq83dhUtacONKmO6sXFbC+e+6WSa7ZwWZ/tVWijYC3ErWTpHtiInnQmRif2\r\n5hISYzPvIAfTv8SRu18p0iHX0DmUjvX0ZlpmJdLR8AmvVZHU5+6Xv171G2Za8DbW\r\n4JHXSPO9OpjUsICJzNis64d22TecMiCkkiiHEs8PGj+jYXwNAy2qR3EQDjvYLDc8\r\nacM5N36hZRb8w9ybMQiOHDOAV4LGlKEbJPqkisjtkp+hWsahMEuOZoTkbekt8o6C\r\nvpuPQMrX5f2rNtGt63YhduvlTLNE4yeHOkP0czGbpokjjxu0ZUW7tR7TH+tMJ+oR\r\nXueY4/EjyMoBIvBIkiesH9tbAJD59smwH2Kj7q2RUlTo8O1ldJ8FOp3HpFcCzSpB\r\nYyKLUg7Eu8J+oMLbAS4Wr5gy3Jj0gvm6BhE+0d9CE2dEf7h2C4MI/uLwkFCoH4UX\r\noHrZ0/T8XND1GN93NmcgGvOVzZLOWrYub/zfinq8XmcWLTZmGqKtNxPFtY6HHY4A\r\n568MZZSS5NXcZ/WeTFg3ToHI+WQP5EQuR7fv8cmUPorbFOcuEZyyXftdZOz1EJxi\r\nqyPQYfPuTXlAXdmsdd/H13Mf1KKqb1OT6uuJ7v+6lnLjOVhkjT1523IropRRuN7S\r\nXgH84OPsbZ5YtbgTf+7KC0nwLtLAZcXmueqyVU6wpPdJ/MUZWrOw8aBCM2tfcsQT\r\nfo5lguDgsEV3we/V9jnTKOGWiL8AFHlexQo4f+LlpA2G6fwQi62oUcLyi0kx1no=\r\n=voPn\r\n-----END PGP MESSAGE-----\r\n"
Expand Down
6 changes: 4 additions & 2 deletions environments/secrets.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ cloudflare_account_id: ENC[AES256_GCM,data:gJliV7E1eMrfm3IaS73LF1oW608l/VSo1eYbS
cloudflare_zone_id: ENC[AES256_GCM,data:3joVfjPeTV9YgLzvcjMNhWh98WWR2hcBjQc2o/q112M=,iv:wzxQsYkzskPHayejj8EnLpCzQ33KEIl6E9wmRVtEUqI=,tag:xhQL2wg62Z2UqR+SZhpD3g==,type:str]
cloudflare_team_name: ENC[AES256_GCM,data:+NYL4DLI5oE+,iv:aH1YdBSutMH1NXxwdiDhydIoRmYh14tK/E0rsuy74sE=,tag:Ccoz/un96nZbKpQ55ydwFQ==,type:str]
cloudflare_tunnel_secret: ENC[AES256_GCM,data:2H9g3QSni8D73xoz+s/spaqfVBI66ZVZuNPexO/AK7j0G406Qlia00udGvgDdEOyH0NIFuEKV2zy7l0TUU4soGPOY8dYVfLWmwhBr09M/m/K655j3C6KQIa4FNNkurRHWd7Xkp+gPXXyq1Cw/MD1ZAH448cH0DDxetdUhcGGha0=,iv:AEbZ72eMXgVCUvJA46dIZ10i6mVAyXwHoFRIMJr3vbc=,tag:zFCJEdEI9PuYjJ9hNrFWiw==,type:str]
pagerduty_user_email: ENC[AES256_GCM,data:98WnTMRWtNLTiE/495MxW2y/IHFZ,iv:uwSyaKjhy7H22Z0hUv+zLS6gJ5qcp9SNa3O7bNYfnk8=,tag:wHWN8MOr+BEmDLbvsEjkPQ==,type:str]
pagerduty_email_domain: ENC[AES256_GCM,data:9POQOwKlcNhIQK2IE2kC3LIHbf7VZKK6RsA=,iv:MyJ0H313H1dG0F3Stub23Uet+QicgyV+U7fZMbIAfh8=,tag:MLwoyHTySH6vzitOB9rxow==,type:str]
sops:
kms: []
gcp_kms: []
Expand All @@ -20,8 +22,8 @@ sops:
R3RJOVBOK0tCcS9jZU9LRFZodTh1ZVkKIcaLdywrU0YscjNN9+noNjyEeZ7Ei5NH
MULyehZavUvVRh1GLNiFL7zM4Q7dCgNXJrrCKGwzjfEuLzKtxDtWog==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2023-06-06T23:15:29Z"
mac: ENC[AES256_GCM,data:TtQ3uZte2mbN/e9xZ+dQYk5HaH3CnqFL0VC9mvwwe2aHyLHfiOgCt2WPNJ4773dvyBscKKvWljxqrA96XRbsQq7677YvoWV5sFtgi0lw+Dayln0U9q6CtCu6apXe2ZD+vMjU3fAM6Ep/r05Y/yxKw3ZapLzgswY6LDypELv2u7k=,iv:H8G+vHPzV4QWnPhBOfyBEYUdbGyrlK7E76VrnNvE+w0=,tag:z4aQgNF5VbZwZv6ZyI1oAA==,type:str]
lastmodified: "2023-08-17T08:34:26Z"
mac: ENC[AES256_GCM,data:4X3viM1ByU5+e1lkrPcD0uaeXxcEEC2uzh+38bbu46xpt+aw8veeUXHb2sjZxv6lPlbM13DezyYHWOsEQ6cCcMzlhRLagi4x5TUiCtpR+Radg9qJ0kmLNq5OK/vCqwEp7x9T57+ZfkK7QcvyPMMv2t2GxDXJVVZWpwoc+873J7s=,iv:j5DyRVeYN1S1XNG5WXMkHJLizpwqPK2/D9hD9FBitmY=,tag:fd1CKwqun9LPJuUz5pCsmw==,type:str]
pgp:
- created_at: "2023-02-13T21:59:54Z"
enc: "-----BEGIN PGP MESSAGE-----\r\n\r\nhQIMA7Pg+ndCcR5CAQ/+JtyP1aIWLv/o+UIQF74FR1B/qcRzKfmHP+MQhnDQgZBc\r\nXkw3dG5muKxC6PJCRO0d1JUCChHx1HsVi/WLn2EPZABBgL+PBva2xU1m/7khVy3z\r\n7WW+CwRc7qaSzZ79+/AI0B+0PXEQH3IWwThHgBblZokujR97QOkrANPkTz/OTwPO\r\n2KP1ej/aEvDNxybm8PLQjW96fId0UNh8BEoRvs5eVJNKYaICJ907QHhiMyBqOyGc\r\nUY9+dy3wv9wRREBh1mXWu40nhtOdHG0qA2nUNy6BfRa8MEYsKTaWHMJ9udbJoq6J\r\nUD4dV91F/tY422a/r17s0S2flp1hDQxPHdOvQQQ1dgc5KNXM+HhWFWQM4iMiTvOq\r\nTfx0t9e+j/VcC126RmUBy29P0ackKs5+DdV4Px7je2Mf8jABERleGAOxgu+Q5i6m\r\nnObcNYL+ynRGbi+rJZ3gz+8W5atXOGBdnJjahG+pGlRQw7NIhyXCZOpnlJJElae/\r\nxJLl5tbyV58qgltQICSQSoWahupqNnYgxPDGpmlxvIeA7dKwS5ghG1ex8reQ1F2f\r\nryHg2VTiUty8JLdXYBbqkp781fFaGB6rwDGqvSNiKZMuu2yOu1D8LVTlquZnQHfB\r\nu0g8MtfQxSHVKgBM4XaVh7rCt69wLK7FjswuTpoM9wm9+k4xiG/roHM7utmfoCLS\r\nXgGCUEoz9cUDIjm4/kj7PzahKiMEKFWhLJGMnawzYYUgqEwMG9USUTfwKemVkRQz\r\nlYsq8ymcCPyw3VZUOG7vJ6Z1njSgYqFu4OdqaDqd3MbBferqFoJWl8koLab3ks8=\r\n=kNl7\r\n-----END PGP MESSAGE-----\r\n"
Expand Down
27 changes: 27 additions & 0 deletions terraform/pagerduty-app-idrac.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
resource "pagerduty_service" "idrac" {
name = "Dell iDRAC${var.display_name_environment_suffix}"
auto_resolve_timeout = 14400
acknowledgement_timeout = 600
escalation_policy = pagerduty_escalation_policy.default.id
alert_creation = "create_alerts_and_incidents"

alert_grouping_parameters {
type = "intelligent"
}
}

resource "pagerduty_service_integration" "idrac_pve1" {
name = "iDRAC PVE1 E-mail Alerts"
type = "generic_email_inbound_integration"
integration_email = "idrac-pve1@${sensitive(data.sops_file.secrets.data["pagerduty_email_domain"])}"

service = pagerduty_service.idrac.id
}

resource "pagerduty_service_integration" "idrac_pve2" {
name = "iDRAC PVE2 E-mail Alerts"
type = "generic_email_inbound_integration"
integration_email = "idrac-pve2@${sensitive(data.sops_file.secrets.data["pagerduty_email_domain"])}"

service = pagerduty_service.idrac.id
}
19 changes: 19 additions & 0 deletions terraform/pagerduty-app-proxmox.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
resource "pagerduty_service" "proxmox" {
name = "Proxmox VE${var.display_name_environment_suffix}"
auto_resolve_timeout = 14400
acknowledgement_timeout = 600
escalation_policy = pagerduty_escalation_policy.default.id
alert_creation = "create_alerts_and_incidents"

alert_grouping_parameters {
type = "intelligent"
}
}

resource "pagerduty_service_integration" "proxmox" {
name = "Proxmox E-mail Alerts"
type = "generic_email_inbound_integration"
integration_email = "proxmox@${sensitive(data.sops_file.secrets.data["pagerduty_email_domain"])}"

service = pagerduty_service.proxmox.id
}
17 changes: 17 additions & 0 deletions terraform/pagerduty-escalation-policy.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
resource "pagerduty_escalation_policy" "default" {
name = "Default"
description = "Default escalation policy for all services"

teams = [
pagerduty_team.default.id
]

rule {
escalation_delay_in_minutes = 3

target {
type = "user_reference"
id = data.pagerduty_user.me.id
}
}
}
11 changes: 11 additions & 0 deletions terraform/pagerduty-team.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
resource "pagerduty_team" "default" {
name = "Homecentr Support"
description = "Homecentr Support"
}

resource "pagerduty_team_membership" "me" {
user_id = data.pagerduty_user.me.id
team_id = pagerduty_team.default.id

role = "manager"
}
3 changes: 3 additions & 0 deletions terraform/pagerduty-user.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
data "pagerduty_user" "me" {
email = sensitive(data.sops_file.secrets.data["pagerduty_user_email"])
}
9 changes: 9 additions & 0 deletions terraform/terraform.tf
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ terraform {
version = "4.4.0"
}

pagerduty = {
source = "pagerduty/pagerduty"
version = "2.15.3"
}

sops = {
source = "carlpett/sops"
version = "~> 0.5"
Expand All @@ -38,4 +43,8 @@ provider "azuread" {

provider "cloudflare" {
api_token = var.cloudflare_api_token
}

provider "pagerduty" {
token = var.pagerduty_api_token
}
5 changes: 5 additions & 0 deletions terraform/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ variable "cloudflare_api_token" {
sensitive = true
}

variable "pagerduty_api_token" {
type = string
sensitive = true
}

variable "cloudflare_apps" {
type = list(object({
subdomain = string
Expand Down

0 comments on commit 42b8aed

Please sign in to comment.