From ea1f2e55e8d3f64d6fcd6691c408436c11493305 Mon Sep 17 00:00:00 2001 From: Karan Sharma Date: Thu, 3 Nov 2022 09:34:49 +0530 Subject: [PATCH] fix: push notifications in background Sometimes when the alert payload is huge, pushing notification takes a lot of I/O time and Alertmanager might timeout the request. This is undesirable since Alertmanager will retry the request but cAlert will keep sending notifications. To handle this, it's better to push notifications in it's own goroutine and send an early response for the request. The downside is that no error will be sent to Alertmanager if pushing notifications failed, but that is a fair trade-off. Retries can be implemented at cAlert level (in a future version). --- cmd/handlers.go | 17 +- docs/examples/mock_payload.json | 582 +++++++++++++++++++++++++++++++- 2 files changed, 590 insertions(+), 9 deletions(-) diff --git a/cmd/handlers.go b/cmd/handlers.go index ae90127..f598740 100644 --- a/cmd/handlers.go +++ b/cmd/handlers.go @@ -102,14 +102,15 @@ func handleDispatchNotif(w http.ResponseWriter, r *http.Request) { app.lo.WithField("receiver", roomName).Info("dispatching new alert") // Dispatch a list of alerts via Notifier. - if err := app.notifier.Dispatch(payload.Alerts, roomName); err != nil { - app.lo.WithError(err).Error("error dispatching alerts") - app.metrics.Increment(`http_request_errors_total{handler="dispatch"}`) - sendErrorResponse(w, "Error dispatching alerts.", http.StatusInternalServerError, nil) - return - } - - app.metrics.Duration(`http_request_duration_seconds{handler="dispatch"}`, now) + // If there are a lot of alerts (>=10) to push, G-Chat API can be extremely slow to add messages + // to an existing thread. So it's better to enqueue it in background. + go func() { + if err := app.notifier.Dispatch(payload.Alerts, roomName); err != nil { + app.lo.WithError(err).Error("error dispatching alerts") + app.metrics.Increment(`http_request_errors_total{handler="dispatch"}`) + } + app.metrics.Duration(`http_request_duration_seconds{handler="dispatch"}`, now) + }() sendResponse(w, "dispatched") } diff --git a/docs/examples/mock_payload.json b/docs/examples/mock_payload.json index 65ab629..31cba10 100644 --- a/docs/examples/mock_payload.json +++ b/docs/examples/mock_payload.json @@ -1,7 +1,587 @@ { - "receiver": "calert", + "receiver": "dev_alerts", "status": "firing", "alerts": [ + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, { "status": "firing", "labels": {