-
Notifications
You must be signed in to change notification settings - Fork 107
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
basic alerting, refactored to use slack-go instead
- Loading branch information
1 parent
9ec86d4
commit 3994468
Showing
10 changed files
with
184 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| package alerting | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "log/slog" | ||
|
|
||
| "github.com/jackc/pgx/v5" | ||
| "github.com/jackc/pgx/v5/pgxpool" | ||
| ) | ||
|
|
||
| // alerting service, no cool name :( | ||
| type Alerter struct { | ||
| catalogPool *pgxpool.Pool | ||
| logger *slog.Logger | ||
| } | ||
|
|
||
| func registerSendersFromPool(catalogPool *pgxpool.Pool) ([]*slackAlertSender, error) { | ||
| rows, err := catalogPool.Query(context.Background(), | ||
| "SELECT service_type,service_config FROM peerdb_stats.alerting_config") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to read alerter config from catalog: %w", err) | ||
| } | ||
|
|
||
| var slackAlertSenders []*slackAlertSender | ||
| var serviceType, serviceConfig string | ||
| _, err = pgx.ForEachRow(rows, []any{&serviceType, &serviceConfig}, func() error { | ||
| switch serviceType { | ||
| case "slack": | ||
| var slackServiceConfig slackAlertConfig | ||
| err = json.Unmarshal([]byte(serviceConfig), &slackServiceConfig) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to unmarshal Slack service config: %w", err) | ||
| } | ||
|
|
||
| slackAlertSenders = append(slackAlertSenders, newSlackAlertSender(&slackServiceConfig)) | ||
| default: | ||
| return fmt.Errorf("unknown service type: %s", serviceType) | ||
| } | ||
| return nil | ||
| }) | ||
|
|
||
| return slackAlertSenders, nil | ||
| } | ||
|
|
||
| // doesn't take care of closing pool, needs to be done externally. | ||
| func NewAlerter(catalogPool *pgxpool.Pool) *Alerter { | ||
| return &Alerter{ | ||
| catalogPool: catalogPool, | ||
| logger: slog.Default(), | ||
| } | ||
| } | ||
|
|
||
| // Only raises an alert if another alert with the same key hasn't been raised | ||
| // in the past 15 minutes | ||
| func (a *Alerter) AlertIf(ctx context.Context, alertKey string, alertMessage string) { | ||
| if a.catalogPool != nil { | ||
| slackAlertSenders, err := registerSendersFromPool(a.catalogPool) | ||
| if err != nil { | ||
| a.logger.WarnContext(ctx, "failed to set Slack senders", slog.Any("error", err)) | ||
| return | ||
| } | ||
| if len(slackAlertSenders) == 0 { | ||
| a.logger.Warn("no Slack senders configured, returning") | ||
| return | ||
| } | ||
|
|
||
| row := a.catalogPool.QueryRow(context.Background(), | ||
| `SELECT created_timestamp FROM peerdb_stats.alerts_v1 WHERE alert_key=$1 | ||
| ORDER BY created_timestamp DESC LIMIT 1`, | ||
| alertKey) | ||
| var createdTimestamp time.Time | ||
| err = row.Scan(&createdTimestamp) | ||
| if err != nil && err != pgx.ErrNoRows { | ||
| a.logger.Warn("failed to send alert: %v", err) | ||
| return | ||
| } | ||
|
|
||
| if time.Since(createdTimestamp) >= 15*time.Minute { | ||
| for _, slackAlertSender := range slackAlertSenders { | ||
| err = slackAlertSender.sendAlert(context.Background(), | ||
| fmt.Sprintf(":rotating_light:Alert:rotating_light:: %s", alertKey), alertMessage) | ||
| if err != nil { | ||
| a.logger.WarnContext(ctx, "failed to send alert", slog.Any("error", err)) | ||
| return | ||
| } | ||
| _, _ = a.catalogPool.Exec(context.Background(), | ||
| "INSERT INTO peerdb_stats.alerts_v1(alert_key,alert_message) VALUES($1,$2)", | ||
| alertKey, alertMessage) | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package alerting | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/slack-go/slack" | ||
| ) | ||
|
|
||
| type slackAlertSender struct { | ||
| client *slack.Client | ||
| channelIDs []string | ||
| } | ||
|
|
||
| type slackAlertConfig struct { | ||
| AuthToken string `json:"auth_token"` | ||
| ChannelIDs []string `json:"channel_ids"` | ||
| } | ||
|
|
||
| func newSlackAlertSender(config *slackAlertConfig) *slackAlertSender { | ||
| return &slackAlertSender{ | ||
| client: slack.New(config.AuthToken), | ||
| channelIDs: config.ChannelIDs, | ||
| } | ||
| } | ||
|
|
||
| func (s *slackAlertSender) sendAlert(ctx context.Context, alertTitle string, alertMessage string) error { | ||
| for _, channelID := range s.channelIDs { | ||
| _, _, _, err := s.client.SendMessageContext(ctx, channelID, slack.MsgOptionBlocks( | ||
| slack.NewHeaderBlock(slack.NewTextBlockObject("plain_text", alertTitle, true, false)), | ||
| slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn", alertMessage, false, false), nil, nil), | ||
| )) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to send message to Slack channel %s: %w", channelID, err) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| CREATE TABLE IF NOT EXISTS peerdb_stats.alerting_config ( | ||
| id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, | ||
| service_type TEXT NOT NULL CHECK (service_type IN ('slack')), | ||
| service_config JSONB NOT NULL | ||
| ); | ||
|
|
||
| CREATE TABLE IF NOT EXISTS peerdb_stats.alerts_v1 ( | ||
| id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, | ||
| alert_key TEXT NOT NULL, | ||
| alert_level TEXT NOT NULL CHECK (alert_level IN ('critical')) DEFAULT 'critical', | ||
| alert_message TEXT NOT NULL, | ||
| created_timestamp TIMESTAMP DEFAULT now() | ||
| ); |