Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Add job to alert descriptions #273

Merged
merged 1 commit into from
Oct 21, 2023
Merged

fix: Add job to alert descriptions #273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions celery-mixin/alerts/alerts.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
||| % $._config,
annotations: {
summary: 'Celery high task fail rate.',
description: 'More than %(celeryTaskFailedThreshold)s%% tasks failed for the task {{ $labels.queue_name }}/{{ $labels.name }} the past %(celeryTaskFailedInterval)s.' % $._config,
description: 'More than %(celeryTaskFailedThreshold)s%% tasks failed for the task {{ $labels.job }}/{{ $labels.queue_name }}/{{ $labels.name }} the past %(celeryTaskFailedInterval)s.' % $._config,
dashboard_url: $._config.celeryTasksByTaskUrl + '?var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}&var-task={{ $labels.name }}',
},
'for': '1m',
Expand All @@ -67,7 +67,7 @@
},
annotations: {
summary: 'Celery high queue length.',
description: 'More than %(celeryHighQueueLengthThreshold)s tasks in the queue {{ $labels.queue_name }} the past %(celeryHighQueueLengthInterval)s.' % $._config,
description: 'More than %(celeryHighQueueLengthThreshold)s tasks in the queue {{{ $labels.job }}/{ $labels.queue_name }} the past %(celeryHighQueueLengthInterval)s.' % $._config,
dashboard_url: $._config.celeryTasksOverviewUrl + '?&var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}',
},
},
Expand All @@ -82,7 +82,8 @@
},
annotations: {
summary: 'A Celery worker is offline.',
description: 'The Celery worker {{ $labels.hostname }} is offline.',
description: 'The Celery worker {{ $labels.job }}/{{ $labels.hostname }} is offline.',
dashboard_url: $._config.celeryTasksOverviewUrl + '?&var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}',
},
},
]),
Expand Down
7 changes: 4 additions & 3 deletions celery-mixin/prometheus-alerts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
- "alert": "CeleryTaskHighFailRate"
"annotations":
"dashboard_url": "https://grafana.com/d/celery-tasks-by-task-32s3/celery-tasks-by-task?var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}&var-task={{ $labels.name }}"
"description": "More than 5% tasks failed for the task {{ $labels.queue_name }}/{{ $labels.name }} the past 10m."
"description": "More than 5% tasks failed for the task {{ $labels.job }}/{{ $labels.queue_name }}/{{ $labels.name }} the past 10m."
"summary": "Celery high task fail rate."
"expr": |
sum(
Expand Down Expand Up @@ -45,7 +45,7 @@
- "alert": "CeleryHighQueueLength"
"annotations":
"dashboard_url": "https://grafana.com/d/celery-tasks-overview-32s3/celery-tasks-overview?&var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}"
"description": "More than 100 tasks in the queue {{ $labels.queue_name }} the past 20m."
"description": "More than 100 tasks in the queue {{{ $labels.job }}/{ $labels.queue_name }} the past 20m."
"summary": "Celery high queue length."
"expr": |
sum(
Expand All @@ -60,7 +60,8 @@
"severity": "warning"
- "alert": "CeleryWorkerDown"
"annotations":
"description": "The Celery worker {{ $labels.hostname }} is offline."
"dashboard_url": "https://grafana.com/d/celery-tasks-overview-32s3/celery-tasks-overview?&var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}"
"description": "The Celery worker {{ $labels.job }}/{{ $labels.hostname }} is offline."
"summary": "A Celery worker is offline."
"expr": |
celery_worker_up{job=~"celery|celery-exporter"} == 0
Expand Down