prometheus-alerts.yaml

"groups":
- "name": "celery"
  "rules":
  - "alert": "CeleryTaskHighFailRate"
    "annotations":
      "dashboard_url": "https://grafana.com/d/celery-tasks-by-task-32s3/celery-tasks-by-task?var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}&var-task={{ $labels.name }}"
      "description": "More than 5% tasks failed for the task {{ $labels.job }}/{{ $labels.queue_name }}/{{ $labels.name }} the past 10m."
      "summary": "Celery high task fail rate."
    "expr": |
      sum(
        increase(
          celery_task_failed_total{
            job=~"celery|celery-exporter",
            queue_name!~"None",
            name!~"None"
          }[10m]
        )
      )  by (job, namespace, queue_name, name)
      /
      (
        sum(
          increase(
            celery_task_failed_total{
              job=~"celery|celery-exporter",
              queue_name!~"None",
              name!~"None"
            }[10m]
          )
        )  by (job, namespace, queue_name, name)
        +
        sum(
          increase(
            celery_task_succeeded_total{
              job=~"celery|celery-exporter",
              queue_name!~"None",
              name!~"None"
            }[10m]
          )
        )  by (job, namespace, queue_name, name)
      )
      * 100 > 5
    "for": "1m"
    "labels":
      "severity": "warning"
  - "alert": "CeleryHighQueueLength"
    "annotations":
      "dashboard_url": "https://grafana.com/d/celery-tasks-overview-32s3/celery-tasks-overview?&var-job={{ $labels.job }}&var-queue_name={{ $labels.queue_name }}"
      "description": "More than 100 tasks in the queue {{ $labels.job }}/{{ $labels.queue_name }} the past 20m."
      "summary": "Celery high queue length."
    "expr": |
      sum(
        celery_queue_length{
          job=~"celery|celery-exporter",
          queue_name!~"None"
        }
      )  by (job, namespace, queue_name)
      > 100
    "for": "20m"
    "labels":
      "severity": "warning"
  - "alert": "CeleryWorkerDown"
    "annotations":
      "dashboard_url": "https://grafana.com/d/celery-tasks-overview-32s3/celery-tasks-overview?&var-job={{ $labels.job }}"
      "description": "The Celery worker {{ $labels.job }}/{{ $labels.hostname }} is offline."
      "summary": "A Celery worker is offline."
    "expr": |
      celery_worker_up{job=~"celery|celery-exporter"} == 0
    "for": "15m"
    "labels":
      "severity": "warning"