traefik-prometheusrule.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    app: kube-prometheus-stack
    release: prometheus
  name: traefik 
spec:
  groups:
  - name: traefik
    rules:
      - alert: test.cablespaghetti.dev-IngressHigh5xxRate
        annotations:
          message: The percentage of 5xx errors for test.cablespaghetti.dev over the last 5 minutes is over 1%.
        expr: (sum(rate(traefik_backend_request_duration_seconds_count{backend="test.cablespaghetti.dev/",code=~"5[0-9]{2}"}[1m]))/sum(rate(traefik_backend_request_duration_seconds_count{backend="test.cablespaghetti.dev/"}[1m])))*100 > 1
        for: 5m
        labels:
          severity: warning
      - alert: test.cablespaghetti.dev-IngressHigh5xxRate
        annotations:
          message: The percentage of 5xx errors for test.cablespaghetti.dev over the last 1 minute is over 5%.
        expr: (sum(rate(traefik_backend_request_duration_seconds_count{backend="test.cablespaghetti.dev/",code=~"5[0-9]{2}"}[1m]))/sum(rate(traefik_backend_request_duration_seconds_count{backend="test.cablespaghetti.dev/"}[1m])))*100 > 5
        for: 1m
        labels:
          severity: critical

      - alert: test.cablespaghetti.dev-IngressHighLatency
        annotations:
          message: The average latency of test.cablespaghetti.dev over the last 5 minutes is over 1s.
        expr: sum(rate(traefik_backend_request_duration_seconds_sum{backend="test.cablespaghetti.dev/"}[1m]))/sum(rate(traefik_backend_request_duration_seconds_count{backend="test.cablespaghetti.dev/"}[1m])) > 1
        for: 5m
        labels:
          severity: warning
      - alert: test.cablespaghetti.dev-IngressHighLatency
        annotations:
          message: The average latency of test.cablespaghetti.dev over the last 5 minutes is over 5s.
        expr: sum(rate(traefik_backend_request_duration_seconds_sum{backend="test.cablespaghetti.dev/"}[1m]))/sum(rate(traefik_backend_request_duration_seconds_count{backend="test.cablespaghetti.dev/"}[1m])) > 5
        for: 5m
        labels:
          severity: critical