latency.rules

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

groups:
- name: latency
  rules:

    - record: task:latency_ms_bucket:rate10s
      expr: rate(latency_ms_bucket[10s])

    - record: task:latency_count:rate10s
      expr: rate(latency_ms_count{}[10s])

    - record: job:latency_ms_bucket:rate10s
      expr: sum by (job, le) (task:latency_ms_bucket:rate10s)

    - record: job:latency_count:rate10s
      expr: sum by (job) (task:latency_count:rate10s)

    - record: task:backend_latency_ms_bucket:rate10s
      expr: rate(backend_latency_ms_bucket[10s])

    - record: job:backend_latency_ms_bucket:rate10s
      expr: sum by (job, le)(task:backend_latency_ms_bucket:rate10s)

      # Declare that our latency SLI is at 256 milliseconds
    - record: job:latency_sli:ratio_rate10s
      expr:   job:latency_ms_bucket:rate10s{le="256"}    / on (job)  job:latency_count:rate10s

      # the worst
    - record: job:latency_apdex:ratio_rate10s
      expr: (job:latency_ms_bucket:rate10s{le="256"}      + on (job)      job:latency_ms_bucket:rate10s{le="1024"} / 2)      / on (job)      job:latency_count:rate10s

      # and we want to provide a response time SLO of five nines
    - alert: LatencyTooHigh
      expr: job:latency_sli:ratio_rate10s{job="lb"} < 0.555555555
      labels:
        severity: page
      annotations:
        summary: "😱 Latency on {{ $labels.job }} is missing SLO"
        description: "Latency SLI is {{ $value }}, which is below nine fives SLO. http://localhost:9090{{ graphLink \"job:latency_ms_bucket:rate10s\" }}"