diff --git a/examples/kubernetes/micro-service/alert.js b/examples/kubernetes/micro-service/alert.js new file mode 100644 index 00000000..e298d59d --- /dev/null +++ b/examples/kubernetes/micro-service/alert.js @@ -0,0 +1,60 @@ +import * as prometheus from './prometheus'; + +const r = '2m'; +const selector = service => `job=${service.name}`; +const ErrorRate = selector => `rate(http_request_total{${selector},code=~"5.."}[${r}]) + / rate(http_request_duration_seconds_count{${selector}}[${r}])`; + +function RPSHTTPHighErrorRate(service) { + return { + alert: 'HighErrorRate', + expr: `${ErrorRate(selector(service))} * 100 > 10`, + for: '5m', + labels: { + severity: 'critical', + }, + annotations: { + service: service.name, + description: `More than 10% of requests to the ${service.name} service are failing with 5xx errors`, + details: '{{$value | printf "%.1f"}}% errors for more than 5m', + }, + }; +} + +// In real life we probably want something different. I'd like to enable dynamic +// imports so we can import dashboards js definitions from string descriptions: +// https://developers.google.com/web/updates/2017/11/dynamic-import +const alerts = { + 'service.RPS.HTTP.HighErrorRate': RPSHTTPHighErrorRate, +}; + +function rules(service) { + if (!service.alerts) { + return []; + } + + return service.alerts.map(a => alerts[a](service)); +} + +function PrometheusRule(service) { + return new prometheus.PrometheusRule(service.name, { + metadata: { + labels: { + app: service.name, + maintainer: service.maintainer, + prometheus: 'global', + role: 'alert-rules', + }, + }, + spec: { + groups: [{ + name: `${service.name}-alerts.rules`, + rules: rules(service), + }], + }, + }); +} + +export { + PrometheusRule, +}; diff --git a/examples/kubernetes/micro-service/billing.yaml b/examples/kubernetes/micro-service/billing.yaml index 9d12fd74..9da9cfc4 100644 --- a/examples/kubernetes/micro-service/billing.yaml +++ b/examples/kubernetes/micro-service/billing.yaml @@ -9,3 +9,5 @@ service: path: /api/billing dashboards: - service.RPS.HTTP + alerts: + - service.RPS.HTTP.HighErrorRate diff --git a/examples/kubernetes/micro-service/micro-service.js b/examples/kubernetes/micro-service/micro-service.js index 977544a3..80d274e3 100644 --- a/examples/kubernetes/micro-service/micro-service.js +++ b/examples/kubernetes/micro-service/micro-service.js @@ -4,6 +4,7 @@ import { Namespace, Deployment, Service, Ingress, ConfigMap, } from './kubernetes'; import { Dashboard } from './dashboard'; +import { PrometheusRule } from './alert'; const service = param.Object('service'); const ns = service.namespace; @@ -18,4 +19,5 @@ export default [ { file: `${ns}/${service.name}-svc.yaml`, value: Service(service) }, { file: `${ns}/${service.name}-ingress.yaml`, value: Ingress(service) }, { file: `${ns}/${service.name}-dashboards-cm.yaml`, value: ConfigMap(service, `${service.name}-dashboards`, dashboards) }, + { file: `${ns}/${service.name}-prometheus-rule.yaml`, value: PrometheusRule(service) }, ]; diff --git a/examples/kubernetes/micro-service/prometheus.js b/examples/kubernetes/micro-service/prometheus.js new file mode 100644 index 00000000..64b5255a --- /dev/null +++ b/examples/kubernetes/micro-service/prometheus.js @@ -0,0 +1,9 @@ +export class PrometheusRule { + constructor(name, desc) { + this.apiVersion = 'monitoring.coreos.com/v1'; + this.kind = 'PrometheusRule'; + this.metadata = Object.assign({}, (desc && desc.metadata) || {}, { name }); + this.spec = (desc && desc.spec) || undefined; + Object.assign(this.metadata, { name }); + } +}