Merge pull request #213 from jkcfg/2019-06-10-example-prometheus-rule

example: Add generation of a PrometheusRule CR in the micro-service example

examples/kubernetes/micro-service/alert.js

@@ -0,0 +1,60 @@
+import * as prometheus from './prometheus';
+
+const r = '2m';
+const selector = service => `job=${service.name}`;
+const ErrorRate = selector => `rate(http_request_total{${selector},code=~"5.."}[${r}])
+    / rate(http_request_duration_seconds_count{${selector}}[${r}])`;
+
+function RPSHTTPHighErrorRate(service) {
+  return {
+    alert: 'HighErrorRate',
+    expr: `${ErrorRate(selector(service))} * 100 > 10`,
+    for: '5m',
+    labels: {
+      severity: 'critical',
+    },
+    annotations: {
+      service: service.name,
+      description: `More than 10% of requests to the ${service.name} service are failing with 5xx errors`,
+      details: '{{$value | printf "%.1f"}}% errors for more than 5m',
+    },
+  };
+}
+
+// In real life we probably want something different. I'd like to enable dynamic
+// imports so we can import dashboards js definitions from string descriptions:
+//   https://developers.google.com/web/updates/2017/11/dynamic-import
+const alerts = {
+  'service.RPS.HTTP.HighErrorRate': RPSHTTPHighErrorRate,
+};
+
+function rules(service) {
+  if (!service.alerts) {
+    return [];
+  }
+
+  return service.alerts.map(a => alerts[a](service));
+}
+
+function PrometheusRule(service) {
+  return new prometheus.PrometheusRule(service.name, {
+    metadata: {
+      labels: {
+        app: service.name,
+        maintainer: service.maintainer,
+        prometheus: 'global',
+        role: 'alert-rules',
+      },
+    },
+    spec: {
+      groups: [{
+        name: `${service.name}-alerts.rules`,
+        rules: rules(service),
+      }],
+    },
+  });
+}
+
+export {
+  PrometheusRule,
+};

examples/kubernetes/micro-service/billing.yaml

@@ -9,3 +9,5 @@ service:
     path: /api/billing
   dashboards:
     - service.RPS.HTTP
+  alerts:
+    - service.RPS.HTTP.HighErrorRate

examples/kubernetes/micro-service/micro-service.js

@@ -4,6 +4,7 @@ import {
   Namespace, Deployment, Service, Ingress, ConfigMap,
 } from './kubernetes';
 import { Dashboard } from './dashboard';
+import { PrometheusRule } from './alert';
 
 const service = param.Object('service');
 const ns = service.namespace;
@@ -18,4 +19,5 @@ export default [
   { file: `${ns}/${service.name}-svc.yaml`, value: Service(service) },
   { file: `${ns}/${service.name}-ingress.yaml`, value: Ingress(service) },
   { file: `${ns}/${service.name}-dashboards-cm.yaml`, value: ConfigMap(service, `${service.name}-dashboards`, dashboards) },
+  { file: `${ns}/${service.name}-prometheus-rule.yaml`, value: PrometheusRule(service) },
 ];

examples/kubernetes/micro-service/prometheus.js

@@ -0,0 +1,9 @@
+export class PrometheusRule {
+  constructor(name, desc) {
+    this.apiVersion = 'monitoring.coreos.com/v1';
+    this.kind = 'PrometheusRule';
+    this.metadata = Object.assign({}, (desc && desc.metadata) || {}, { name });
+    this.spec = (desc && desc.spec) || undefined;
+    Object.assign(this.metadata, { name });
+  }
+}