add alert for nomad-follower ACL issues

Author: manveru

nix/cloud/alerts.nix

@@ -18,6 +18,28 @@
     ];
   };
 
+  ci-world-nomad-follower = {
+    datasource = "loki";
+    rules = [
+      {
+        alert = "NomadFollowerACLTokenNotFound";
+        expr = ''sum(rate({syslog_identifier="nomad-follower"}[5m] |= `ACL token not found`)) by (host) > 0'';
+        for = "1m";
+        labels.severity = "critical";
+        annotations = {
+          description = ''
+            Detected nomad-follower ACL issue on {{ $labels.host }}
+            This may be due to a vault-agent issue and can usually be resolved
+            by restarting both services.
+            Without proper ACL, nomad-follower cannot send logs from Nomad jobs
+            to Loki, and Cicero will not be able to display logs for actions.
+          '';
+          summary = "nomad-follower ACL issue on {{ $labels.host }}";
+        };
+      }
+    ];
+  };
+
   # inherit (inputs.bitte-cells.bitte.alerts)
   # ;
 }

nix/cloud/hydrationProfile.nix

@@ -205,6 +205,7 @@ in {
           inherit
             (cell.alerts)
             ci-world-spongix
+            ci-world-nomad-follower
             # Upstream alerts which may have downstream deps can be imported here
 
             ;