✨ added support muted notifications

README.md

@@ -432,7 +432,7 @@ The `match` regular expression is used to match against the monitor identifier t
 - `identifier` - the name of the failing identifier (example: www.codeo.co.za)
 
 The value is composed as follows: `type|label|identifier`. For example: `web|web-performance|www.codeo.co.za`. The regular expression for match
-will thus be compared against this string value (case insensitive).
+will thus be compared against this string value (case-insensitive).
 
 ### SMS
 

src/digest/alerter.spec.ts

@@ -217,9 +217,8 @@ describe("alerter", () => {
                 expect(Array.from(alerts[0].affectedKeys)).toEqual([result.uniqueId, result2.uniqueId]);
             });
             describe("but if muted", () => {
-                it("should not alert", async () => {
+                it("should send muted notification", async () => {
                     // arrange
-
                     const config = new DigestConfiguration({
                         "mute-windows": [
                             {
@@ -235,21 +234,7 @@ describe("alerter", () => {
                         identifier: "www.codeo.co.za",
                     });
                     const context = new DigestContext([previousSnapshot], []);
-                    const result = new Result(
-                        new Date(),
-                        "web",
-                        "health",
-                        "www.codeo.co.za",
-                        false,
-                        "FAIL",
-                        0,
-                        false,
-                        {
-                            alert: {
-                                channels: ["console"]
-                            }
-                        }
-                    );
+                    const result = getTestResult();
                     context.addSnapshotForResult(result);
                     const alert = new AlertState({
                         channel: "console",
@@ -258,28 +243,14 @@ describe("alerter", () => {
                         affected: JSON.stringify([[result.uniqueId, previousSnapshot]])
                     });
                     await persistAlerts([alert]);
-                    const result2 = new Result(
-                        new Date(),
-                        "web",
-                        "health",
-                        "www.codeo2.co.za",
-                        false,
-                        "FAIL",
-                        0,
-                        false,
-                        {
-                            alert: {
-                                channels: ["console"]
-                            }
-                        }
-                    );
+                    const result2 = getTestResult();
                     context.addSnapshotForResult(result2);
 
                     // act
                     await executeAlerts(config, context);
 
                     // assert
-                    expect(console.log).not.toHaveBeenCalled();
+                    expect(console.log).toHaveBeenCalledWith(expect.stringMatching("🔕 Alerts muted at"));
                 });
             });
         });
@@ -367,7 +338,7 @@ describe("alerter", () => {
         });
         describe("but if some muted muted", () => {
             describe("if none left", () => {
-                it("should not send notification", async () => {
+                it("should send muted notification", async () => {
                     // arrange
                     const config = new DigestConfiguration({
                         "mute-windows": [
@@ -396,7 +367,7 @@ describe("alerter", () => {
                     await executeAlerts(config, context);
 
                     // assert
-                    expect(console.log).not.toHaveBeenCalled();
+                    expect(console.log).toHaveBeenCalledWith(expect.stringContaining("🔕 Alerts muted at "));
                     const alerts = await getAlerts();
                     expect(alerts.length).toEqual(0);
                 });

src/digest/alerter.ts

@@ -8,12 +8,12 @@ import { DigestConfiguration } from "../models/digest";
 export async function executeAlerts(
     config: DigestConfiguration,
     context: DigestContext) {
-    const alerts = await getAlerts();
-    setPreviousSnapshotContextForAlerts(alerts, context);
-    const distinctChannels = getChannelsAffected(context.alertableSnapshots(config));
-    const newAlerts = detectNewAlerts(alerts, distinctChannels);
-    const existingAlerts = detectExistingAlerts(alerts, distinctChannels);
-    const resolvedAlerts = detectResolvedAlerts(alerts, distinctChannels);
+    const alertsFromLastRound = await getAlerts();
+    setPreviousSnapshotContextForAlerts(alertsFromLastRound, context);
+    const distinctChannels = getChannelsAffected(context.digestableSnapshots);
+    const newAlerts = detectNewAlerts(alertsFromLastRound, distinctChannels);
+    const existingAlerts = detectExistingAlerts(alertsFromLastRound, distinctChannels);
+    const resolvedAlerts = detectResolvedAlerts(alertsFromLastRound, distinctChannels);
     await triggerAlerts(
         config,
         context,
@@ -35,6 +35,9 @@ async function sendNewAlerts(
             return;
         }
         const snapshots = context.getAlertableSnapshotsForChannel(config, channel);
+        if (snapshots.length === 0) {
+            return;
+        }
         alert.start_date = earliestDateFor(snapshots);
         await channel.sendNewAlert(
             snapshots,
@@ -63,6 +66,12 @@ async function sendOngoingAlerts(
             return;
         }
         const snapshots = context.getAlertableSnapshotsForChannel(config, channel);
+        if (snapshots.length === 0) {
+            // only way we can get here is if the channels alerts are muted
+            alert.setMuted();
+            await sendMutedAlert(config, alert);
+            return;
+        }
         if (channel.canSendAlert(alert)) {
             await channel.sendOngoingAlert(snapshots, alert);
             alert.last_alert_date = new Date();
@@ -79,7 +88,10 @@ async function sendResolvedAlerts(
     await Promise.all(alerts.map(async alert => {
         alert.removeMuted(config.muteWindows);
         alert.resolve();
-        if (alert.affected.size === 0) {
+        if (alert.size === 0) {
+            if (alert.isMuted) {
+                await sendMutedAlert(config, alert);
+            }
             return;
         }
         const channel = config.getChannelConfig(alert.channel);
@@ -90,6 +102,14 @@ async function sendResolvedAlerts(
     }));
 }
 
+async function sendMutedAlert(config: DigestConfiguration, alert: AlertState) {
+    const channel = config.getChannelConfig(alert.channel);
+    if (!channel) {
+        return;
+    }
+    await channel.sendMutedAlert(alert);
+}
+
 async function triggerAlerts(
     config: DigestConfiguration,
     context: DigestContext,

src/digest/digest.ts

@@ -118,11 +118,14 @@ export class DigestContext {
         return this._snapshots;
     }
 
+    get digestableSnapshots(): Snapshot[] {
+        return this._snapshots.filter(x => x.isDigestable);
+    }
+
     public getAlertableSnapshotsForChannel(
         config: DigestConfiguration,
         channel: ChannelConfig) {
         return this.alertableSnapshots(config)
-            .filter(x => x.isDigestable)
             .filter(x => x.alert?.channels?.some(c => channel.isMatchFor(c)));
     }
 
@@ -192,8 +195,7 @@ export class DigestContext {
     }
 
     public alertableSnapshots(config: DigestConfiguration): Snapshot[] {
-        return this.snapshots
-            .filter(x => x.isDigestable)
+        return this.digestableSnapshots
             .filter(x => !config.muteWindows.some(m => m.isMuted(x.uniqueId)));
     }
 }

src/models/alerts.ts

@@ -19,6 +19,7 @@ export class AlertState {
     public state?: any;
     private _resolved: boolean;
     private _previousSnapshots: Map<string, Snapshot> = new Map<string, Snapshot>();
+    private _wasMuted: boolean;
 
     constructor(data: any) {
         for(const key in data) {
@@ -55,7 +56,7 @@ export class AlertState {
     }
 
     public get endTime() {
-        return this.isResolved ? toLocalTimeString(new Date()) : null;
+        return this.isResolved || this.isMuted ? toLocalTimeString(new Date()) : null;
     }
 
     public get durationMinutes() {
@@ -71,6 +72,14 @@ export class AlertState {
         return this._resolved;
     }
 
+    public get isMuted() {
+        return this._wasMuted;
+    }
+
+    public get size() {
+        return this.affected.size;
+    }
+
     public get affectedKeys(): string[] {
         return Array.from(this.affected.keys());
     }
@@ -104,10 +113,17 @@ export class AlertState {
         const keys = Array.from(this.affected.keys());
         const muted = keys.filter(x => muteWindows.some(y => y.isMuted(x)));
         muted.forEach(x => this.affected.delete(x));
+        const hadAlerts = keys.length > 0;
+        const noAlertsNow = this.affected.size === 0;
+        this._wasMuted = hadAlerts && noAlertsNow;
     }
 
     setPreviousSnapshots(previous: Snapshot[]) {
         this._previousSnapshots = new Map<string, Snapshot>();
         previous.forEach(x => this._previousSnapshots.set(x.uniqueId, x));
     }
+
+    setMuted() {
+        this._wasMuted = true;
+    }
 }

src/models/channels/base.ts

@@ -41,6 +41,7 @@ export abstract class ChannelConfig {
     public abstract sendNewAlert(snapshots: Snapshot[], alert: AlertState): Promise<void>;
     public abstract sendOngoingAlert(snapshots: Snapshot[], alert: AlertState): Promise<void>;
     public abstract sendResolvedAlert(alert: AlertState): Promise<void>;
+    public abstract sendMutedAlert(alert: AlertState): Promise<void>;
     public abstract pingAboutOngoingAlert(snapshots: Snapshot[], alert: AlertState): Promise<void>;
 
     canSendAlert(alert: AlertState): boolean {

src/models/channels/console.ts

@@ -24,7 +24,13 @@ export class ConsoleChannelConfig extends ChannelConfig {
     sendResolvedAlert(alert: AlertState): Promise<void> {
         const message = `${ this.prefix } ✅ Outage ended at ${ alert.endTime }. Duration was ${ alert.durationHuman }. ${ this.postfix }`.trim();
         console.log(message);
-        return Promise  .resolve();
+        return Promise.resolve();
+    }
+
+    sendMutedAlert(alert: AlertState): Promise<void> {
+        const message = `${ this.prefix } 🔕 Alerts muted at ${ alert.endTime }. ${ this.postfix }`.trim();
+        console.log(message);
+        return Promise.resolve();
     }
 
     public async pingAboutOngoingAlert(

src/models/channels/slack.ts

@@ -21,7 +21,7 @@ export class SlackChannelConfig extends ChannelConfig {
         snapshots: Snapshot[],
         alert: AlertState): string {
         const msg = this._generateFull(snapshots, alert);
-        if (msg.length < 3800) {
+        if (msg.length <= 3000) {
             return msg;
         }
         return this._generateSummary(snapshots, alert);
@@ -71,6 +71,8 @@ export class SlackChannelConfig extends ChannelConfig {
         const parts = [];
         if (alert.isResolved) {
             parts.push(`${ this.prefix } ✅ Outage Resolved!`);
+        } else if (alert.isMuted) {
+            parts.push(`${ this.prefix } 🔕 Outage Muted!`);
         } else {
             parts.push(`${ this.prefix } 🔥 Ongoing Outage!`);
         }
@@ -96,7 +98,7 @@ export class SlackChannelConfig extends ChannelConfig {
         }
         const resolved = alert.getResolvedSnapshotList(snapshots.map(x => x.uniqueId));
         if (resolved.length > 0) {
-            parts.push(`*☑️ ${ resolved.length } resolved ${ pluraliseWithS("check", resolved.length) }:*`);
+            parts.push(`*☑️ ${ resolved.length } resolved/muted ${ pluraliseWithS("check", resolved.length) }:*`);
             resolved.forEach(x => {
                 const lastResult = x.lastSnapshot ? `(last result before resolution: _${ x.lastSnapshot.result }_)` : "";
                 parts.push(`    • ${ x.key.type }:${ x.key.label } → ${ x.key.identifier } ${ lastResult } ${ this.generateLinks(x.lastSnapshot) }`);
@@ -150,6 +152,19 @@ export class SlackChannelConfig extends ChannelConfig {
         ]);
     }
 
+    async sendMutedAlert(alert: AlertState): Promise<void> {
+        await Promise.all([
+            this.postToSlack(
+                this.generateMessage([], alert),
+                alert.state),
+            this.postToSlack(
+                `🔕 <!channel> Affected alerts were muted at ${ alert.endTime }.\n_See above for more details about affected services._`,
+                alert.state,
+                true
+            )
+        ]);
+    }
+
     async postToSlack(
         message: string,
         state?: any,

src/models/channels/sms.ts

@@ -38,6 +38,11 @@ export class SMSChannelConfig extends ChannelConfig {
         await this.sendSMSToAllContacts(message);
     }
 
+    public async sendMutedAlert(alert: AlertState): Promise<void> {
+        const message = `${ this.prefix } Current Alert MUTED at ${ alert.endTime }.\n\n${ this.postfix }`.trim();
+        await this.sendSMSToAllContacts(message);
+    }
+
     public async pingAboutOngoingAlert(
         _snapshots: Snapshot[],
         _alert: AlertState): Promise<void> {

src/models/db.spec.ts

@@ -344,6 +344,7 @@ describe("persistResults", () => {
                 // arrange
                 const alert = AlertState.New("test");
                 alert.state = { test: 123 };
+                alert.track([getTestSnapshot()]);
                 await persistAlerts([alert]);
 
                 // act
@@ -352,6 +353,36 @@ describe("persistResults", () => {
                 // assert
                 expect(alerts[0].state).toEqual(alert.state);
             });
+            describe("with muted", () => {
+                it("should not persist", async () => {
+                    // arrange
+                    const alert = AlertState.New("test");
+                    alert.state = { test: 123 };
+                    alert.track([getTestSnapshot()]);
+                    alert.setMuted();
+                    await persistAlerts([alert]);
+
+                    // act
+                    const alerts = await getAlerts();
+
+                    // assert
+                    expect(alerts.length).toEqual(0)
+                });
+            });
+            describe("with no affected", () => {
+                it("should not persist", async () => {
+                    // arrange
+                    const alert = AlertState.New("test");
+                    alert.state = { test: 123 };
+                    await persistAlerts([alert]);
+
+                    // act
+                    const alerts = await getAlerts();
+
+                    // assert
+                    expect(alerts.length).toEqual(0);
+                });
+            });
             describe("with affected", () => {
                 it("should be able to retrieve state", async () => {
                     // arrange

src/models/db.ts

@@ -60,12 +60,13 @@ async function saveSnapshots(conn: Knex, snapshots: Snapshot[]) {
 }
 
 export async function persistAlerts(alerts: AlertState[]) {
+    const persistable = alerts.filter(x => x.size > 0 && !x.isMuted);
     await _connection.transaction(async trx => {
         await trx("alerts").truncate();
-        if (alerts.length === 0) {
+        if (persistable.length === 0) {
             return;
         }
-        await trx("alerts").insert(alerts.map(x => {
+        await trx("alerts").insert(persistable.map(x => {
             return {
                 channel: x.channel,
                 start_date: x.start_date.toISOString(),
@@ -77,7 +78,6 @@ export async function persistAlerts(alerts: AlertState[]) {
     });
 }
 
-
 export async function mutateAndPersistSnapshotState(
     snapshots: Snapshot[],
     logIdsToDelete: number[]) {