diff --git a/README.md b/README.md index 13566eb..fd3921f 100644 --- a/README.md +++ b/README.md @@ -432,7 +432,7 @@ The `match` regular expression is used to match against the monitor identifier t - `identifier` - the name of the failing identifier (example: www.codeo.co.za) The value is composed as follows: `type|label|identifier`. For example: `web|web-performance|www.codeo.co.za`. The regular expression for match -will thus be compared against this string value (case insensitive). +will thus be compared against this string value (case-insensitive). ### SMS diff --git a/src/digest/alerter.spec.ts b/src/digest/alerter.spec.ts index c41bc49..45790f7 100644 --- a/src/digest/alerter.spec.ts +++ b/src/digest/alerter.spec.ts @@ -217,9 +217,8 @@ describe("alerter", () => { expect(Array.from(alerts[0].affectedKeys)).toEqual([result.uniqueId, result2.uniqueId]); }); describe("but if muted", () => { - it("should not alert", async () => { + it("should send muted notification", async () => { // arrange - const config = new DigestConfiguration({ "mute-windows": [ { @@ -235,21 +234,7 @@ describe("alerter", () => { identifier: "www.codeo.co.za", }); const context = new DigestContext([previousSnapshot], []); - const result = new Result( - new Date(), - "web", - "health", - "www.codeo.co.za", - false, - "FAIL", - 0, - false, - { - alert: { - channels: ["console"] - } - } - ); + const result = getTestResult(); context.addSnapshotForResult(result); const alert = new AlertState({ channel: "console", @@ -258,28 +243,14 @@ describe("alerter", () => { affected: JSON.stringify([[result.uniqueId, previousSnapshot]]) }); await persistAlerts([alert]); - const result2 = new Result( - new Date(), - "web", - "health", - "www.codeo2.co.za", - false, - "FAIL", - 0, - false, - { - alert: { - channels: ["console"] - } - } - ); + const result2 = getTestResult(); context.addSnapshotForResult(result2); // act await executeAlerts(config, context); // assert - expect(console.log).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith(expect.stringMatching("🔕 Alerts muted at")); }); }); }); @@ -367,7 +338,7 @@ describe("alerter", () => { }); describe("but if some muted muted", () => { describe("if none left", () => { - it("should not send notification", async () => { + it("should send muted notification", async () => { // arrange const config = new DigestConfiguration({ "mute-windows": [ @@ -396,7 +367,7 @@ describe("alerter", () => { await executeAlerts(config, context); // assert - expect(console.log).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("🔕 Alerts muted at ")); const alerts = await getAlerts(); expect(alerts.length).toEqual(0); }); diff --git a/src/digest/alerter.ts b/src/digest/alerter.ts index 82aceac..4f82a93 100644 --- a/src/digest/alerter.ts +++ b/src/digest/alerter.ts @@ -8,12 +8,12 @@ import { DigestConfiguration } from "../models/digest"; export async function executeAlerts( config: DigestConfiguration, context: DigestContext) { - const alerts = await getAlerts(); - setPreviousSnapshotContextForAlerts(alerts, context); - const distinctChannels = getChannelsAffected(context.alertableSnapshots(config)); - const newAlerts = detectNewAlerts(alerts, distinctChannels); - const existingAlerts = detectExistingAlerts(alerts, distinctChannels); - const resolvedAlerts = detectResolvedAlerts(alerts, distinctChannels); + const alertsFromLastRound = await getAlerts(); + setPreviousSnapshotContextForAlerts(alertsFromLastRound, context); + const distinctChannels = getChannelsAffected(context.digestableSnapshots); + const newAlerts = detectNewAlerts(alertsFromLastRound, distinctChannels); + const existingAlerts = detectExistingAlerts(alertsFromLastRound, distinctChannels); + const resolvedAlerts = detectResolvedAlerts(alertsFromLastRound, distinctChannels); await triggerAlerts( config, context, @@ -35,6 +35,9 @@ async function sendNewAlerts( return; } const snapshots = context.getAlertableSnapshotsForChannel(config, channel); + if (snapshots.length === 0) { + return; + } alert.start_date = earliestDateFor(snapshots); await channel.sendNewAlert( snapshots, @@ -63,6 +66,12 @@ async function sendOngoingAlerts( return; } const snapshots = context.getAlertableSnapshotsForChannel(config, channel); + if (snapshots.length === 0) { + // only way we can get here is if the channels alerts are muted + alert.setMuted(); + await sendMutedAlert(config, alert); + return; + } if (channel.canSendAlert(alert)) { await channel.sendOngoingAlert(snapshots, alert); alert.last_alert_date = new Date(); @@ -79,7 +88,10 @@ async function sendResolvedAlerts( await Promise.all(alerts.map(async alert => { alert.removeMuted(config.muteWindows); alert.resolve(); - if (alert.affected.size === 0) { + if (alert.size === 0) { + if (alert.isMuted) { + await sendMutedAlert(config, alert); + } return; } const channel = config.getChannelConfig(alert.channel); @@ -90,6 +102,14 @@ async function sendResolvedAlerts( })); } +async function sendMutedAlert(config: DigestConfiguration, alert: AlertState) { + const channel = config.getChannelConfig(alert.channel); + if (!channel) { + return; + } + await channel.sendMutedAlert(alert); +} + async function triggerAlerts( config: DigestConfiguration, context: DigestContext, diff --git a/src/digest/digest.ts b/src/digest/digest.ts index 2ad29e1..4ec5e74 100644 --- a/src/digest/digest.ts +++ b/src/digest/digest.ts @@ -118,11 +118,14 @@ export class DigestContext { return this._snapshots; } + get digestableSnapshots(): Snapshot[] { + return this._snapshots.filter(x => x.isDigestable); + } + public getAlertableSnapshotsForChannel( config: DigestConfiguration, channel: ChannelConfig) { return this.alertableSnapshots(config) - .filter(x => x.isDigestable) .filter(x => x.alert?.channels?.some(c => channel.isMatchFor(c))); } @@ -192,8 +195,7 @@ export class DigestContext { } public alertableSnapshots(config: DigestConfiguration): Snapshot[] { - return this.snapshots - .filter(x => x.isDigestable) + return this.digestableSnapshots .filter(x => !config.muteWindows.some(m => m.isMuted(x.uniqueId))); } } diff --git a/src/models/alerts.ts b/src/models/alerts.ts index f25c65a..86e71e4 100644 --- a/src/models/alerts.ts +++ b/src/models/alerts.ts @@ -19,6 +19,7 @@ export class AlertState { public state?: any; private _resolved: boolean; private _previousSnapshots: Map = new Map(); + private _wasMuted: boolean; constructor(data: any) { for(const key in data) { @@ -55,7 +56,7 @@ export class AlertState { } public get endTime() { - return this.isResolved ? toLocalTimeString(new Date()) : null; + return this.isResolved || this.isMuted ? toLocalTimeString(new Date()) : null; } public get durationMinutes() { @@ -71,6 +72,14 @@ export class AlertState { return this._resolved; } + public get isMuted() { + return this._wasMuted; + } + + public get size() { + return this.affected.size; + } + public get affectedKeys(): string[] { return Array.from(this.affected.keys()); } @@ -104,10 +113,17 @@ export class AlertState { const keys = Array.from(this.affected.keys()); const muted = keys.filter(x => muteWindows.some(y => y.isMuted(x))); muted.forEach(x => this.affected.delete(x)); + const hadAlerts = keys.length > 0; + const noAlertsNow = this.affected.size === 0; + this._wasMuted = hadAlerts && noAlertsNow; } setPreviousSnapshots(previous: Snapshot[]) { this._previousSnapshots = new Map(); previous.forEach(x => this._previousSnapshots.set(x.uniqueId, x)); } + + setMuted() { + this._wasMuted = true; + } } diff --git a/src/models/channels/base.ts b/src/models/channels/base.ts index ca8fc81..539dfcb 100644 --- a/src/models/channels/base.ts +++ b/src/models/channels/base.ts @@ -41,6 +41,7 @@ export abstract class ChannelConfig { public abstract sendNewAlert(snapshots: Snapshot[], alert: AlertState): Promise; public abstract sendOngoingAlert(snapshots: Snapshot[], alert: AlertState): Promise; public abstract sendResolvedAlert(alert: AlertState): Promise; + public abstract sendMutedAlert(alert: AlertState): Promise; public abstract pingAboutOngoingAlert(snapshots: Snapshot[], alert: AlertState): Promise; canSendAlert(alert: AlertState): boolean { diff --git a/src/models/channels/console.ts b/src/models/channels/console.ts index db605bd..0b76e30 100644 --- a/src/models/channels/console.ts +++ b/src/models/channels/console.ts @@ -24,7 +24,13 @@ export class ConsoleChannelConfig extends ChannelConfig { sendResolvedAlert(alert: AlertState): Promise { const message = `${ this.prefix } ✅ Outage ended at ${ alert.endTime }. Duration was ${ alert.durationHuman }. ${ this.postfix }`.trim(); console.log(message); - return Promise .resolve(); + return Promise.resolve(); + } + + sendMutedAlert(alert: AlertState): Promise { + const message = `${ this.prefix } 🔕 Alerts muted at ${ alert.endTime }. ${ this.postfix }`.trim(); + console.log(message); + return Promise.resolve(); } public async pingAboutOngoingAlert( diff --git a/src/models/channels/slack.ts b/src/models/channels/slack.ts index 9fbb1d5..b03b67a 100644 --- a/src/models/channels/slack.ts +++ b/src/models/channels/slack.ts @@ -21,7 +21,7 @@ export class SlackChannelConfig extends ChannelConfig { snapshots: Snapshot[], alert: AlertState): string { const msg = this._generateFull(snapshots, alert); - if (msg.length < 3800) { + if (msg.length <= 3000) { return msg; } return this._generateSummary(snapshots, alert); @@ -71,6 +71,8 @@ export class SlackChannelConfig extends ChannelConfig { const parts = []; if (alert.isResolved) { parts.push(`${ this.prefix } ✅ Outage Resolved!`); + } else if (alert.isMuted) { + parts.push(`${ this.prefix } 🔕 Outage Muted!`); } else { parts.push(`${ this.prefix } 🔥 Ongoing Outage!`); } @@ -96,7 +98,7 @@ export class SlackChannelConfig extends ChannelConfig { } const resolved = alert.getResolvedSnapshotList(snapshots.map(x => x.uniqueId)); if (resolved.length > 0) { - parts.push(`*☑️ ${ resolved.length } resolved ${ pluraliseWithS("check", resolved.length) }:*`); + parts.push(`*☑️ ${ resolved.length } resolved/muted ${ pluraliseWithS("check", resolved.length) }:*`); resolved.forEach(x => { const lastResult = x.lastSnapshot ? `(last result before resolution: _${ x.lastSnapshot.result }_)` : ""; parts.push(` • ${ x.key.type }:${ x.key.label } → ${ x.key.identifier } ${ lastResult } ${ this.generateLinks(x.lastSnapshot) }`); @@ -150,6 +152,19 @@ export class SlackChannelConfig extends ChannelConfig { ]); } + async sendMutedAlert(alert: AlertState): Promise { + await Promise.all([ + this.postToSlack( + this.generateMessage([], alert), + alert.state), + this.postToSlack( + `🔕 Affected alerts were muted at ${ alert.endTime }.\n_See above for more details about affected services._`, + alert.state, + true + ) + ]); + } + async postToSlack( message: string, state?: any, diff --git a/src/models/channels/sms.ts b/src/models/channels/sms.ts index c9cc363..bbadccc 100644 --- a/src/models/channels/sms.ts +++ b/src/models/channels/sms.ts @@ -38,6 +38,11 @@ export class SMSChannelConfig extends ChannelConfig { await this.sendSMSToAllContacts(message); } + public async sendMutedAlert(alert: AlertState): Promise { + const message = `${ this.prefix } Current Alert MUTED at ${ alert.endTime }.\n\n${ this.postfix }`.trim(); + await this.sendSMSToAllContacts(message); + } + public async pingAboutOngoingAlert( _snapshots: Snapshot[], _alert: AlertState): Promise { diff --git a/src/models/db.spec.ts b/src/models/db.spec.ts index 6c2c470..1b38c13 100644 --- a/src/models/db.spec.ts +++ b/src/models/db.spec.ts @@ -344,6 +344,7 @@ describe("persistResults", () => { // arrange const alert = AlertState.New("test"); alert.state = { test: 123 }; + alert.track([getTestSnapshot()]); await persistAlerts([alert]); // act @@ -352,6 +353,36 @@ describe("persistResults", () => { // assert expect(alerts[0].state).toEqual(alert.state); }); + describe("with muted", () => { + it("should not persist", async () => { + // arrange + const alert = AlertState.New("test"); + alert.state = { test: 123 }; + alert.track([getTestSnapshot()]); + alert.setMuted(); + await persistAlerts([alert]); + + // act + const alerts = await getAlerts(); + + // assert + expect(alerts.length).toEqual(0) + }); + }); + describe("with no affected", () => { + it("should not persist", async () => { + // arrange + const alert = AlertState.New("test"); + alert.state = { test: 123 }; + await persistAlerts([alert]); + + // act + const alerts = await getAlerts(); + + // assert + expect(alerts.length).toEqual(0); + }); + }); describe("with affected", () => { it("should be able to retrieve state", async () => { // arrange diff --git a/src/models/db.ts b/src/models/db.ts index 1cd8bb2..486a50d 100644 --- a/src/models/db.ts +++ b/src/models/db.ts @@ -60,12 +60,13 @@ async function saveSnapshots(conn: Knex, snapshots: Snapshot[]) { } export async function persistAlerts(alerts: AlertState[]) { + const persistable = alerts.filter(x => x.size > 0 && !x.isMuted); await _connection.transaction(async trx => { await trx("alerts").truncate(); - if (alerts.length === 0) { + if (persistable.length === 0) { return; } - await trx("alerts").insert(alerts.map(x => { + await trx("alerts").insert(persistable.map(x => { return { channel: x.channel, start_date: x.start_date.toISOString(), @@ -77,7 +78,6 @@ export async function persistAlerts(alerts: AlertState[]) { }); } - export async function mutateAndPersistSnapshotState( snapshots: Snapshot[], logIdsToDelete: number[]) {