From b4540033353ff42e247016d90b72863e8d9dba95 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 10 Mar 2024 22:48:38 -0700 Subject: [PATCH] feat: alert template message pt4 (#338) Introduce conditional alert routing helper `#is_match` --- .changeset/green-bulldogs-behave.md | 6 + .../clickhouse/__tests__/clickhouse.test.ts | 9 + packages/api/src/clickhouse/index.ts | 40 +++- .../src/tasks/__tests__/checkAlerts.test.ts | 183 +++++++++++++++++- packages/api/src/tasks/checkAlerts.ts | 41 +++- 5 files changed, 262 insertions(+), 17 deletions(-) create mode 100644 .changeset/green-bulldogs-behave.md diff --git a/.changeset/green-bulldogs-behave.md b/.changeset/green-bulldogs-behave.md new file mode 100644 index 000000000..e64667c09 --- /dev/null +++ b/.changeset/green-bulldogs-behave.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +feat: introduce conditional alert routing helper #is_match diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index cc6bc92ee..5378a0852 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -1271,6 +1271,9 @@ Array [ expect(data).toMatchInlineSnapshot(` Array [ Object { + "attributes": Object { + "testGroup": "group2", + }, "data": 777, "group": Array [ "group2", @@ -1278,6 +1281,9 @@ Array [ "ts_bucket": 1641340800, }, Object { + "attributes": Object { + "testGroup": "group1", + }, "data": 77, "group": Array [ "group1", @@ -1285,6 +1291,9 @@ Array [ "ts_bucket": 1641340800, }, Object { + "attributes": Object { + "testGroup": "group1", + }, "data": 7, "group": Array [ "group1", diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index 7add58cb4..7f0f2298a 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -1818,21 +1818,39 @@ export const getMultiSeriesChartLegacyFormat = async ({ const flatData = result.data.flatMap(row => { if (seriesReturnType === 'column') { - return series.map((_, i) => { + return series.map((s, i) => { + const groupBy = + s.type === 'number' ? [] : 'groupBy' in s ? s.groupBy : []; + const attributes = groupBy.reduce((acc, curVal, curIndex) => { + acc[curVal] = row.group[curIndex]; + return acc; + }, {} as Record); return { - ts_bucket: row.ts_bucket, - group: row.group, + attributes, data: row[`series_${i}.data`], + group: row.group, + ts_bucket: row.ts_bucket, }; }); } // Ratio only has 1 series + const groupBy = + series[0].type === 'number' + ? [] + : 'groupBy' in series[0] + ? series[0].groupBy + : []; + const attributes = groupBy.reduce((acc, curVal, curIndex) => { + acc[curVal] = row.group[curIndex]; + return acc; + }, {} as Record); return [ { - ts_bucket: row.ts_bucket, - group: row.group, + attributes, data: row['series_0.data'], + group: row.group, + ts_bucket: row.ts_bucket, }, ]; }); @@ -2550,8 +2568,9 @@ export const checkAlert = async ({ ` SELECT ? - count(*) as data, - toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ?)) as ts_bucket + count(*) AS data, + any(_string_attributes) AS attributes, + toUnixTimestamp(toStartOfInterval(timestamp, INTERVAL ?)) AS ts_bucket FROM ?? WHERE ? AND (?) GROUP BY ? @@ -2596,7 +2615,12 @@ export const checkAlert = async ({ }, }); const result = await rows.json< - ResponseJSON<{ data: string; group?: string; ts_bucket: number }> + ResponseJSON<{ + data: string; + group?: string; + ts_bucket: number; + attributes: Record; + }> >(); logger.info({ message: 'checkAlert', diff --git a/packages/api/src/tasks/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/__tests__/checkAlerts.test.ts index 77f02318d..4db96a9aa 100644 --- a/packages/api/src/tasks/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/__tests__/checkAlerts.test.ts @@ -283,9 +283,16 @@ describe('checkAlerts', () => { ).toMatchInlineSnapshot( `"{{__hdx_notify_channel__ channel=\\"action\\" id=\\"id-with-multiple-dashes\\"}}"`, ); + + // custom template id + expect( + translateExternalActionsToInternal('@action-{{action_id}}'), + ).toMatchInlineSnapshot( + `"{{__hdx_notify_channel__ channel=\\"action\\" id=\\"{{action_id}}\\"}}"`, + ); }); - it('renderAlertTemplate', async () => { + it('renderAlertTemplate - custom body with single action', async () => { jest .spyOn(slack, 'postMessageToWebhook') .mockResolvedValueOnce(null as any); @@ -356,6 +363,180 @@ describe('checkAlerts', () => { }, ); }); + + it('renderAlertTemplate - single action with custom action id', async () => { + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValueOnce(null as any); + jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ + data: [ + { + timestamp: '2023-11-16T22:10:00.000Z', + severity_text: 'error', + body: 'Oh no! Something went wrong!', + }, + { + timestamp: '2023-11-16T22:15:00.000Z', + severity_text: 'info', + body: 'All good!', + }, + ], + } as any); + + const team = await createTeam({ name: 'My Team' }); + await new Webhook({ + team: team._id, + service: 'slack', + url: 'https://hooks.slack.com/services/123', + name: 'My_Webhook', + }).save(); + + await renderAlertTemplate({ + template: 'Custom body @slack_webhook-{{attributes.webhookName}}', // partial name should work + view: { + ...defaultSearchView, + alert: { + ...defaultSearchView.alert, + channel: { + type: null, // using template instead + }, + }, + attributes: { + webhookName: 'My_Webhook', + }, + team: { + id: team._id.toString(), + logStreamTableVersion: team.logStreamTableVersion, + }, + }, + title: 'Alert for "My Search" - 10 lines found', + }); + + expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( + 1, + 'https://hooks.slack.com/services/123', + { + text: 'Alert for "My Search" - 10 lines found', + blocks: [ + { + text: { + text: [ + '**', + 'Group: "http"', + '10 lines found, expected less than 1 lines', + 'Custom body ', + '```', + 'Nov 16 22:10:00Z [error] Oh no! Something went wrong!', + 'Nov 16 22:15:00Z [info] All good!', + '```', + ].join('\n'), + type: 'mrkdwn', + }, + type: 'section', + }, + ], + }, + ); + }); + + it('renderAlertTemplate - #is_match with single action', async () => { + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValueOnce(null as any); + jest.spyOn(clickhouse, 'getLogBatch').mockResolvedValueOnce({ + data: [ + { + timestamp: '2023-11-16T22:10:00.000Z', + severity_text: 'error', + body: 'Oh no! Something went wrong!', + }, + { + timestamp: '2023-11-16T22:15:00.000Z', + severity_text: 'info', + body: 'All good!', + }, + ], + } as any); + + const team = await createTeam({ name: 'My Team' }); + await new Webhook({ + team: team._id, + service: 'slack', + url: 'https://hooks.slack.com/services/123', + name: 'My_Webhook', + }).save(); + + await renderAlertTemplate({ + template: + '{{#is_match "host" "web"}} @slack_webhook-My_Web {{/is_match}}', // partial name should work + view: { + ...defaultSearchView, + alert: { + ...defaultSearchView.alert, + channel: { + type: null, // using template instead + }, + }, + team: { + id: team._id.toString(), + logStreamTableVersion: team.logStreamTableVersion, + }, + attributes: { + host: 'web', + }, + }, + title: 'Alert for "My Search" - 10 lines found', + }); + + // @slack_webhook should not be called + await renderAlertTemplate({ + template: + '{{#is_match "host" "web"}} @slack_webhook-My_Web {{/is_match}}', // partial name should work + view: { + ...defaultSearchView, + alert: { + ...defaultSearchView.alert, + channel: { + type: null, // using template instead + }, + }, + team: { + id: team._id.toString(), + logStreamTableVersion: team.logStreamTableVersion, + }, + attributes: { + host: 'web2', + }, + }, + title: 'Alert for "My Search" - 10 lines found', + }); + + expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( + 1, + 'https://hooks.slack.com/services/123', + { + text: 'Alert for "My Search" - 10 lines found', + blocks: [ + { + text: { + text: [ + '**', + 'Group: "http"', + '10 lines found, expected less than 1 lines', + '', + '```', + 'Nov 16 22:10:00Z [error] Oh no! Something went wrong!', + 'Nov 16 22:15:00Z [info] All good!', + '```', + ].join('\n'), + type: 'mrkdwn', + }, + type: 'section', + }, + ], + }, + ); + }); }); describe('processAlert', () => { diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index 16e14aebd..e191bf9d8 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -3,7 +3,7 @@ // -------------------------------------------------------- import * as fns from 'date-fns'; import * as fnsTz from 'date-fns-tz'; -import Handlebars from 'handlebars'; +import Handlebars, { HelperOptions } from 'handlebars'; import { escapeRegExp, isString } from 'lodash'; import mongoose from 'mongoose'; import ms from 'ms'; @@ -34,6 +34,7 @@ type EnhancedDashboard = Omit & { team: ITeam }; const MAX_MESSAGE_LENGTH = 500; const NOTIFY_FN_NAME = '__hdx_notify_channel__'; +const IS_MATCH_FN_NAME = 'is_match'; const getLogViewEnhanced = async (logViewId: ObjectId) => { const logView = await LogView.findById(logViewId).populate<{ @@ -113,6 +114,7 @@ export const doesExceedThreshold = ( type AlertMessageTemplateDefaultView = { // FIXME: do we want to include groupBy in the external alert schema? alert: z.infer & { groupBy?: string }; + attributes: Record; dashboard: ReturnType< typeof translateDashboardDocumentToExternalDashboard > | null; @@ -386,7 +388,8 @@ export const getDefaultExternalAction = ( export const translateExternalActionsToInternal = (template: string) => { // ex: @webhook-1234_5678 -> "{{NOTIFY_FN_NAME channel="webhook" id="1234_5678}}" - return template.replace(/(?: |^)@([a-zA-Z0-9.@_-]+)/g, (match, input) => { + // ex: @webhook-{{attributes.webhookId}} -> "{{NOTIFY_FN_NAME channel="webhook" id="{{attributes.webhookId}}"}}" + return template.replace(/(?: |^)@([a-zA-Z0-9.{}@_-]+)/g, (match, input) => { const prefix = match.startsWith(' ') ? ' ' : ''; const [channel, ...ids] = input.split('-'); const id = ids.join('-'); @@ -407,6 +410,7 @@ export const renderAlertTemplate = async ({ }) => { const { alert, + attributes, dashboard, endTime, group, @@ -427,26 +431,43 @@ export const renderAlertTemplate = async ({ const _hb = Handlebars.create(); _hb.registerHelper(NOTIFY_FN_NAME, () => null); + _hb.registerHelper(IS_MATCH_FN_NAME, () => null); const hb = PromisedHandlebars(Handlebars); const registerHelpers = (rawTemplateBody: string) => { + const attributesMap = new Map(Object.entries(attributes ?? {})); + + hb.registerHelper( + IS_MATCH_FN_NAME, + function ( + targetKey: string, + targetValue: string, + options: HelperOptions, + ) { + if (attributesMap.get(targetKey) === targetValue) { + options.fn(this); + } + }, + ); + hb.registerHelper( NOTIFY_FN_NAME, async (options: { hash: Record }) => { const { channel, id } = options.hash; - // const [channel, id] = input.split('-'); if (channel !== 'slack_webhook') { throw new Error(`Unsupported channel type: ${channel}`); } - // rendered body template - const compiledTemplateBody = _hb.compile(rawTemplateBody); + // render id template + const renderedId = _hb.compile(id)(view); + // render body template + const renderedBody = _hb.compile(rawTemplateBody)(view); await notifyChannel({ channel, - id, + id: renderedId, message: { hdxLink: buildAlertMessageTemplateHdxLink(view), title, - body: compiledTemplateBody(view), + body: renderedBody, }, teamId: team.id, }); @@ -533,6 +554,7 @@ ${targetTemplate}`; const fireChannelEvent = async ({ alert, + attributes, dashboard, endTime, group, @@ -542,10 +564,11 @@ const fireChannelEvent = async ({ windowSizeInMins, }: { alert: AlertDocument; - logView: Awaited> | null; + attributes: Record; // TODO: support other types than string dashboard: EnhancedDashboard | null; endTime: Date; group?: string; + logView: Awaited> | null; startTime: Date; totalCount: number; windowSizeInMins: number; @@ -559,6 +582,7 @@ const fireChannelEvent = async ({ ...translateAlertDocumentToExternalAlert(alert), groupBy: alert.groupBy, }, + attributes, dashboard: dashboard ? translateDashboardDocumentToExternalDashboard({ _id: dashboard._id, @@ -783,6 +807,7 @@ export const processAlert = async (now: Date, alert: AlertDocument) => { try { await fireChannelEvent({ alert, + attributes: checkData.attributes, dashboard: targetDashboard, endTime: fns.addMinutes(bucketStart, windowSizeInMins), group: Array.isArray(checkData.group)