Add Probe Fail Notification (#947)

* initial commit, adding separate fail detector * update tests * improve recovery detection * refactor recovery detection * update response checker tests * update flag for clarity * add minor comment * update tls check * fix typo --------- Co-authored-by: Nico Prananta <[email protected]> Co-authored-by: Denny Pradipta <[email protected]>
hyperjumptech · Feb 21, 2023 · 3452038 · 3452038
1 parent 9a82bf1
commit 3452038
Show file tree

Hide file tree

Showing 19 changed files with 200 additions and 89 deletions.
diff --git a/src/components/http-request/index.ts b/src/components/http-request/index.ts
@@ -154,15 +154,16 @@ export async function httpRequest({
 
     const responseTime = Date.now() - requestStartedAt
     const { data, headers, status } = resp
-    const requestType = 'HTTP'
 
     return {
-      requestType,
+      requestType: 'HTTP',
       data,
       body: data,
       status,
       headers,
       responseTime,
+
+      isProbeResponsive: true,
     }
   } catch (error: any) {
     const responseTime = Date.now() - requestStartedAt
@@ -176,20 +177,25 @@ export async function httpRequest({
         status: error?.response?.status,
         headers: error?.response?.headers,
         responseTime,
+
+        isProbeResponsive: true, // http status received, so connection ok
       }
     }
 
     // The request was made but no response was received
     // timeout is here, ECONNABORTED, ENOTFOUND, ECONNRESET, ECONNREFUSED
     if (error?.request) {
-      const status = errorRequestCodeToNumber(error?.code)
+      const status = errorRequestCodeToNumber(error?.code) // TODO: remove axios driver error mapping here
 
       return {
         data: '',
         body: '',
         status,
         headers: '',
         responseTime,
+
+        isProbeResponsive: false,
+        errMessage: error?.code,
       }
     }
 
@@ -200,6 +206,9 @@ export async function httpRequest({
       status: error.code || 'Unknown error',
       headers: '',
       responseTime,
+
+      isProbeResponsive: false,
+      errMessage: error.code || 'Unknown error',
     }
   }
 }

diff --git a/src/components/icmp-request/index.ts b/src/components/icmp-request/index.ts
@@ -56,6 +56,7 @@ export async function icmpRequest(
     status: 0,
     headers: '',
     responseTime: 0,
+    isProbeResponsive: false,
   }
 
   try {
@@ -68,17 +69,18 @@ export async function icmpRequest(
     icmpResp.numericHost = resp.numeric_host
     icmpResp.output = resp.output
 
-    return processICMPRequstResult(icmpResp)
+    return processICMPRequestResult(icmpResp)
   } catch (error: any) {
-    console.error('icmp got errror:', error)
+    console.error('icmp got error:', error)
     baseResponse.data = error // map error to data
+    baseResponse.errMessage = error
   }
 
   return baseResponse
 }
 
 // translates icmp specific response to base monika response
-export function processICMPRequstResult(
+export function processICMPRequestResult(
   params: icmpResponse
 ): ProbeRequestResponse {
   // build log message
@@ -93,5 +95,7 @@ export function processICMPRequstResult(
     body: msg,
     headers: {},
     responseTime: params.average || 0,
+
+    isProbeResponsive: true,
   }
 }
diff --git a/src/components/mariadb-request/index.ts b/src/components/mariadb-request/index.ts
@@ -1,3 +1,27 @@
+/**********************************************************************************
+ * MIT License                                                                    *
+ *                                                                                *
+ * Copyright (c) 2021 Hyperjump Technology                                        *
+ *                                                                                *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy   *
+ * of this software and associated documentation files (the "Software"), to deal  *
+ * in the Software without restriction, including without limitation the rights   *
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      *
+ * copies of the Software, and to permit persons to whom the Software is          *
+ * furnished to do so, subject to the following conditions:                       *
+ *                                                                                *
+ * The above copyright notice and this permission notice shall be included in all *
+ * copies or substantial portions of the Software.                                *
+ *                                                                                *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     *
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       *
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    *
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         *
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  *
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  *
+ * SOFTWARE.                                                                      *
+ **********************************************************************************/
+
 import { ProbeRequestResponse } from '../../interfaces/request'
 import { differenceInMilliseconds } from 'date-fns'
 import { createConnection } from 'mariadb'
@@ -21,6 +45,7 @@ export async function mariaRequest(
     status: 0,
     headers: '',
     responseTime: 0,
+    isProbeResponsive: false,
   }
 
   const startTime = new Date()
@@ -35,6 +60,7 @@ export async function mariaRequest(
     })
   } catch (error: any) {
     baseResponse.body = error.message
+    baseResponse.errMessage = error
     isConnected = false
   }
 
@@ -44,7 +70,8 @@ export async function mariaRequest(
   if (isConnected) {
     baseResponse.responseTime = duration
     baseResponse.body = 'database ok'
-    baseResponse.status = 200
+    baseResponse.status = 200 // TODO: Remove http mapping
+    baseResponse.isProbeResponsive = true
   }
 
   return baseResponse

diff --git a/src/components/mongodb-request/index.ts b/src/components/mongodb-request/index.ts
@@ -56,6 +56,7 @@ export async function mongoRequest(
     status: 0,
     headers: '',
     responseTime: 0,
+    isProbeResponsive: false,
   }
   const startTime = new Date()
   const result = await sendMongoRequest(params)
@@ -66,8 +67,10 @@ export async function mongoRequest(
     baseResponse.responseTime = duration
     baseResponse.body = result.message
     baseResponse.status = 200 // TODO: improve this up/down flag
+    baseResponse.isProbeResponsive = true
   } else {
     baseResponse.body = result.message
+    baseResponse.errMessage = result.message
   }
 
   return baseResponse

diff --git a/src/components/notification/__tests__/alert-message.test.ts b/src/components/notification/__tests__/alert-message.test.ts
@@ -52,6 +52,7 @@ describe('Alert message', () => {
           status: 404,
           headers: '',
           responseTime: 1000,
+          isProbeResponsive: true,
         },
       })
 
@@ -80,6 +81,7 @@ describe('Alert message', () => {
           status: 404,
           headers: '',
           responseTime: 1000,
+          isProbeResponsive: true,
         },
       })
 
@@ -115,6 +117,7 @@ describe('Alert message', () => {
           status: 404,
           headers: '',
           responseTime: 1000,
+          isProbeResponsive: true,
         },
       })
 
@@ -165,6 +168,7 @@ describe('Alert message', () => {
           status: 404,
           headers: '',
           responseTime: 1000,
+          isProbeResponsive: true,
         },
       })
 

diff --git a/src/components/notification/alert-message.ts b/src/components/notification/alert-message.ts
@@ -66,29 +66,11 @@ const getExpectedMessage = (
   isRecovery: boolean
 ): string | number => {
   const { status, data, headers, responseTime } = response
-  const isHTTPStatusCode = status >= 100 && status <= 599
 
   if (!alert.message) {
     if (isRecovery)
-      return `The request is back to normal and pass the query: ${alert.query}`
-    return `The request failed because the response does not pass the query: ${alert.query}. The actual response status is ${status} and the response time is ${responseTime}.`
-  }
-
-  if (!isHTTPStatusCode) {
-    switch (status) {
-      case 0:
-        return 'Host cannot be reached'
-      case 1:
-        return 'Connection reset'
-      case 2:
-        return 'Connection refused'
-      // for TCP request
-      case 5:
-        return isRecovery ? 'The request is back to normal' : alert?.message
-
-      default:
-        return status
-    }
+      return `The request is back to normal and passed the assertion: ${alert.query}`
+    return `The request failed because the response did not pass the query: ${alert.query}. The actual response status is ${status} and the response time is ${responseTime}.`
   }
 
   return Handlebars.compile(alert.message)({

diff --git a/src/components/notification/process-server-status.ts b/src/components/notification/process-server-status.ts
@@ -126,6 +126,7 @@ export const processThresholds = ({
 }) => {
   const { requests, incidentThreshold, recoveryThreshold, socket, name } = probe
   const request = requests?.[requestIndex]
+
   const id = `${probe?.id}:${name}:${requestIndex}:${
     request?.id || ''
   }-${incidentThreshold}:${recoveryThreshold} ${

diff --git a/src/components/postgres-request/index.ts b/src/components/postgres-request/index.ts
@@ -51,6 +51,7 @@ export async function postgresRequest(
     status: 0,
     headers: '',
     responseTime: 0,
+    isProbeResponsive: false,
   }
   const startTime = new Date()
   const result = await sendPsqlRequest(params)
@@ -61,8 +62,10 @@ export async function postgresRequest(
     baseResponse.responseTime = duration
     baseResponse.body = result.message
     baseResponse.status = 200
+    baseResponse.isProbeResponsive = result.isAlive
   } else {
     baseResponse.body = result.message
+    baseResponse.errMessage = result.message
   }
 
   return baseResponse

diff --git a/src/components/probe/index.ts b/src/components/probe/index.ts
@@ -61,6 +61,9 @@ interface ProbeSendNotification extends Omit<ProbeStatusProcessed, 'statuses'> {
   probeState?: ServerAlertState
 }
 
+const connectionRecoveryMsg = `Probe is accessible again`
+const connectionIncidentMsg = `Probe not accessible`
+
 const probeSendNotification = async (data: ProbeSendNotification) => {
   const eventEmitter = getEventEmitter()
 
@@ -218,14 +221,16 @@ async function responseProcessing({
   }
 }
 
+const isConnectionDown = new Map<string, boolean>()
+
 type doProbeParams = {
   checkOrder: number // the order of probe being processed
   probe: Probe // probe contains all the probes
   notifications: Notification[] // notifications contains all the notifications
 }
 /**
  * doProbe sends out the http request
- * @param {object} param object parameter
+ * @param {object} doProbeParams doProbe parameter
  * @returns {Promise<void>} void
  */
 export async function doProbe({
@@ -433,6 +438,9 @@ export async function doProbe({
       const request = probe.requests?.[requestIndex]
       const requestLog = new RequestLog(probe, requestIndex, checkOrder)
 
+      // create id-request
+      const id = `${probe?.id}:${request.url}:${requestIndex}:${request?.id} `
+
       try {
         // intentionally wait for a request to finish before processing next request in loop
         // eslint-disable-next-line no-await-in-loop
@@ -453,19 +461,6 @@ export async function doProbe({
         responses.push(probeRes)
         requestLog.setResponse(probeRes)
 
-        // decode error message based on returned driver status
-        if ([0, 1, 2, 3, 4, 599].includes(probeRes.status)) {
-          const errorMessageMap: Record<number, string> = {
-            0: 'URI not found', // axios error
-            1: 'Connection reset', // axios error
-            2: 'Connection refused', // axios error
-            3: 'Unknown error', // axios error
-            599: 'Request Timed out', // axios error
-          }
-
-          requestLog.addError(errorMessageMap[probeRes.status])
-        }
-
         // combine global probe alerts with all individual request alerts
         const probeAlerts = probe.alerts ?? []
         const combinedAlerts = [...probeAlerts, ...(request.alerts || [])]
@@ -486,6 +481,30 @@ export async function doProbe({
           validatedResponse,
         })
 
+        // so we've got a status that need to be reported/alerted
+        // 1. check first, this connection is up, but was it ever down? if yes then use a specific connection recovery msg
+        // 2. if this connection is down, save to map and send specific connection incident msg
+        // 3. if event is not for connection failure, send user specified notification msg
+        if (statuses[0].shouldSendNotification) {
+          if (
+            probeRes.isProbeResponsive && // if connection is successful but
+            isConnectionDown.has(id) // if connection was down then send custom alert. Else use user's alert.
+          ) {
+            validatedResponse[0].alert = {
+              assertion: '',
+              message: `${connectionRecoveryMsg}`,
+            }
+            isConnectionDown.delete(id) // connection is up, so remove from entry
+          } else if (!probeRes.isProbeResponsive) {
+            // if connection has failed, then lets send out specific notification
+            validatedResponse[0].alert = {
+              assertion: '',
+              message: `${connectionIncidentMsg}`,
+            }
+            isConnectionDown.set(id, true) // connection is down, so add to map
+          }
+        }
+
         // Done processing results, check if need to send out alerts
         checkThresholdsAndSendAlert(
           {

diff --git a/src/components/redis-request/index.ts b/src/components/redis-request/index.ts
@@ -55,6 +55,7 @@ export async function redisRequest(
     status: 0,
     headers: '',
     responseTime: 0,
+    isProbeResponsive: false,
   }
   const startTime = new Date()
   const result = await sendRedisRequest(params)
@@ -65,8 +66,10 @@ export async function redisRequest(
     baseResponse.responseTime = duration
     baseResponse.body = result.message
     baseResponse.status = 200 // TODO: improve this up/down flag
+    baseResponse.isProbeResponsive = true
   } else {
     baseResponse.body = result.message
+    baseResponse.errMessage = result.message
   }
 
   return baseResponse