fix: Add missing exclude_regex logic

A previous refactor accidentally removed the code to ignore lines that match a specific pattern. Semver: patch Ref: LOG-7472
logdna · Oct 5, 2020 · 4924bd4 · 4924bd4
1 parent 06bf28e
commit 4924bd4
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 16 deletions.
diff --git a/.taprc b/.taprc
@@ -15,6 +15,6 @@ coverage-report:
 files:
 - test/**/*.js
 statements: 71
-branches: 58
+branches: 57
 functions: 77
 lines: 71
diff --git a/index.js b/index.js
@@ -113,11 +113,6 @@ function loadConfig(program) {
         parsedConfig.exclude = process.env.LOGDNA_EXCLUDE
       }
 
-      // allow exclude regex to be passed via env
-      if (process.env.LOGDNA_EXCLUDE_REGEX) {
-        parsedConfig.exclude_regex = process.env.LOGDNA_EXCLUDE_REGEX
-      }
-
       if (process.env.USEJOURNALD) {
         parsedConfig.usejournald = process.env.USEJOURNALD
       }
@@ -190,12 +185,12 @@ function loadConfig(program) {
         saveMessages.push(`Exclusions: ${processed.diff} been saved to config.`)
       }
 
-      if (program.excludeRegex) {
-        parsedConfig.exclude_regex = program.excludeRegex
-        // strip leading and trailing / if exists
-        if (parsedConfig.exclude_regex.substring(0, 1) === '/' && parsedConfig.exclude_regex.substring(parsedConfig.exclude_regex.length - 1) === '/') {
-          parsedConfig.exclude_regex = parsedConfig.exclude_regex.substring(1, parsedConfig.exclude_regex.length - 1)
-        }
+      if (program.excludeRegex || process.env.LOGDNA_EXCLUDE_REGEX) {
+        const re = program.excludeRegex || process.env.LOGDNA_EXCLUDE_REGEX
+        // FIXME(darinspivey) - Error prone. Doesn't support modifiers or ending with
+        // a slash.  The user should be forced to provide open and closing slashes,
+        // otherwise it's too hard to know what's part of the pattern text.
+        parsedConfig.exclude_regex = re.replace(/^\//, '').replace(/\/$/, '')
         saveMessages.push(`Exclude pattern: /${parsedConfig.exclude_regex}/ been saved to config.`)
       }
 
@@ -230,6 +225,10 @@ function loadConfig(program) {
       , rescanTimer: null
       }
 
+      if (config.exclude_regex) {
+        config.exclude_regex = new RegExp(config.exclude_regex)
+      }
+
       return getos(cb)
     }
   , (distro, cb) => {

diff --git a/lib/file-utilities.js b/lib/file-utilities.js
@@ -102,16 +102,17 @@ function getFiles(config, dir, callback) {
 
 function streamFiles(config, logfiles, callback) {
   logfiles.forEach((file) => {
-    let tail, i
+    let tail
 
     if (os.platform() !== 'win32' && config.TAIL_MODE === 'unix') {
       debug('tailing: ' + file)
       tail = spawn('tail', ['-n0', '-F', file])
       tail.stdout.on('data', (data) => {
         const lines = data.toString().trim().split('\n')
-        for (i = 0; i < lines.length; i++) {
+        for (const line of lines) {
+          if (config.exclude_regex && config.exclude_regex.test(line)) continue
           client.log.agentLog({
-            line: lines[i]
+            line
           , f: file
           })
         }
@@ -128,6 +129,7 @@ function streamFiles(config, logfiles, callback) {
       tail = TailReadStream.tail(file, config)
       tail.pipe(new Splitter())
         .on('data', (line) => {
+          if (config.exclude_regex && config.exclude_regex.test(line)) return
           client.log.agentLog({
             line
           , f: file

diff --git a/test/integration/start.js b/test/integration/start.js
@@ -10,7 +10,7 @@ const fileUtilities = require('../../lib/file-utilities.js')
 
 nock.disableNetConnect()
 
-test('start() calls healthcheck and tails a log', (t) => {
+test('start() creates a client logger and tails a log', (t) => {
   t.plan(2)
   t.on('end', () => {
     nock.cleanAll()
@@ -69,3 +69,71 @@ test('start() calls healthcheck and tails a log', (t) => {
     await fs.promises.appendFile(logPath, 'Here is my line1\nHere is my line 2\n', 'utf8')
   }, 500)
 })
+
+test('exclude_regex successfully ignores lines', (t) => {
+  t.plan(2)
+  t.on('end', () => {
+    nock.cleanAll()
+    fileUtilities.gracefulShutdown()
+  })
+
+  const logname = 'regex-testing.log'
+  const tempDir = t.testdir({
+    [logname]: ''
+  })
+  const logPath = path.join(tempDir, logname)
+
+  const config = {
+    ...constants
+  , hostname: 'MyHostMachine'
+  , UserAgent: 'logdna-agent/1.6.5 (darwin)'
+  , mac: '3c:22:fb:27:72:f5'
+  , ip: '192.168.1.9'
+  , COMPRESS: false
+  , logdir: [tempDir]
+  , key: '<MY KEY HERE>'
+  , exclude_regex: /\bNOPE\b/
+  }
+
+  nock(config.LOGDNA_URL)
+    .post(/.*/, (body) => {
+      const expected = {
+        e: 'ls'
+      , ls: [
+          {
+            line: 'This line is good'
+          , f: logPath
+          }
+        , {
+            line: 'Good also'
+          , f: logPath
+          }
+        , {
+            line: 'Good because of boundary failure with xNOPEx'
+          , f: logPath
+          }
+        ]
+      }
+      t.match(body, expected, 'Ingester POST body is correct')
+      return true
+    })
+    .query((qs) => {
+      t.match(qs, {
+        hostname: config.hostname
+      , ip: config.ip
+      , mac: config.mac
+      , tags: ''
+      }, 'Ingester POST QUERY_STRING is correct')
+      return true
+    })
+    .reply(200, 'Ingested')
+
+  start(config)
+
+  setTimeout(async () => {
+    const lines = 'This line is good\nThis line is NOPE\nGood also\n'
+      + 'Good because of boundary failure with xNOPEx\nAnd NOPE way this is good'
+
+    await fs.promises.appendFile(logPath, lines, 'utf8')
+  }, 500)
+})