Merge pull request #49 from nttgin/dev

v1.21.0

.eslintrc.json

@@ -0,0 +1,21 @@
+{
+  "parser": "babel-eslint",
+  "ecmaFeatures": {
+    "modules": true,
+    "spread" : true,
+    "restParams" : true
+  },
+  "env" : {
+    "browser" : false,
+    "node" : true,
+    "es6" : true,
+    "mocha": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 6,
+    "sourceType": "module",
+    "ecmaFeatures": {
+      "jsx": true
+    }
+  }
+}

.hound.yml

@@ -0,0 +1,6 @@
+jshint:
+  enabled: false
+
+eslint:
+  enabled: true
+  config_file: .eslintrc.json

README.md

@@ -1,10 +1,11 @@
 [![Build Status](https://api.travis-ci.org/nttgin/bgpalerter.svg)](https://travis-ci.org/nttgin/bgpalerter)
 ![Dependabot Status](https://badgen.net/dependabot/nttgin/BGPalerter/?icon=dependabot)
+[![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com)
 
 # BGPalerter
 Real-time BGP monitoring tool, pre-configured for visibility loss and hijacks detection.
 
-You just run it. You don't need to provide any data source or connect it to anything in your network since it connects to publiic repos.
+You just run it. You don't need to provide any data source or connect it to anything in your network since it connects to public repos.
 
 It can deliver alerts on files, by email, on slack, and more.
 
@@ -27,8 +28,6 @@ If you want to know more about the source code (which is completely open) please
 In `config.yml.example` you can find other reporting mechanisms (e.g. email and slack) in addition to logging to files. 
 Please uncomment the related section and configure according to your needs.
 
-If you enable email reporting, download also the directory `reports/email_templates` in the same directory of the executable.
-
 
 ## Documentation
 

config.yml.example

@@ -53,6 +53,7 @@ reports:
 #      - visibility
 #      - path
 #    params:
+#      showPaths: 5 # Amount of AS_PATHs to report in the alert
 #      senderEmail: bgpalerter@xxxx
 #      # BGPalerter uses nodemailer.
 #      # The smtp section can be configured with all the parameters available at https://nodemailer.com/smtp/
@@ -86,13 +87,36 @@ reports:
 #        path: '#42cbf5'
 #      hooks:
 #        default: _YOUR_SLACK_WEBHOOK_URL_
+#
+#  - file: reportKafka
+#    channels:
+#      - hijack
+#      - newprefix
+#      - visibility
+#      - path
+#    params:
+#      host: localhost:9092
+#      topics:
+#        default: bgpalerter
+
+
+############################
+# Notification settings:
+# - notificationIntervalSeconds
+#       Defines the amount of seconds after which an alert can be repeated. An alert is repeated only if the event that
+#       triggered it is not yet solved. Please, don't set this value to Infinity, use instead alertOnlyOnce.
+#
+# - alertOnlyOnce - A boolean that, if set to true, will prevent repetitions of the same alert even if the event that
+#       triggered it is not yet solved. In this case notificationIntervalSeconds will be ignored.
+#       If set to true, the signature of all alerts will be cached in order to recognize if they already happened in
+#       the past. This may lead to a memory leak if the amount of alerts is considerable.
 
+notificationIntervalSeconds: 7200
+alertOnlyOnce: false
 
-checkStaleNotificationsSeconds: 60
-notificationIntervalSeconds: 1800 # Repeat the same alert (which keeps being triggered) after x seconds
-clearNotificationQueueAfterSeconds: 1900 # Stop with the alert for an event which didn't happen again in x seconds
+############################
 
-# The file containing the monitored prefixes. Please see prefixes.yml for an example
+# Below the files containing the monitored prefixes. Please see prefixes.yml for an example.
 # This is an array (use new lines and dashes!)
 monitoredPrefixesFiles:
   - prefixes.yml
@@ -101,7 +125,7 @@ logging:
   directory: logs
   logRotatePattern: YYYY-MM-DD  # Whenever the pattern changes, a new file is created and the old one rotated
   zippedArchive: true
-  maxSize: 20m
+  maxSize: 80m
   maxFiles: 14d
 
-checkForUpdatesAtBoot: true
+checkForUpdatesAtBoot: true

docs/configuration.md

@@ -7,9 +7,8 @@ The following are common parameters which it is possible to specify in the confi
 | Parameter | Description  | Expected format | Example  |  Required |
 |---|---|---|---|---|
 |environment| You can specify various environments. The values "production" (not verbose) and "development" (verbose) will affect the verbosity of the error/debug logs. Other values don't affect the functionalities, they will be used to identify from which environment the log is coming from. | A string | production | Yes |
-|notificationIntervalSeconds| The amount of seconds before the same alert can be repeated. An alert is repeated only if the cause of it has not being solved. | An integer | 1800 | Yes |
-|clearNotificationQueueAfterSeconds| If the cause of an alert is resolved, then stop waiting for more information about the issue. | An integer (greater than notificationIntervalSeconds) | 1900 |  Yes |
-|checkStaleNotificationsSeconds| The amount of seconds between a check on stale alerts. A stale alert happens when the cause of an alert is resolved before the next notification round, in such a case send it anyway. | An integer | 60 | Yes |
+|notificationIntervalSeconds|Defines the amount of seconds after which an alert can be repeated. An alert is repeated only if the event that triggered it is not yet solved. Please, don't set this value to Infinity, use instead alertOnlyOnce. | An integer | 1800 | Yes |
+|alertOnlyOnce| A boolean that, if set to true, will prevent repetitions of the same alert even if the event that triggered it is not yet solved. In this case notificationIntervalSeconds will be ignored. If set to true, the signature of all alerts will be cached in order to recognize if they already happened in the past. This may lead to a memory leak if the amount of alerts is considerable. | A boolean | false | No |
 |monitoredPrefixesFiles| The [list](docs/prefixes.md#array) of files containing the prefixes to monitor. See [here](docs/prefixes.md#prefixes) for more informations. | A list of strings (valid .yml files) | -prefixes.yml | Yes |
 |logging| A dictionary of parameters containing the configuration for the file logging. | || Yes|
 |logging.directory| The directory where the log files will be generated. The directory will be created if not existent. | A string | logs | Yes |
@@ -159,6 +158,7 @@ Parameters for this report module:
 
 |Parameter| Description| 
 |---|---|
+|showPaths| Amount of AS_PATHs to report in the alert (0 to disable). | 
 |senderEmail| The email address that will be used as sender for the alerts. | 
 |smtp| A dictionary containing the SMTP configuration. Some parameters are described in `config.yml.example`. For all the options refer to the [nodemailer documentation](https://nodemailer.com/smtp/). | 
 |notifiedEmails| A dictionary containing email addresses grouped by user groups.  (key: group, value: list of emails)| 
@@ -179,3 +179,14 @@ Parameters for this report module:
 |hooks.default| The default user group. Each user group is a WebHook (url). | 
 
 
+#### reportKafka
+
+This report sends the alerts (including the BGP messages triggering them) to Kafka. By default it creates a topic `bgpalerter`.
+
+Parameters for this report module:
+
+|Parameter| Description| 
+|---|---|
+|host| Host and port of the Kafka instance/broker (e.g. localhost:9092).| 
+|topics| A dictionary containing a mapping from BGPalerter channels to Kafka topics (e.g. `hijack: hijack-topic`). By default all channels are sent to the topic `bgpalerter` (`default: bgpalerter`) |
+

docs/develop.md

@@ -9,10 +9,6 @@ To start development see how to install the source [here](installation.md#runnin
 
 * `npm run serve` to run the application from the source
 
-* `npm run start|stop|restart` to run the application as a service in background
-
-* `npm run status` to see the status of the service
-
 * `npm run test` to run the tests
 
 * `npm run build` to compile and build OS native applications

docs/prefixes.md

@@ -14,10 +14,12 @@ Below the list of possible parameters. **Remember to prepend them with a `--` in
 
 | Parameter | Description  | Expected format | Example  |  Required |
 |---|---|---|---|---|
-| -a  | The AS number(s) you want to generate the list for  | A comma-separated list of integers  | 2914,3333  | Yes |
-| -o  | The YAML output file | A string ending in ".yml" | prefixes.yml | Yes
+| -o  | The YAML output file | A string ending in ".yml" | prefixes.yml | Yes |
+| -a  | The AS number(s) you want to generate the list for  | A comma-separated list of integers  | 2914,3333  | No (one among -a, -p, -pf is required) |
 | -e  | Prefixes to exclude from the list | A comma-separated list of prefixes | 165.254.255.0/24,192.147.168.0/24 | No |
 | -i  | Avoid monitoring delegated prefixes. If a more specific prefix is found and it results announced by an AS different from the one declared in -a, then set `ignore: true` and `ignoreMorespecifics: true` | Nothing | | No
+| -p  | Prefixes for which the list will be generated | A comma-separated list of prefixes | 165.254.255.0/24,192.147.168.0/24 | No (one among -a, -p, -pf is required) |
+| -pf  | A file containing the prefixes for which the list will be generated | A text file having a prefix for each line | prefixes.txt | No (one among -a, -p, -pf is required) |
 
 
 ## <a name="prefixes-fields"></a>Prefixes list fields

env.js

@@ -106,15 +106,14 @@ let config = {
             channels: ["hijack", "newprefix", "visibility", "path"]
         }
     ],
-    checkStaleNotificationsSeconds : 60,
-    notificationIntervalSeconds: 1800,
-    clearNotificationQueueAfterSeconds: 1900,
+    notificationIntervalSeconds: 7200,
+    alarmOnlyOnce: false,
     monitoredPrefixesFiles: ["prefixes.yml"],
     logging: {
         directory: "logs",
         logRotatePattern: "YYYY-MM-DD",
         zippedArchive: true,
-        maxSize: "20m",
+        maxSize: "80m",
         maxFiles: "14d",
     },
     checkForUpdatesAtBoot: true

generatePrefixesList.js

@@ -4,20 +4,22 @@ import yaml from "js-yaml";
 import fs from "fs";
 const batchPromises = require('batch-promises');
 
-module.exports = function generatePrefixes(asns, outputFile, exclude, excludeDelegated) {
+module.exports = function generatePrefixes(asnList, outputFile, exclude, excludeDelegated, prefixes) {
     const generateList = {};
     let someNotValidatedPrefixes = false;
 
-    if (!asns) {
-        throw new Error("One or more comma-separated AS numbers have to be specified");
+    if (!asnList && !prefixes) {
+        throw new Error("You need to specify at least an AS number or a list of prefixes.");
+    }
+
+    if (asnList && prefixes) {
+        throw new Error("You can specify an AS number or a list of prefixes, not both.");
     }
 
     if (!outputFile) {
         throw new Error("Output file not specified");
     }
 
-    const asnList = asns.split(",");
-
     const getMultipleOrigins = (prefix) => {
         const url = brembo.build("https://stat.ripe.net", {
             path: ["data", "prefix-overview", "data.json"],
@@ -73,7 +75,6 @@ module.exports = function generatePrefixes(asns, outputFile, exclude, excludeDel
 
     };
 
-
     const generateRule = (prefix, asn, ignoreMorespecifics, description, excludeDelegated) =>
         getMultipleOrigins(prefix)
             .then(asns => {
@@ -152,8 +153,17 @@ module.exports = function generatePrefixes(asns, outputFile, exclude, excludeDel
             })
     };
 
-    return Promise
-        .all(asnList.map(getAnnouncedPrefixes))
+    const getBaseRules = () => {
+        if (prefixes) {
+            return Promise
+                .all(prefixes.map(p => generateRule(p, null, false, null, false)))
+                .then(() => prefixes);
+        } else {
+            return Promise.all(asnList.map(getAnnouncedPrefixes));
+        }
+    };
+
+    return getBaseRules()
         .then(items => [].concat.apply([], items))
         .then(prefixes => {
             return batchPromises(20, prefixes, prefix => {
@@ -179,7 +189,7 @@ module.exports = function generatePrefixes(asns, outputFile, exclude, excludeDel
             fs.writeFileSync(outputFile, yamlContent);
 
             if (someNotValidatedPrefixes) {
-                console.log("WARNING: The generated configuration is a snapshot of what is currently announced by " + asns + " but some of the prefixes don't have ROA objects associated. Please, verify the config file by hand!");
+                console.log("WARNING: The generated configuration is a snapshot of what is currently announced. Some of the prefixes don't have ROA objects associated or are RPKI invalid. Please, verify the config file by hand!");
             }
             console.log("Done!");
         })

index.js

@@ -53,11 +53,19 @@ const params = yargs
             .nargs('e', 1)
             .describe('e', 'Prefixes to exclude')
 
+            .alias('p', 'prefixes')
+            .nargs('p', 1)
+            .describe('p', 'Prefixes to include')
+
+            .alias('pf', 'prefixes-file')
+            .nargs('pf', 1)
+            .describe('pf', 'File containing the prefixes to include')
+
             .alias('i', 'ignore-delegated')
             .nargs('i', 0)
             .describe('i', 'Ignore delegated prefixes')
 
-            .demandOption(['o', 'a'])
+            .demandOption(['o']);
     })
     .example('$0 generate -a 2914 -o prefixes.yml', 'Generate prefixes for AS2914')
 
@@ -70,12 +78,28 @@ const params = yargs
 switch(params._[0]) {
     case "generate":
         const generatePrefixes = require("./generatePrefixesList");
+        let prefixes = null;
+        if (params.p && params.pf) {
+            throw new Error("The argument -p is not compatible with the argument -pf");
+        } else if (params.p) {
+            prefixes = params.p.split(",");
+        } else if (params.pf) {
+            const fs = require("fs");
+            if (fs.existsSync(params.pf)) {
+                prefixes = fs.readFileSync(params.pf, 'utf8').split(/\r?\n/).filter(i => i && true);
+            } else {
+                throw new Error("The prefix list file (-pf) is not readable");
+            }
+        }
+
         generatePrefixes(
-            params.a.toString(),
+            (params.a) ? params.a.toString().split(",") : null,
             params.o,
             (params.e || "").split(","),
-            params.i || false
+            params.i || false,
+            prefixes
         );
+
         break;
 
     default: // Run monitor

monitors/monitor.js

@@ -44,8 +44,14 @@ export default class Monitor {
         this.monitored = [];
         this.alerts = {};
         this.sent = {};
+
+        this.internalConfig = {
+            notificationIntervalSeconds: this.config.notificationIntervalSeconds,
+            checkStaleNotificationsSeconds: 60,
+            clearNotificationQueueAfterSeconds: (this.config.notificationIntervalSeconds * 3) / 2
+        };
         this.updateMonitoredPrefixes();
-        setInterval(this._publish, this.config.checkStaleNotificationsSeconds * 1000)
+        setInterval(this._publish, this.internalConfig.checkStaleNotificationsSeconds * 1000);
     };
 
     updateMonitoredPrefixes = () => {
@@ -128,8 +134,9 @@ export default class Monitor {
     };
 
     _clean = (group) => {
-
-        if (new Date().getTime() > group.latest + (this.config.clearNotificationQueueAfterSeconds * 1000)) {
+        if (this.config.alertOnlyOnce) {
+            delete this.alerts[group.id];
+        } else if (this.config.alertOnlyOnce && new Date().getTime() > group.latest + (this.internalConfig.clearNotificationQueueAfterSeconds * 1000)) {
             delete this.alerts[group.id];
             delete this.sent[group.id];
 
@@ -142,10 +149,12 @@ export default class Monitor {
     _checkLastSent = (group) => {
         const lastTimeSent = this.sent[group.id];
 
-        if (lastTimeSent) {
+        if (lastTimeSent && this.config.alertOnlyOnce) {
+            return false;
+        } else if (lastTimeSent) {
 
             const isThereSomethingNew = lastTimeSent < group.latest;
-            const isItTimeToSend = new Date().getTime() > lastTimeSent + (this.config.notificationIntervalSeconds * 1000);
+            const isItTimeToSend = new Date().getTime() > lastTimeSent + (this.internalConfig.notificationIntervalSeconds * 1000);
 
             return isThereSomethingNew && isItTimeToSend;
         } else {