Merge branch 'release/2.3.0'

.travis.yml

@@ -23,20 +23,23 @@ before_install:
         build-essential
         cmake
         libfuzzy-dev
+        p7zip-full
         unrar
+        unzip
     - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
 
-    - git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH
 
     # Build latest images spamscope-root, spamscope-elasticsearch
     # make images
     - if [ "$TRAVIS_BRANCH" == "master" ]; then
+        git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH;
         cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=master -t $DOCKER_USERNAME/spamscope-elasticsearch . && cd -;
         docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch /bin/sh -c 'python -m unittest discover -s tests -f -v';
         docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v';
       fi
 
     - if [ "$TRAVIS_BRANCH" == "develop" ]; then
+        git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH;
         cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=develop -t $DOCKER_USERNAME/spamscope-elasticsearch:develop . && cd -;
         docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch:develop /bin/sh -c 'python -m unittest discover -s tests -f -v';
         docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch:develop /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v';

conf/spamscope.example.yml

@@ -107,11 +107,22 @@ raw_mail:
 
 # Attachments bolt configuration
 attachments:
-    content_types_blacklist:
-        # All content types to remove from results
-        # Example in content_types/blacklist/generic.example.yml
-        generic: /path/to/generic_content_types
-        custom: /path/to/custom_content_types
+    # The lists of all components must be under lists keyword to load them
+    # automatically
+    commons:
+        lists:
+            blacklist_content_types:
+                # All content types to remove from results
+                # Example in content_types/blacklist/generic.example.yml
+                generic: /path/to/generic_content_types
+                custom: /path/to/custom_content_types
+
+            not_extract_content_types:
+                # All content types that you don't want extract from archive
+                # Example: application/java-archive (jar), you can save the jar
+                # but do not extract the class inside.
+                generic: /path/to/generic_content_types
+                custom: /path/to/custom_content_types
 
     # Apache Tika analysis: https://tika.apache.org/
     tika:
@@ -125,9 +136,10 @@ attachments:
 
         # All content types to extract details
         # Example in content_types/tika/generic.example.yml
-        valid_content_types:
-            generic: /path/to/generic_content_types
-            custom: /path/to/custom_content_types
+        lists:
+            whitelist_content_types:
+                generic: /path/to/generic_content_types
+                custom: /path/to/custom_content_types
 
     # VirusTotal analysis: https://www.virustotal.com/
     virustotal:
@@ -138,9 +150,10 @@ attachments:
         # All content types to analyze with virustotal
         # Example in content_types/virustotal/generic.example.yml
         # Now is not active
-        content_types_details:
-            generic: /path/to/generic_content_types
-            custom: /path/to/custom_content_types
+        lists:
+            whitelist_content_types:
+                generic: /path/to/generic_content_types
+                custom: /path/to/custom_content_types
 
     # Thug analysis: https://github.com/buffer/thug
     thug:
@@ -188,6 +201,12 @@ attachments:
         ApiKey: xxxxx
         useragent: SpamScope
 
+    # This plugin store the samples on file system
+    # in date format subfolders (YYYY-MM-DD)
+    store_samples:
+        enabled: false
+        base_path: /tmp
+
 
 # Urls
 urls:

conf/templates/spamscope_attachments.json

@@ -0,0 +1,81 @@
+{
+  "order": 0,
+  "version": 1,
+  "index_patterns": "spamscope_attachments-*",
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "header": {
+          "tokenizer": "uax_url_email",
+          "filter": [
+            "lowercase"
+          ]
+        },
+        "html_body": {
+          "char_filter": [
+            "html_strip"
+          ],
+          "tokenizer": "uax_url_email",
+          "filter": [
+            "lowercase"
+          ]
+        },
+        "path_pattern": {
+          "tokenizer": "path_hierarchy",
+          "filter": [
+            "lowercase"
+          ]
+        }
+      }
+    },
+    "index.codec": "best_compression",
+    "index.number_of_shards": 1,
+    "index.number_of_replicas": 0,
+    "index.refresh_interval": "5s",
+    "index.mapping.ignore_malformed": true
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic_templates": [
+        {
+          "hashes": {
+            "mapping": {
+              "type": "keyword",
+              "eager_global_ordinals": true
+            },
+            "match_pattern": "regex",
+            "match": "(^|.*\\.)(md5|sha1|sha256|sha512|ssdeep|extension)$"
+          }
+        },
+        {
+          "payload": {
+            "mapping": {
+              "type": "binary",
+              "norms": false
+            },
+            "match_pattern": "regex",
+            "match": "(^|.*\\.)(payload)$"
+          }
+        },
+        {
+          "structured": {
+            "mapping": {
+              "type": "keyword",
+              "eager_global_ordinals": true
+            },
+            "match_pattern": "regex",
+            "match": "(^|.*\\.)(charset|mail_server|mailbox|message_id|Content-Type|content_transfer_encoding|mail_content_type)$"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}

conf/templates/spamscope.json → conf/templates/spamscope_mails.json

@@ -1,7 +1,7 @@
 {
   "order": 0,
-  "version": 5,
-  "index_patterns": "spamscope_*-*",
+  "version": 6,
+  "index_patterns": "spamscope_mails-*",
   "settings": {
     "analysis": {
       "analyzer": {
@@ -29,18 +29,19 @@
       }
     },
     "index.codec": "best_compression",
-    "index.number_of_shards": 2,
+    "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.refresh_interval": "5s",
     "index.mapping.ignore_malformed": true
   },
   "mappings": {
-    "analysis": {
+    "_doc": {
       "dynamic_templates": [
         {
           "cert_serial": {
             "mapping": {
-              "type": "text"
+              "type": "text",
+              "norms": false
             },
             "path_match": "network.shodan.data.ssl.cert.serial"
           }
@@ -58,7 +59,8 @@
         {
           "payload": {
             "mapping": {
-              "type": "binary"
+              "type": "binary",
+              "norms": false
             },
             "match_pattern": "regex",
             "match": "(^|.*\\.)(payload)$"
@@ -71,7 +73,7 @@
               "analyzer": "header"
             },
             "match_pattern": "regex",
-            "match": "^(from|to|headers)$"
+            "match": "^(from|to|headers|bcc|cc|delivered_to|reply_to)$"
           }
         },
         {
@@ -103,28 +105,13 @@
           }
         },
         {
-          "all_not_analyzed": {
+          "structured": {
             "mapping": {
               "type": "keyword",
               "eager_global_ordinals": true
             },
             "match_pattern": "regex",
-            "match": "(^|.*\\.)(charset|mail_server|mailbox|message_id|Content-Type|content_transfer_encoding|mail_content_type)$"
-          }
-        },
-        {
-          "all_string": {
-            "mapping": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "ignore_above": 256,
-                  "type": "keyword",
-                  "eager_global_ordinals": true
-                }
-              }
-            },
-            "match_mapping_type": "string"
+            "match": "(^|.*\\.)(charset|mail_server|mailbox|message_id|Content-Type|content_transfer_encoding|mail_content_type|to_domains)$"
           }
         }
       ],

config.json

@@ -11,9 +11,9 @@
                 "localhost"
             ],
             "log": {
-                "path": "/var/log/spamscope",
+                "path": "/logs",
                 "max_bytes": 5000000,
-                "backup_count": 3,
+                "backup_count": 1,
                 "level": "info"
             },
             "use_virtualenv": false,

project.clj

@@ -1,4 +1,4 @@
-(defproject spamscope "2.2.1-SNAPSHOT"
+(defproject spamscope "2.3.0-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"

src/bolts/attachments.py

@@ -34,28 +34,20 @@ def initialize(self, stormconf, context):
 
     def _load_settings(self):
         # Loading configuration
-        self._load_lists()
-
-        settings = copy.deepcopy(self.conf)
-        settings.update(
-            {"filter_cont_types": self._filter_cont_types,
-             "tika_whitelist_cont_types": self._tika_whitelist_cont_types})
-
+        settings = self._load_lists()
         self.attach.reload(**settings)
 
     def _load_lists(self):
+        settings = copy.deepcopy(self.conf)
 
-        # Load content types to filter
-        self._filter_cont_types = load_keywords_list(
-            self.conf.get("content_types_blacklist", {}), lower=False)
-        self.log("Content types to filter reloaded", "debug")
-
-        # Load Tika content types to analyze
-        self._tika_whitelist_cont_types = set()
-        if self.conf["tika"]["enabled"]:
-            self._tika_whitelist_cont_types = load_keywords_list(
-                self.conf["tika"].get("valid_content_types", {}), lower=False)
-            self.log("Whitelist Tika content types reloaded", "debug")
+        for k in self.conf:
+            for i, j in self.conf[k].get("lists", {}).items():
+                settings[k][i] = load_keywords_list(j)
+                self.log("Loaded lists {!r} for {!r}".format(i, k), "debug")
+                self.log("Keys[{!r}][{!r}]: {}".format(
+                    k, i, ", ".join(settings[k][i])), "debug")
+        else:
+            return settings
 
     def process_tick(self, freq):
         """Every freq seconds you reload the keywords. """

src/cli/spamscope_elasticsearch.py

@@ -200,7 +200,8 @@ def update_nr_replicas(es, max_retry, nr_replicas, index):
         try:
             es.indices.put_settings(
                 body={"index": {"number_of_replicas": int(nr_replicas)}},
-                index=index)
+                index=index,
+                allow_no_indices=True)
             log.info("Updating replicas done")
             return
 

src/modules/attachments/__init__.py

@@ -16,4 +16,4 @@
 
 from .utils import *
 from .attachments import Attachments as MailAttachments
-from .post_processing import processors, tika, virustotal, thug, zemana
+from .post_processing import processors, tika, virustotal, thug, zemana, store_samples