Merge branch 'release/2.5.0'

.travis.yml

@@ -84,12 +84,12 @@ after_success:
     - coveralls
 
     - if [ "$TRAVIS_BRANCH" == "master" ]; then
-        docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
+        echo $DOCKER_PASSWORD | docker login -u="$DOCKER_USERNAME" --password-stdin;
         docker push $DOCKER_USERNAME/spamscope-elasticsearch;
       fi
 
     - if [ "$TRAVIS_BRANCH" == "develop" ]; then
-        docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
+        echo $DOCKER_PASSWORD | docker login -u="$DOCKER_USERNAME" --password-stdin;
         docker push $DOCKER_USERNAME/spamscope-elasticsearch:develop;
       fi
 

README.rst

@@ -17,8 +17,8 @@ Concepts <http://storm.apache.org/releases/1.2.1/Concepts.html>`__ -
 `Streamparse
 Quickstart <http://streamparse.readthedocs.io/en/stable/quickstart.html>`__
 
-In general the first step is start Apache Storm, then you can run the
-topologies. SpamScope has some topologies in `topologies
+In general the first step is run Apache Storm, then you can run the
+topologies on it. SpamScope has some topologies in `topologies
 folder <./topologies/>`__, but you can make others topologies.
 
 .. figure:: docs/images/schema_topology.png?raw=true
@@ -117,15 +117,15 @@ Mantuano <https://www.linkedin.com/in/fmantuano/>`__)
 Requirements
 ============
 
-For operating system requirements you can read ``Ansible`` playbook,
-that goes into details.
+For operating system requirements you can read `Ansible
+playbooks <./ansible>`__, that go into details.
 
 For Python requirements you can read: \* `mandatory
 requirements <./requirements.txt>`__ \* `optional
 requirements <./requirements_optional.txt>`__
 
-*Thug* is another optional requirement. See Thug section for more
-details.
+*Thug* is another optional requirement, that it's not in requirements.
+See `Thug section <#thug-optional>`__ for more details.
 
 Apache Storm
 ------------
@@ -181,7 +181,8 @@ From release v1.3 SpamScope can analyze Javascript and HTML attachments
 with `Thug <https://github.com/buffer/thug>`__. If you want to analyze
 the attachments with Thug, follow `these
 instructions <http://buffer.github.io/thug/doc/build.html>`__ to install
-it. Enable it in ``attachments`` section.
+it. Enable it in ``attachments`` section of `main configuration
+file <./conf/spamscope.example.yml>`__.
 
 What is Thug? From README project: > Thug is a Python low-interaction
 honeyclient aimed at mimicing the behavior of a web browser in order to
@@ -190,8 +191,9 @@ detect and emulate malicious contents.
 You can see a complete SpamScope report with Thug analysis
 `here <https://goo.gl/Y4kWCv>`__.
 
-Thug analysis can be very slow and you can have ``heartbeat timeout`` in
-Apache Storm. To avoid any issue set ``supervisor.worker.timeout.secs``:
+Thug analysis can be very slow and you can have ``heartbeat timeout``
+errors in Apache Storm. To avoid any issue set
+``supervisor.worker.timeout.secs``:
 
 ::
 
@@ -226,8 +228,9 @@ install ``redis`` package.
 Configuration
 =============
 
-Read the `example configuration file <./conf/spamscope.example.yml>`__.
-The default value where SpamScope will search the configuration file is
+Read the `example of main configuration
+file <./conf/spamscope.example.yml>`__. The default value where
+SpamScope will search the configuration file is
 ``/etc/spamscope/spamscope.yml``, but it's possible to set the
 environment variable ``SPAMSCOPE_CONF_FILE``:
 

ansible/02_submit_spamscope_debug.yml

@@ -7,12 +7,12 @@
 
   tasks:
     - name: Submit spamscope_debug topology
-      shell: "spamscope-topology submit -g spamscope_debug -p 100 -t 30 -e prod_vm"
+      shell: "spamscope-topology submit -g spamscope_debug -p 1 -t 20 -e prod_vm"
       args:
         chdir: "{{ spamscope_path }}"
       tags:
         - submit
       when: spamscope_debug_submit == "true"
       environment:
-          PATH: "{{ spamscope_path }}/venv/bin:{{ install_path }}/storm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-          LEIN_ROOT: "yes"
+        PATH: "{{ spamscope_path }}/venv/bin:{{ install_path }}/storm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+        LEIN_ROOT: "yes"

conf/spamscope.example.yml

@@ -6,17 +6,20 @@ files-mails:
     # Reload new mails after reload.mails analyzed
     reload.mails: 1000
 
+    # The mails in processing older that fail.after.seconds will be failed
+    fail.after.seconds: 60
+
     # Post processing
     post_processing:
 
-        # move or remove mails?
+        # move or remove mails analyzed, default remove
         what: remove
 
-        # if move where
-        where: /mnt/testing/spamscope/mails/moved
+        # Where you want move the analyzed mails, default /tmp/moved
+        where: /tmp/moved
 
-        # if failed move in where.failed
-        where.failed: /mnt/testing/spamscope/mails/failed
+        # Where you want move the failed mails, default /tmp/failed
+        where.failed: /tmp/failed
 
     # Mailboxes
     mailboxes:
@@ -64,6 +67,9 @@ phishing:
 
 
 tokenizer:
+    # Persistent where store dumps of hashes.
+    persistent_path: /tmp
+
     # If true mails with same hash are filtered and not analyzed.
     # Only the body will not saved
     filter_mails: true
@@ -110,6 +116,11 @@ attachments:
     # The lists of all components must be under lists keyword to load them
     # automatically
     commons:
+        # enable or disable filter on size
+        size.filter.enabled: false
+        # max size to analyze in bytes
+        max.size: 3145728
+
         lists:
             blacklist_content_types:
                 # All content types to remove from results

conf/templates/commons.json

@@ -4,6 +4,8 @@
   "settings": {
     "index.codec": "best_compression",
     "index.number_of_replicas": 0,
-    "index.refresh_interval": "5s"
+    "index.refresh_interval": "5s",
+    "index.mapping.total_fields.limit": 100000,
+    "index.mapping.ignore_malformed": true
   }
 }

conf/templates/spamscope_attachments.json

@@ -1,6 +1,6 @@
 {
   "order": 0,
-  "version": 1,
+  "version": 2,
   "index_patterns": "spamscope_attachments-*",
   "settings": {
     "analysis": {
@@ -57,6 +57,15 @@
             "match": "(^|.*\\.)(payload)$"
           }
         },
+        {
+          "number": {
+            "mapping": {
+              "type": "long"
+            },
+            "match_pattern": "regex",
+            "match": "(^|.*\\.)(size)$"
+          }
+        },
         {
           "structured": {
             "mapping": {

conf/templates/spamscope_mails.json

@@ -1,6 +1,6 @@
 {
   "order": 0,
-  "version": 6,
+  "version": 8,
   "index_patterns": "spamscope_mails-*",
   "settings": {
     "analysis": {
@@ -38,14 +38,23 @@
     "_doc": {
       "dynamic_templates": [
         {
-          "cert_serial": {
+          "shodan_cert_serial": {
             "mapping": {
               "type": "text",
               "norms": false
             },
             "path_match": "network.shodan.data.ssl.cert.serial"
           }
         },
+        {
+          "shodan_vulns": {
+            "mapping": {
+              "type": "text",
+              "norms": false
+            },
+            "path_match": "network.shodan.data.vulns.*.cvss"
+          }
+        },
         {
           "hashes": {
             "mapping": {
@@ -104,6 +113,15 @@
             "match": "(^|.*\\.)(sender_ip|srcip|http_iv_remote_address|clientip|syslog_host|ip_str)$"
           }
         },
+        {
+          "number": {
+            "mapping": {
+              "type": "long"
+            },
+            "match_pattern": "regex",
+            "match": "(^|.*\\.)(size)$"
+          }
+        },
         {
           "structured": {
             "mapping": {
@@ -126,13 +144,13 @@
               "type": "ip"
             },
             "latitude": {
-              "type": "half_float"
+              "type": "float"
             },
             "location": {
               "type": "geo_point"
             },
             "longitude": {
-              "type": "half_float"
+              "type": "float"
             }
           }
         },

project.clj

@@ -1,4 +1,4 @@
-(defproject spamscope "2.4.0-SNAPSHOT"
+(defproject spamscope "2.5.0-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"

requirements.txt

@@ -3,7 +3,7 @@ astropy>=1.3.3
 backports.functools-lru-cache>=1.3
 chainmap
 lxml
-mail-parser>=3.2.6
+mail-parser>=3.4.1
 patool
 pyparsing
 python-magic

setup.py

@@ -19,6 +19,8 @@
 
 import os
 import runpy
+import subprocess
+import sys
 from setuptools import setup, find_packages
 from setuptools.command.install import install as SetupToolsInstall
 
@@ -34,17 +36,17 @@
 with open(os.path.join(current, 'requirements.txt')) as f:
     requirements = f.read().splitlines()
 
-with open(os.path.join(current, 'requirements_editable.txt')) as f:
-    requirements_editable = f.read().splitlines()
-
 
 class Install(SetupToolsInstall):
     """Customized setuptools install command which uses pip. """
 
     def run(self, *args, **kwargs):
-        import pip
-        pip.main(["install", "-e"] + requirements_editable)
-        pip.main(['install'] + requirements)
+        subprocess.check_call([
+            sys.executable, '-m',
+            'pip', 'install', '-r', "requirements_editable.txt"])
+        subprocess.check_call([
+            sys.executable, '-m',
+            'pip', 'install', '-r', "requirements.txt"])
         SetupToolsInstall.run(self, *args, **kwargs)
 
 

src/bolts/attachments.py

@@ -57,6 +57,8 @@ def process_tick(self, freq):
     def process(self, tup):
         try:
             sha256_random = tup.values[0]
+            sha256 = sha256_random.split("_")[0]
+            self.log("Processing started: {}".format(sha256))
             with_attachments = tup.values[1]
 
             # Remove all values

src/bolts/network.py

@@ -36,6 +36,8 @@ class Network(AbstractBolt):
 
     def process(self, tup):
         sha256_random = tup.values[0]
+        sha256 = sha256_random.split("_")[0]
+        self.log("Processing started: {}".format(sha256))
         ipaddress = tup.values[1]
         is_filtered = tup.values[2]
 

src/bolts/output_debug.py

@@ -41,6 +41,8 @@ def initialize(self, stormconf, context):
 
     def process(self, tup):
         sha256_random = tup.values[0]
+        sha256 = sha256_random.split("_")[0]
+        self.log("Processing started: {}".format(sha256))
         mail = json.dumps(tup.values[1], ensure_ascii=False,
                           indent=self._json_indent)
 

src/bolts/output_elasticsearch.py

@@ -64,6 +64,9 @@ def flush(self):
         self._count = 1
 
     def process(self, tup):
+        sha256_random = tup.values[0]
+        sha256 = sha256_random.split("_")[0]
+        self.log("Processing started: {}".format(sha256))
         raw_mail = tup.values[1]
 
         # Convert back to object strings convert manually

src/bolts/output_redis.py

@@ -72,6 +72,9 @@ def flush_attachments(self):
         self._counter["attachments"] = 0
 
     def process(self, tup):
+        sha256_random = tup.values[0]
+        sha256 = sha256_random.split("_")[0]
+        self.log("Processing started: {}".format(sha256))
         raw_mail = tup.values[1]
 
         # Convert back to object strings converted manually

src/bolts/phishing.py

@@ -82,6 +82,8 @@ def process_tick(self, freq):
     def process(self, tup):
         bolt = tup.component
         sha256_random = tup.values[0]
+        sha256 = sha256_random.split("_")[0]
+        self.log("Processing started: {}".format(sha256))
         values = tup.values
 
         self._mails.setdefault(sha256_random, {})[bolt] = values

src/bolts/raw_mail.py

@@ -36,6 +36,8 @@ class RawMail(AbstractBolt):
 
     def process(self, tup):
         sha256_random = tup.values[0]
+        sha256 = sha256_random.split("_")[0]
+        self.log("Processing started: {}".format(sha256))
         raw_mail = tup.values[1]
         mail_type = tup.values[2]
         is_filtered = tup.values[3]