Merge branch 'release/1.5.0'

README.md

@@ -98,7 +98,10 @@ Thug is a Python low-interaction honeyclient aimed at mimicing the behavior of a
 You can see a complete SpamScope report with Thug analysis [here](https://goo.gl/Y4kWCv).
 
 ### VirusTotal (optional)
-It's possible add to results (for mail attachments) VirusTotal report. You need a private API key.
+It's possible add to results (for mail attachments and sender ip address) the VirusTotal report. You need a private API key.
+
+### Shodan (optional)
+It's possible add to results the Shodan report for sender ip address. You need a private API key.
 
 ### Elasticsearch (optional)
 It's possible to store the results in Elasticsearch. In this case you should install `elasticsearch` package.
@@ -205,6 +208,8 @@ $ export ZEMANA_ENABLED=True
 $ export ZEMANA_APIKEY="your key"
 $ export ZEMANA_PARTNERID="your partner id"
 $ export ZEMANA_USERID="your userid" 
+$ export SHODAN_ENABLED=True
+$ export SHODAN_APIKEY="your key"
 ```
 
 

conf/spamscope.example.yml

@@ -70,6 +70,23 @@ tokenizer:
     # Max number of hashes saved for filter function
     maxlen_attachments: 1000000
 
+    # If True the same ip address is filtered and not analyzed.
+    filter_network: True
+
+    # Max number of hashes saved for filter function
+    maxlen_network: 1000000
+
+
+# Network bolt configuration
+network:
+    shodan:
+        enabled: False
+        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+    virustotal:
+        enabled: False
+        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
 
 # Attachments bolt configuration
 attachments:

project.clj

@@ -1,10 +1,10 @@
-(defproject spamscope "1.4.10-SNAPSHOT"
+(defproject spamscope "1.5.0-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"
   :jvm-opts ["-client"]
-  :dependencies  [[org.apache.storm/storm-core "1.0.3"]
-                  [org.apache.storm/flux-core "1.0.3"]]
+  :dependencies  [[org.apache.storm/storm-core "1.1.0"]
+                  [org.apache.storm/flux-core "1.1.0"]]
   :jar-exclusions     [#"log4j\.properties" #"org\.apache\.storm\.(?!flux)" #"trident" #"META-INF" #"meta-inf" #"\.yaml"]
   :uberjar-exclusions [#"log4j\.properties" #"org\.apache\.storm\.(?!flux)" #"trident" #"META-INF" #"meta-inf" #"\.yaml"]
   )

requirements.txt

@@ -1,12 +1,13 @@
 PyYAML==3.12
 backports.functools-lru-cache==1.3
 chainmap==1.0.2
-elasticsearch==5.2.0
+elasticsearch==5.3.0
 mail-parser==1.1.10
 patool==1.12
 pyparsing==2.2.0
 python-magic==0.4.12
 redis==2.10.5
+shodan==1.6.5
 simplejson==3.10.0
 six==1.10.0
 ssdeep==3.2

src/bolts/__init__.py

@@ -24,3 +24,4 @@
 from .tokenizer import Tokenizer
 from .urls_handler_attachments import UrlsHandlerAttachments
 from .urls_handler_body import UrlsHandlerBody
+from .network import Network

src/bolts/json_maker.py

@@ -49,10 +49,14 @@ def _compose_output(self, greedy_data):
         mail["is_filtered"] = greedy_data["tokenizer"][2]
 
         # Attachments
-        mail["with_attachments"] = greedy_data["attachments"][1]
+        # with_raw_attachments: the mail has attachments
+        # with_attachments: the mail has not filtered attachments
+        mail["with_raw_attachments"] = greedy_data["attachments"][1]
+        attachments = greedy_data["attachments"][2]
 
-        if mail["with_attachments"]:
-            mail["attachments"] = greedy_data["attachments"][2]
+        if attachments:
+            mail["with_attachments"] = True
+            mail["attachments"] = attachments
 
         # Urls in attachments:
         # Add urls attachments because you can have more differents attachments
@@ -64,6 +68,11 @@ def _compose_output(self, greedy_data):
             urls = greedy_data["urls-handler-attachments"][2]
             mail["urls_attachments"] = reformat_urls(urls)
 
+        # Network
+        network = greedy_data["network"][1]
+        if network:
+            mail["network"] = network
+
         # Add intelligence output only if mail is not filtered
         if not mail["is_filtered"]:
 

src/bolts/network.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+from modules import AbstractBolt
+from modules.networks import processors
+
+
+class Network(AbstractBolt):
+    outputs = ['sha256_random', 'network', 'is_filtered']
+
+    def process(self, tup):
+        sha256_random = tup.values[0]
+        ipaddress = tup.values[1]
+        is_filtered = tup.values[2]
+
+        try:
+            results = {}
+
+            if not is_filtered and ipaddress:
+                for p in processors:
+                    try:
+                        p(self.conf[p.__name__], ipaddress, results)
+                    except KeyError:
+                        self.log("KeyError: {!r} doesn't exist in conf".format(
+                            p.__name__), "error")
+
+        except Exception as e:
+            self.log("Failed process network for mail: {}".format(
+                sha256_random), "error")
+            self.raise_exception(e, tup)
+
+        else:
+            self.emit([sha256_random, results])

src/bolts/phishing.py

@@ -62,12 +62,13 @@ def _load_lists(self):
         self.log("Phishing targets keywords reloaded")
 
     def _search_phishing(self, greedy_data):
+        with_urls = False
 
         # If mail is filtered don't check for phishing
         is_filtered = greedy_data["tokenizer"][2]
 
         if is_filtered:
-            return False, False
+            return False, False, False
 
         # Reset phishing bitmap
         self._pb.reset_score()
@@ -83,8 +84,9 @@ def _search_phishing(self, greedy_data):
         urls_body = greedy_data["urls-handler-body"][2]
         urls_attachments = greedy_data["urls-handler-attachments"][2]
 
-        # TODO: if an attachment is filtered the score is not complete
-        # more different mails can have same attachment
+        # TODO: if an attachment is filtered, the score is not complete
+        # more different mails can have the same attachment
+        # more different attachments can have the same mail
         attachments = MailAttachments(greedy_data["attachments"][2])
 
         urls = (
@@ -110,14 +112,15 @@ def _search_phishing(self, greedy_data):
         # Target not added because urls come already analyzed text
         for k, v in urls:
             if k:
+                with_urls = True
                 if any(check_urls(k, i) for i in self._t_keys.values()):
                     self._pb.set_property_score(v)
 
         # Check subject
         if swt(subject, self._s_keys):
             self._pb.set_property_score("mail_subject")
 
-        return self._pb.score, list(targets)
+        return self._pb.score, list(targets), with_urls
 
     def process_tick(self, freq):
         """Every freq seconds you reload the keywords. """
@@ -135,10 +138,15 @@ def process(self, tup):
         if not diff:
             with_phishing = False
 
-            score, targets = self._search_phishing(
+            score, targets, with_urls = self._search_phishing(
                 self._mails.pop(sha256_random))
 
-            if score:
+            # There is phishing only if there is also urls
+            # Mail can have target without phishing
+            if score and with_urls:
                 with_phishing = True
+            else:
+                with_phishing = False
+                score = 0
 
             self.emit([sha256_random, with_phishing, score, targets])

src/bolts/tokenizer.py

@@ -43,6 +43,8 @@ class Tokenizer(AbstractBolt):
     outputs = [
         Stream(fields=['sha256_random', 'mail', 'is_filtered'], name='mail'),
         Stream(fields=['sha256_random', 'body', 'is_filtered'], name='body'),
+        Stream(fields=['sha256_random', 'network', 'is_filtered'],
+               name='network'),
         Stream(fields=['sha256_random', 'with_attachments', 'attachments'],
                name='attachments')]
 
@@ -51,18 +53,24 @@ def initialize(self, stormconf, context):
 
         self._parser = MailParser()
         self._mails_analyzed = deque(maxlen=self.conf["maxlen_mails"])
+        self._network_analyzed = deque(maxlen=self.conf["maxlen_network"])
         self._attachments_analyzed = deque(
             maxlen=self.conf["maxlen_attachments"])
         self._load_filters()
 
     def _load_filters(self):
         self._filter_mails_enabled = self.conf["filter_mails"]
+        self._filter_network_enabled = self.conf["filter_network"]
         self._filter_attachments_enabled = self.conf["filter_attachments"]
 
     @property
     def filter_mails_enabled(self):
         return self._filter_mails_enabled
 
+    @property
+    def filter_network_enabled(self):
+        return self._filter_network_enabled
+
     @property
     def filter_attachments_enabled(self):
         return self._filter_attachments_enabled
@@ -131,6 +139,19 @@ def process(self, tup):
         attachments = []
         body = self.parser.body
 
+        # If filter network is enabled
+        is_filtered = False
+        if self.filter_network_enabled:
+            if mail["sender_ip"] in self._network_analyzed:
+                is_filtered = True
+
+            # Update databese mail analyzed
+            self._network_analyzed.append(mail["sender_ip"])
+
+        # Emit network
+        self.emit([sha256_rand, mail["sender_ip"], is_filtered],
+                  stream="network")
+
         # If filter mails is enabled
         is_filtered = False
         if self.filter_mails_enabled:

src/cli/spamscope_elasticsearch.py

@@ -163,8 +163,13 @@ def get_payload(client_host, index, hash_value, file_output):
         log.info("Filename: {!r}, Content-Type: {!r}, sha256: {!r}".format(
             r["filename"], r["Content-Type"], r["sha256"]))
 
+        try:
+            content_transfer_encoding = r["content_transfer_encoding"]
+        except KeyError:
+            # All archived files have base64 payload
+            content_transfer_encoding = "base64"
+
         payload = r["payload"]
-        content_transfer_encoding = r["content_transfer_encoding"]
         write_type = "w"
 
         if content_transfer_encoding == "base64":

src/modules/abstracts.py

@@ -89,10 +89,6 @@ class AbstractSpout(Spout, AbstractComponentMixin):
     def initialize(self, stormconf, context):
         self._conf_loader()
 
-    def process_tick(self, freq):
-        """Every freq seconds you reload configuration """
-        self._conf_loader()
-
 
 class AbstractUrlsHandlerBolt(AbstractBolt):
 

src/modules/attachments/attachments.py

@@ -155,6 +155,15 @@ def popcontenttype(self, content_type):
                     for j in inner_remove:
                         i["files"].remove(j)
 
+            # Patch
+            # If attach is filtered and content_type in whitelist
+            # you should remove sample from results.
+            # You can't use Content-Type because we don't have payload, so
+            # we use mail_content_type
+            elif (i.get("is_filtered") and
+                  i["mail_content_type"].lower() == content_type):
+                remove.append(i)
+
         else:
             # Remove
             for i in remove: