Merge branch 'release/2.8.0'

Makefile

@@ -80,3 +80,12 @@ dist: clean ## builds source and wheel package
 
 install: clean ## install the package to the active Python's site-packages
 	python setup.py install
+
+debug-iter-topology:
+	mkdir /tmp/logs/ 2>/dev/null || echo /tmp/logs/ already exist
+	sparse run \
+		-n spamscope_debug_iter \
+		-e debug \
+		-o topology.max.spout.pending=1 \
+		-o "topology.sleep.spout.wait.strategy.time.ms=10" \
+		-o "topology.tick.tuple.freq.secs=10"

README.md

@@ -177,10 +177,13 @@ You can use:
   * [Ansible](./ansible/README.md): to install and run SpamScope on server
 
 # Topologies
-SpamScope comes with three topologies:
+SpamScope comes with six topologies:
    - [spamscope_debug](./topologies/spamscope_debug.py): the output are JSON files on file system.
    - [spamscope_elasticsearch](./topologies/spamscope_elasticsearch.py): the output are stored in Elasticsearch indexes.
    - [spamscope_redis](./topologies/spamscope_redis.py): the output are stored in Redis.
+   - [spamscope_debug_iter](./topologies/spamscope_debug_iter.py): It uses generator to send mails in topology. The output are JSON files on file system.
+   - [spamscope_elasticsearch_iter](./topologies/spamscope_elasticsearch_iter.py): It uses generator to send mails in topology. The output are stored in Elasticsearch indexes.
+   - [spamscope_redis_iter](./topologies/spamscope_redis_iter.py): It uses generator to send mails in topology. The output are stored in Redis.
 
 If you want submit SpamScope topology use `spamscope-topology submit` tool. For more details [see SpamScope cli tools](src/cli/README.md):
 

README.rst

@@ -291,13 +291,21 @@ run SpamScope on server
 Topologies
 ==========
 
-SpamScope comes with three topologies: -
+SpamScope comes with six topologies: -
 `spamscope_debug <./topologies/spamscope_debug.py>`__: the output are
 JSON files on file system. -
 `spamscope_elasticsearch <./topologies/spamscope_elasticsearch.py>`__:
 the output are stored in Elasticsearch indexes. -
 `spamscope_redis <./topologies/spamscope_redis.py>`__: the output are
-stored in Redis.
+stored in Redis. -
+`spamscope_debug_iter <./topologies/spamscope_debug_iter.py>`__: It uses
+generator to send mails in topology. The output are JSON files on file
+system. -
+`spamscope_elasticsearch_iter <./topologies/spamscope_elasticsearch_iter.py>`__:
+It uses generator to send mails in topology. The output are stored in
+Elasticsearch indexes. -
+`spamscope_redis_iter <./topologies/spamscope_redis_iter.py>`__: It uses
+generator to send mails in topology. The output are stored in Redis.
 
 If you want submit SpamScope topology use ``spamscope-topology submit``
 tool. For more details `see SpamScope cli tools <src/cli/README.md>`__:

conf/spamscope.example.yml

@@ -49,6 +49,50 @@ files-mails:
             path_mails: /path/mails2
 
 
+# Spout file on file system
+# Use an iterator. Safe for RAM
+iter-files-mails:
+
+    # The mails in processing older that fail.after.seconds will be failed
+    fail.after.seconds: 60
+
+    # Post processing
+    post_processing:
+
+        # move or remove mails analyzed, default remove
+        what: remove
+
+        # Where you want move the analyzed mails, default /tmp/moved
+        where: /tmp/moved
+
+        # Where you want move the failed mails, default /tmp/failed
+        where.failed: /tmp/failed
+
+    # Mailboxes
+    mailboxes:
+        test:
+            mail_server: hostname
+            # Trust string is used to get sender IP address from mail server.
+            # More details:
+            # https://github.com/SpamScope/mail-parser/blob/v0.4.6/mailparser/__init__.py#L221
+            trust_string: "test_trust_string"
+            files_pattern: "*untroubled*"
+            path_mails: /path/mails1
+
+            # This flag enables Outlook msg parsing for every mails in mailbox
+            # Default value is false
+            outlook: false
+
+            # List of others headers to get
+            headers:
+                - custom_one
+                - custom_two
+        test1:
+            mail_server: hostname
+            trust_string: "test1_trust_string"
+            files_pattern: "*"
+            path_mails: /path/mails2
+
 # Bolts configurations
 # Phishing bolt configuration
 phishing:

project.clj

@@ -1,4 +1,4 @@
-(defproject spamscope "2.7.0-SNAPSHOT"
+(defproject spamscope "2.8.0-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"

src/bolts/tokenizer.py

@@ -24,7 +24,7 @@
 import random
 import six
 from collections import deque
-from cPickle import BadPickleGet
+from cPickle import PickleError
 
 from streamparse import Stream
 import mailparser
@@ -85,7 +85,7 @@ def load_filters(self):
                 try:
                     obj = load_obj(path)
                     setattr(self, "analyzed_" + i, obj)
-                except (IOError, EOFError, ValueError, BadPickleGet):
+                except (IOError, EOFError, ValueError, PickleError):
                     setattr(self, "analyzed_" + i, deque(
                         maxlen=getattr(self, "maxlen_" + i)))
 

src/cli/spamscope_topology.py

@@ -57,8 +57,14 @@ def get_args():
     submit.add_argument(
         "-g",
         "--topology",
-        choices=["spamscope_debug", "spamscope_elasticsearch",
-                 "spamscope_redis"],
+        choices=[
+            "spamscope_debug_iter",
+            "spamscope_debug",
+            "spamscope_elasticsearch_iter",
+            "spamscope_elasticsearch",
+            "spamscope_redis_iter",
+            "spamscope_redis",
+        ],
         default="debug",
         help="SpamScope topology.",
         dest="topology")

src/modules/attachments/attachments.py

@@ -353,9 +353,11 @@ def withhashes(cls, attachments=[]):
                 try:
                     payload = base64.b64decode(i["payload"])
                 except TypeError, e:
-                    payload = base64.b64decode(i["payload"] + "===")
-                    i.setdefault("errors", []).append(repr(e))
-
+                    try:
+                        payload = base64.b64decode(i["payload"] + "===")
+                        i.setdefault("errors", []).append(repr(e))
+                    except TypeError:
+                        continue
             else:
                 payload = i["payload"]
 

src/modules/attachments/thug_analysis.py

@@ -58,8 +58,12 @@ def generate_json_report():
     if m is None:
         return
 
-    report = json.loads(m(tempfile.gettempdir()))
-    return report
+    try:
+        report = json.loads(m(tempfile.gettempdir()))
+    except TypeError:
+        return
+    else:
+        return report
 
 
 class CustomWatchdog(Watchdog):

src/options.py

@@ -19,7 +19,7 @@
 
 from os.path import join
 
-__version__ = "2.7.0"
+__version__ = "2.8.0"
 __configuration_path__ = "/etc/spamscope"
 
 __defaults__ = {

src/spouts/README.md

@@ -0,0 +1,13 @@
+# Overview
+In this folder there are all SpamScope `spouts`.
+
+# How add a new spout
+These are the steps to add a new `spout` to Spamscope:
+
+ - add a new module in [spouts](./) folder. This module should implement a new class that has `AbstractSpout` as base.
+
+ - import the new class in [\_\_init\_\_.py](./__init__.py)
+
+ - add the new section in [main configuration file](../../conf/spamscope.example.yml). The name of this section will be used in topology file
+
+ - add the new spout in [topology](../../topologies) 

src/spouts/__init__.py

@@ -15,3 +15,4 @@
 """
 
 from .files_mails import FilesMailSpout
+from .iter_files_mails import IterFilesMailSpout

src/spouts/files_mails.py

@@ -128,17 +128,21 @@ def next_tuple(self):
             self.log("EMITTED - {!r}".format(mail_string))
 
             processing = mail.filename + ".processing"
-            shutil.move(mail.filename, processing)
-
-            self.emit([
-                processing,  # 0
-                mail.mail_server,  # 1
-                mail.mailbox,  # 2
-                mail.priority,  # 3
-                mail.trust,  # 4
-                mail.mail_type,  # 5
-                mail.headers],  # 6
-                tup_id=mail.filename)
+
+            try:
+                shutil.move(mail.filename, processing)
+            except IOError:
+                self.log("ALREADY EMITTED - {!r}".format(mail_string))
+            else:
+                self.emit([
+                    processing,  # 0
+                    mail.mail_server,  # 1
+                    mail.mailbox,  # 2
+                    mail.priority,  # 3
+                    mail.trust,  # 4
+                    mail.mail_type,  # 5
+                    mail.headers],  # 6
+                    tup_id=mail.filename)
 
     def ack(self, tup_id):
         """Acknowledge tup_id, that is the path_mail. """