Merge branch 'release/1.4.6'

project.clj

@@ -1,4 +1,4 @@
-(defproject spamscope "1.4.5-SNAPSHOT"
+(defproject spamscope "1.4.6-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"

requirements.txt

@@ -2,7 +2,7 @@ PyYAML==3.12
 backports.functools-lru-cache==1.3
 chainmap==1.0.2
 elasticsearch==5.2.0
-mail-parser==1.1.6
+mail-parser==1.1.9
 patool==1.12
 pyparsing==2.2.0
 python-magic==0.4.12
@@ -11,5 +11,5 @@ simplejson==3.10.0
 six==1.10.0
 ssdeep==3.2
 streamparse==3.4.0
-tika-app==1.0.1
+tika-app==1.1.0
 virustotal-api==1.1.2

src/cli/spamscope_topology.py

@@ -98,7 +98,7 @@ def get_args():
     submit.add_argument(
         "-t",
         "--timeout",
-        default=600,
+        default=720,
         type=int,
         help=("How long (in s) between heartbeats until supervisor considers "
               "that worker dead."),

src/modules/attachments/post_processing.py

@@ -85,9 +85,16 @@ def tika(conf, attachments):
 
         for a in attachments:
             if not a.get("is_filtered", False):
+
                 if a["Content-Type"] in conf["whitelist_cont_types"]:
+                    payload = a["payload"]
+
+                    if a["content_transfer_encoding"] != "base64":
+                        payload = payload.encode("base64")
+
+                    # tika-app only gets payload in base64
                     a["tika"] = tika.extract_all_content(
-                        payload=a["payload"], convert_to_obj=True)
+                        payload=payload, convert_to_obj=True)
 
 
 @register(active=True)

src/modules/utils.py

@@ -80,11 +80,13 @@ def write_payload(payload, extension, content_transfer_encoding="base64"):
     """
 
     temp = tempfile.mkstemp()[1] + extension
+    write_type = "w"
 
-    with open(temp, 'wb') as f:
-        if content_transfer_encoding == "base64":
-            payload = payload.decode("base64")
+    if content_transfer_encoding == "base64":
+        payload = payload.decode("base64")
+        write_type = "wb"
 
+    with open(temp, write_type) as f:
         f.write(payload)
 
     return temp

src/options.py

@@ -19,7 +19,7 @@
 
 from os.path import join
 
-__version__ = "1.4.5"
+__version__ = "1.4.6"
 __configuration_path__ = "/etc/spamscope"
 
 __defaults__ = {

topologies/spamscope_elasticsearch.py

@@ -21,7 +21,7 @@ class OutputElasticsearchTopology(Topology):
     attachments = Attachments.spec(
         name="attachments",
         inputs={tokenizer['attachments']: Grouping.fields('sha256_random')},
-        par=1)
+        par=4)
 
     urls_body = UrlsHandlerBody.spec(
         name="urls-handler-body",

topologies/spamscope_redis.py

@@ -21,7 +21,7 @@ class OutputRedisTopology(Topology):
     attachments = Attachments.spec(
         name="attachments",
         inputs={tokenizer['attachments']: Grouping.fields('sha256_random')},
-        par=2)
+        par=4)
 
     urls_body = UrlsHandlerBody.spec(
         name="urls-handler-body",

topologies/spamscope_testing.py

@@ -21,7 +21,7 @@ class OutputTestingTopology(Topology):
     attachments = Attachments.spec(
         name="attachments",
         inputs={tokenizer['attachments']: Grouping.fields('sha256_random')},
-        par=1)
+        par=4)
 
     urls_body = UrlsHandlerBody.spec(
         name="urls-handler-body",