Merge branch 'release/1.5.3'

.travis.yml

@@ -1,33 +1,97 @@
 sudo: required
 
+services:
+    - docker
+
 language: python
 
 python:
     - "2.7"
 
 env:
-    - TIKA_APP_PATH=/tmp/tika-app-1.14.jar FAUP_PATH=/tmp/faup
+    - TIKA_APP_PATH=/tmp/tika-app-1.14.jar 
+      FAUP_PATH=/tmp/faup 
+      ZEMANA_PATH=/tmp/zemana 
+      DOCKER_ROOT_PATH=/tmp/docker-root 
+      DOCKER_ELASTICSEARCH_PATH=/tmp/docker-elasticsearch
 
-before_script:
-    - curl -o ${TIKA_APP_PATH} https://archive.apache.org/dist/tika/tika-app-1.14.jar
+branches:
+    only:
+        - master
+        - develop
 
 before_install:
     - sudo apt-get -qq update
     - sudo apt-get install -y build-essential cmake libfuzzy-dev unrar
+
+    # Build latest images spamscope-root, spamscope-elasticsearch
+
+    # clone dockerfile repositories
+    - git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-root.git $DOCKER_ROOT_PATH
+    - git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH;
+
+    # make images
+    - if [ "$TRAVIS_BRANCH" == "master" ]; then
+        cd $DOCKER_ROOT_PATH;
+        docker build -t $DOCKER_USERNAME/spamscope-root .;
+        cd -;
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'for f in tests/test_*.py; do python "$f"; done';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'thug -V';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'spamscope-topology -v';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'spamscope-elasticsearch -v';
+        cd $DOCKER_ELASTICSEARCH_PATH; 
+        docker build -t $DOCKER_USERNAME/spamscope-elasticsearch .;
+        cd -;
+      fi
+    - if [ "$TRAVIS_BRANCH" == "develop" ]; then
+        cd $DOCKER_ROOT_PATH;
+        docker build -t $DOCKER_USERNAME/spamscope-root:develop .;
+        cd -;
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root:develop /bin/sh -c 'for f in tests/test_*.py; do python "$f"; done';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root:develop /bin/sh -c 'thug -V';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root:develop /bin/sh -c 'spamscope-topology -v';
+        docker run --rm -it $DOCKER_USERNAME/spamscope-root:develop /bin/sh -c 'spamscope-elasticsearch -v';
+        cd $DOCKER_ELASTICSEARCH_PATH;
+        docker build -t $DOCKER_USERNAME/spamscope-elasticsearch:develop .;
+        cd -;
+      fi
 
 # command to install dependencies
 install: 
-    - pip install -r requirements.txt
-    - python setup.py install
+    - pip install -r requirements.txt && python setup.py install
+    - git clone https://$BITBUCKET_USER:[email protected]/$BITBUCKET_USER/zemana-api.git $ZEMANA_PATH;
+      cd $ZEMANA_PATH && python setup.py install && cd -
     - src/cli/faup.sh
     - cd ${FAUP_PATH}/src/lib/bindings/python && python setup.py install && cd -
 
+before_script:
+    - curl -o ${TIKA_APP_PATH} https://archive.apache.org/dist/tika/tika-app-1.14.jar
+
 # command to run tests
 script: 
+    # Unittests
     - python tests/test_attachments.py
     - python tests/test_attachments_post_processing.py
     - python tests/test_attachments_utils.py
     - python tests/test_bitmap.py
     - python tests/test_utils.py
+
+    # cli help
     - spamscope-elasticsearch -h
     - spamscope-topology -h
+
+after_success:
+    - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
+    - if [ "$TRAVIS_BRANCH" == "master" ]; then
+        docker push $DOCKER_USERNAME/spamscope-root;
+        docker push $DOCKER_USERNAME/spamscope-elasticsearch;
+      fi
+    - if [ "$TRAVIS_BRANCH" == "develop" ]; then
+      docker push $DOCKER_USERNAME/spamscope-root:develop;
+      docker push $DOCKER_USERNAME/spamscope-elasticsearch:develop;
+      fi
+
+notifications:
+    slack:
+        rooms:
+            - spamscope:$SLACK_KEY#developer

project.clj

@@ -1,4 +1,4 @@
-(defproject spamscope "1.5.2-SNAPSHOT"
+(defproject spamscope "1.5.3-SNAPSHOT"
   :resource-paths ["_resources"]
   :target-path "_build"
   :min-lein-version "2.0.0"

src/modules/attachments/attachments.py

@@ -24,6 +24,7 @@
 except ImportError:
     from collections import UserList
 
+import base64
 import copy
 import datetime
 import logging
@@ -116,13 +117,13 @@ def payloadstext(self):
 
                 if not i.get("is_archive"):
                     if i["content_transfer_encoding"] == "base64":
-                        text += i["payload"].decode("base64") + "\n"
+                        text += base64.b64decode(i["payload"]) + "\n"
                     else:
                         text += i["payload"] + "\n"
                     continue
 
                 for j in i.get("files", []):
-                    text += j["payload"].decode("base64") + "\n"
+                    text += base64.b64decode(j["payload"]) + "\n"
 
             except UnicodeDecodeError:
                 # This exception happens with binary payloads
@@ -220,7 +221,12 @@ def filter(self, check_list, hash_type="sha1"):
     def _metadata(raw_dict):
         """ Return payload, file size and extension of raw data. """
         if raw_dict["content_transfer_encoding"] == "base64":
-            payload = raw_dict["payload"].decode("base64")
+            try:
+                payload = base64.b64decode(raw_dict["payload"])
+            except TypeError, e:
+                # https://gist.github.com/perrygeo/ee7c65bb1541ff6ac770
+                payload = base64.b64decode(raw_dict["payload"] + "===")
+                raw_dict.setdefault("errors", []).append(e)
         else:
             payload = raw_dict["payload"]
 
@@ -300,7 +306,12 @@ def withhashes(cls, attachments=[]):
 
         for i in r:
             if i.get("content_transfer_encoding") == "base64":
-                payload = i["payload"].decode("base64")
+                try:
+                    payload = base64.b64decode(i["payload"])
+                except TypeError, e:
+                    payload = base64.b64decode(i["payload"] + "===")
+                    i.setdefault("errors", []).append(e)
+
             else:
                 payload = i["payload"]
 

src/options.py

@@ -19,7 +19,7 @@
 
 from os.path import join
 
-__version__ = "1.5.2"
+__version__ = "1.5.3"
 __configuration_path__ = "/etc/spamscope"
 
 __defaults__ = {