Skip to content

Commit

Permalink
Merge branch 'release/2.2.1'
Browse files Browse the repository at this point in the history
  • Loading branch information
fedelemantuano committed Feb 4, 2018
2 parents 14588b4 + a924373 commit feeaeab
Show file tree
Hide file tree
Showing 25 changed files with 219 additions and 52 deletions.
15 changes: 15 additions & 0 deletions .coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[run]
source = src/modules/

[report]
omit = src/modules/abstracts.py
src/modules/mails/spamassassin_analysis.py
src/modules/rabbitmq_client.py
src/modules/attachments/thug_analysis.py

exclude_lines =
pragma: no cover
def __repr__
raise AssertionError
raise NotImplementedError
if __name__ == .__main__.:
39 changes: 21 additions & 18 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ env:
TIKA_APP_JAR=/tmp/tika-app-${TIKA_VER}.jar
FAUP_PATH=/tmp/faup
ZEMANA_PATH=/tmp/zemana
DOCKER_ROOT_PATH=/tmp/docker-root
DOCKER_ELASTICSEARCH_PATH=/tmp/docker-elasticsearch

before_install:
Expand All @@ -25,26 +24,22 @@ before_install:
cmake
libfuzzy-dev
unrar
#- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce

- git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH

# Build latest images spamscope-root, spamscope-elasticsearch
# make images
- if [ "$TRAVIS_BRANCH" == "master" ]; then
git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-root.git $DOCKER_ROOT_PATH &&
cd $DOCKER_ROOT_PATH && docker build --build-arg SPAMSCOPE_VER=$TRAVIS_BRANCH -t $DOCKER_USERNAME/spamscope-root . && cd - &&
docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'python -m unittest discover -s tests -f -v' &&
docker run --rm -it $DOCKER_USERNAME/spamscope-root /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v' &&
git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH &&
cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=latest -t $DOCKER_USERNAME/spamscope-elasticsearch . && cd -;
cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=master -t $DOCKER_USERNAME/spamscope-elasticsearch . && cd -;
docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch /bin/sh -c 'python -m unittest discover -s tests -f -v';
docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v';
fi

- if [ "$TRAVIS_BRANCH" == "develop" ]; then
git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-root.git $DOCKER_ROOT_PATH &&
cd $DOCKER_ROOT_PATH && docker build --build-arg SPAMSCOPE_VER=$TRAVIS_BRANCH -t $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH . && cd - &&
docker run --rm -it $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH /bin/sh -c 'python -m unittest discover -s tests -f -v' &&
docker run --rm -it $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v' &&
git clone -b $TRAVIS_BRANCH --single-branch https://github.com/SpamScope/spamscope-dockerfile-elasticsearch.git $DOCKER_ELASTICSEARCH_PATH &&
cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=$TRAVIS_BRANCH -t $DOCKER_USERNAME/spamscope-elasticsearch:$TRAVIS_BRANCH . && cd -;
cd $DOCKER_ELASTICSEARCH_PATH && docker build --build-arg SPAMSCOPE_VER=develop -t $DOCKER_USERNAME/spamscope-elasticsearch:develop . && cd -;
docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch:develop /bin/sh -c 'python -m unittest discover -s tests -f -v';
docker run --rm -it $DOCKER_USERNAME/spamscope-elasticsearch:develop /bin/sh -c 'thug -V && spamscope-topology -v && spamscope-elasticsearch -v';
fi

# command to install dependencies
Expand All @@ -62,7 +57,7 @@ before_script:
# command to run tests
script:
# Unittests and coverage
- coverage run --include=src/modules/* --omit=src/modules/abstracts.py,src/modules/mails/spamassassin_analysis.py -m unittest discover -s tests -f -v
- coverage run -m unittest discover -s tests -f -v

# timing
- python tests/timing_test_search_keywords.py
Expand All @@ -73,19 +68,27 @@ script:
- spamscope-elasticsearch -v
- spamscope-topology -v

deploy:
provider: pypi
user: fmantuano
password:
secure: "lPuvh9if9jZrtgk/i9++D7v1BGfyEj9kalJy52yfjOARZ3xskrvHLjJMIxu9wFndYYdUAz8J1Y2WsKLDyYyfMwiAFadU2xmsobNunGO95CNUadbfJojaH7LartWQN+gnw98svJTms9GLHLmO8eH2j5BsH2mubOaWm8T0lF3Pjs0NncXdi6HzSQ63qGj6ctjFzAmTqoK14EzbGVAu1KRBzO6hdu41/nhWx1ufRZYBNeEjtNCZfIl74ALdCpEEQfqTl5hgMA9ybuXvcAmZMOWVjBQiXjSeAUoz+u2FYBkzSxx0Sst1MkKoC3iG6p+Z2E4WHya63ew1GQyyug7XSmq8JpaKRFtktU2b5g+AcqKmLsf0nw+2x/pjGxJIyy69OnX7oGr9fQTGGMHg36OU8vdyhOX+40nHEXC+M45rS/ma+tYSrL+j2Mn1kKreOjoC6kV1g+bgX2eSVFicFonB++ySFSt8D5b7zQibIx0BCxzN/WpNu99/0DTFpA0n9ox7+4F4Jzrn3a8IM56I/RyqBqh8ce7TjhOwsF6n8Egjk9ywVuZW8OuohBTxdosU+TAntlNCmGgECcdXzSGqTXLPWjfe2U7C4qtBVwHMlptYxZNs8XvlJksysM8Cyp+dUjXJCLqPH2n55aK9+x6Pzq/cSM1ldzXg+iZcO9XXoEDJ1bLXVDE="
on:
tags: true
repo: SpamScope/spamscope
branch: master

after_success:
- coveralls

- if [ "$TRAVIS_BRANCH" == "master" ]; then
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
docker push $DOCKER_USERNAME/spamscope-root;
docker push $DOCKER_USERNAME/spamscope-elasticsearch;
fi

- if [ "$TRAVIS_BRANCH" == "develop" ]; then
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
docker push $DOCKER_USERNAME/spamscope-root:$TRAVIS_BRANCH;
docker push $DOCKER_USERNAME/spamscope-elasticsearch:$TRAVIS_BRANCH;
docker push $DOCKER_USERNAME/spamscope-elasticsearch:develop;
fi

notifications:
Expand Down
8 changes: 3 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
[![Build Status](https://travis-ci.org/SpamScope/spamscope.svg?branch=master)](https://travis-ci.org/SpamScope/spamscope)
[![Coverage Status](https://coveralls.io/repos/github/SpamScope/spamscope/badge.svg?branch=develop)](https://coveralls.io/github/SpamScope/spamscope?branch=develop)
[![BCH compliance](https://bettercodehub.com/edge/badge/SpamScope/spamscope?branch=develop)](https://bettercodehub.com/)
[![](https://images.microbadger.com/badges/image/fmantuano/spamscope-elasticsearch.svg)](https://microbadger.com/images/fmantuano/spamscope-elasticsearch "Get your own image badge on microbadger.com")

![SpamScope](https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png)

Expand Down Expand Up @@ -225,11 +226,8 @@ $ export SPAMASSASSIN_ENABLED=True
## Docker images
It's possible to use complete Docker images with Apache Storm and SpamScope. Take the following images:

- [Root](https://hub.docker.com/r/fmantuano/spamscope-root/)
- [Elasticsearch](https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/)

For each image there are two tags: **develop** and **latest**.

- [Deps](https://hub.docker.com/r/fmantuano/spamscope-deps/): to use as base image
- [Elasticsearch](https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/): integrated with Elasticsearch


## Screenshots
Expand Down
11 changes: 7 additions & 4 deletions README.rst
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
|PyPI version| |Build Status| |Coverage Status| |BCH compliance|
|image4|

.. figure:: https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png
:alt: SpamScope
Expand Down Expand Up @@ -314,10 +315,10 @@ Docker images
It's possible to use complete Docker images with Apache Storm and
SpamScope. Take the following images:

- `Root <https://hub.docker.com/r/fmantuano/spamscope-root/>`__
- `Elasticsearch <https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/>`__

For each image there are two tags: **develop** and **latest**.
- `Deps <https://hub.docker.com/r/fmantuano/spamscope-deps/>`__: to use
as base image
- `Elasticsearch <https://hub.docker.com/r/fmantuano/spamscope-elasticsearch/>`__:
integrated with Elasticsearch

Screenshots
-----------
Expand Down Expand Up @@ -350,5 +351,7 @@ Screenshots
:target: https://coveralls.io/github/SpamScope/spamscope?branch=develop
.. |BCH compliance| image:: https://bettercodehub.com/edge/badge/SpamScope/spamscope?branch=develop
:target: https://bettercodehub.com/
.. |image4| image:: https://images.microbadger.com/badges/image/fmantuano/spamscope-elasticsearch.svg
:target: https://microbadger.com/images/fmantuano/spamscope-elasticsearch
.. |Donate| image:: https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif
:target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=VEPXYP745KJF2
5 changes: 3 additions & 2 deletions conf/templates/spamscope.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"order": 0,
"version": 4,
"version": 5,
"index_patterns": "spamscope_*-*",
"settings": {
"analysis": {
Expand Down Expand Up @@ -31,7 +31,8 @@
"index.codec": "best_compression",
"index.number_of_shards": 2,
"index.number_of_replicas": 0,
"index.refresh_interval": "5s"
"index.refresh_interval": "5s",
"index.mapping.ignore_malformed": true
},
"mappings": {
"analysis": {
Expand Down
27 changes: 27 additions & 0 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
FROM fmantuano/spamscope-deps

MAINTAINER Fedele Mantuano "[email protected]"

# environment variables
ARG SPAMSCOPE_VER="develop"

ENV SPAMASSASSIN_ENABLED="True" \
SPAMSCOPE_CONF_FILE="/etc/spamscope/spamscope.yml" \
SPAMSCOPE_PATH="/opt/spamscope" \
THUG_ENABLED="True"

# labels
LABEL description="Spamscope: Advanced Spam Analysis" \
spamscope_version=${SPAMSCOPE_VER}

# install SpamScope
RUN set -ex; \
mkdir -p "/var/log/spamscope" "/etc/spamscope"; \
git clone -b ${SPAMSCOPE_VER} --single-branch https://github.com/SpamScope/spamscope.git ${SPAMSCOPE_PATH}; \
cd $SPAMSCOPE_PATH; \
pip install -r requirements_optional.txt; \
python setup.py install; \
sparse jar -s; \
pip install -U thug;

WORKDIR ${SPAMSCOPE_PATH}
22 changes: 8 additions & 14 deletions docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,22 @@ version: '2.1'
services:

# SpamScope
spamscope:
image: ${SPAMSCOPE_IMAGE_NAME}
spamscope-debug:
image: spamscope-debug
build: .
container_name: spamscope
mem_limit: ${SPAMSCOPE_MEM_LIMIT}
dns:
- 8.8.8.8
- 8.8.4.4
mem_limit: 4g
ports:
- "${SPAMSCOPE_BIND_IP}:8080:8080"
- "${SPAMSCOPE_BIND_IP}:8000:8000"
- "127.0.0.1:8080:8080"
- "127.0.0.1:8000:8000"
volumes:
- ${HOST_SPAMSCOPE_CONF}:/etc/spamscope
- ${HOST_MAILS_FOLDER}:${DOCKER_MAILS_FOLDER}
healthcheck:
test: curl -fs http://localhost:8080/
networks:
- ${NET_NAME}

volumes:
esdata:
driver: local
- spamscope

networks:
esnet:
spamscope:
driver: bridge
2 changes: 1 addition & 1 deletion project.clj
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
(defproject spamscope "2.2.0-SNAPSHOT"
(defproject spamscope "2.2.1-SNAPSHOT"
:resource-paths ["_resources"]
:target-path "_build"
:min-lein-version "2.0.0"
Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ astropy>=1.3.3
backports.functools-lru-cache>=1.3
chainmap
lxml
mail-parser>=3
mail-parser>=3.2.6
patool
pyparsing
python-magic
simplejson
six
ssdeep
streamparse==3.13.0
streamparse==3.13.1
1 change: 1 addition & 0 deletions src/bolts/tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ def _make_mail(self, tup):
mail["mailbox"] = tup.values[2]
mail["priority"] = tup.values[3]
mail["sender_ip"] = self.parser.get_server_ipaddress(tup.values[4])
mail["to_domains"] = self.parser.to_domains

# Fingerprints of body mail
(mail["md5"], mail["sha1"], mail["sha256"], mail["sha512"],
Expand Down
8 changes: 5 additions & 3 deletions src/modules/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import re
import signal
import tempfile
from functools import wraps
import functools

import six
import yaml
Expand Down Expand Up @@ -87,6 +87,7 @@ def decorator(func):
def _handle_timeout(signum, frame):
raise TimeoutError(error_message)

@functools.wraps(func)
def wrapper(*args, **kwargs):
signal.signal(signal.SIGALRM, _handle_timeout)
signal.alarm(seconds)
Expand All @@ -96,7 +97,7 @@ def wrapper(*args, **kwargs):
signal.alarm(0)
return result

return wraps(func)(wrapper)
return wrapper

return decorator

Expand Down Expand Up @@ -163,7 +164,8 @@ def urls_extractor(text, faup):
for i in set(match.group().strip() for match in RE_URL.finditer(text)):
faup.decode(i)
tokens = faup.get()
results.setdefault(tokens["domain"], []).append(tokens)
if tokens["domain"]:
results.setdefault(tokens["domain"], []).append(tokens)
else:
return results

Expand Down
2 changes: 1 addition & 1 deletion src/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from os.path import join

__version__ = "2.2.0"
__version__ = "2.2.1"
__configuration_path__ = "/etc/spamscope"

__defaults__ = {
Expand Down
79 changes: 79 additions & 0 deletions tests/samples/mail_test_7

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions tests/test_attachments.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
limitations under the License.
"""

import logging
import os
import sys
import unittest
Expand Down Expand Up @@ -51,6 +52,8 @@

OPTIONS = ChainMap(os.environ, DEFAULTS)

logging.getLogger().addHandler(logging.NullHandler())


class TestAttachments(unittest.TestCase):

Expand Down
3 changes: 3 additions & 0 deletions tests/test_attachments_post_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
limitations under the License.
"""

import logging
import os
import sys
import unittest
Expand Down Expand Up @@ -45,6 +46,8 @@

OPTIONS = ChainMap(os.environ, DEFAULTS)

logging.getLogger().addHandler(logging.NullHandler())


class TestPostProcessing(unittest.TestCase):

Expand Down
3 changes: 3 additions & 0 deletions tests/test_attachments_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
limitations under the License.
"""

import logging
import os
import sys
import unittest
Expand All @@ -34,6 +35,8 @@
sample_txt = os.path.join(base_path, 'samples', 'test.txt')
vt_report = os.path.join(base_path, 'samples', 'vt_report.json')

logging.getLogger().addHandler(logging.NullHandler())


class TestAttachmentsUtils(unittest.TestCase):

Expand Down
3 changes: 3 additions & 0 deletions tests/test_bitmap.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
limitations under the License.
"""

import logging
import os
import sys
import unittest
Expand All @@ -28,6 +29,8 @@
import src.modules.bitmap as bitmap
from src.modules.bitmap import PhishingBitMap

logging.getLogger().addHandler(logging.NullHandler())


class ValidBitMap(bitmap.BitMap):

Expand Down
3 changes: 3 additions & 0 deletions tests/test_network_post_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
limitations under the License.
"""

import logging
import os
import six
import sys
Expand All @@ -40,6 +41,8 @@

OPTIONS = ChainMap(os.environ, DEFAULTS)

logging.getLogger().addHandler(logging.NullHandler())


class TestPostProcessing(unittest.TestCase):

Expand Down
Loading

0 comments on commit feeaeab

Please sign in to comment.