Merge branch 'release/2.2.0'

.gitignore

@@ -10,5 +10,5 @@ _build
 _resources
 build/
 dist/
-logs
-virtualenvs
+logs/
+venv/

.travis.yml

@@ -20,12 +20,12 @@ env:
 before_install:
     - sudo apt-get -qq update
     - >
-        sudo apt-get install -y
+        sudo apt-get -yqq --no-install-recommends install
         build-essential
         cmake
         libfuzzy-dev
         unrar
-    - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+    #- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
 
     # Build latest images spamscope-root, spamscope-elasticsearch
     # make images
@@ -51,6 +51,7 @@ before_install:
 install:
     - pip install --upgrade pip setuptools
     - python setup.py install
+    - pip install -r requirements_optional.txt
     - git clone https://$BITBUCKET_USER:[email protected]/$BITBUCKET_USER/zemana-api.git $ZEMANA_PATH && cd $ZEMANA_PATH && python setup.py install && cd -
     - src/cli/faup.sh
     - pip install coveralls
@@ -61,7 +62,7 @@ before_script:
 # command to run tests
 script:
     # Unittests and coverage
-    - coverage run --include=src/modules/* --omit=src/modules/abstracts.py -m unittest discover -s tests -f -v
+    - coverage run --include=src/modules/* --omit=src/modules/abstracts.py,src/modules/mails/spamassassin_analysis.py -m unittest discover -s tests -f -v
 
     # timing
     - python tests/timing_test_search_keywords.py

README.md

@@ -87,20 +87,29 @@ For more details please visit the [wiki page](https://github.com/SpamScope/spams
 Clone repository
 
 ```
-git clone https://github.com/SpamScope/spamscope.git
+$ git clone https://github.com/SpamScope/spamscope.git
 ```
 
 then enter in SpamScope directory and install it:
 
 ```
-python setup.py install
+$ python setup.py install
 ```
 or
 
 ```
-pip install SpamScope
+$ pip install SpamScope
 ```
 
+If you want to install all optional packages:
+
+```
+$ git clone https://github.com/SpamScope/spamscope.git
+$ pip install -r requirements_optional
+```
+
+Thug is not in requirements_optional. To install it go in Thug section.
+
 ### Faup
 [Faup](https://github.com/stricaud/faup) stands for Finally An Url Parser and is a library and command line tool to parse URLs and normalize fields.
 To install it follow the [wiki](https://github.com/SpamScope/spamscope/wiki/Installation#faup).
@@ -125,6 +134,15 @@ Thug is a Python low-interaction honeyclient aimed at mimicing the behavior of a
 
 You can see a complete SpamScope report with Thug analysis [here](https://goo.gl/Y4kWCv).
 
+Thug analysis can be very slow and you can have `heartbeat timeout` in Apache Storm.
+To avoid any issue set `supervisor.worker.timeout.secs`:
+
+```
+nr. user agents * timeout_thug < supervisor.worker.timeout.secs
+```
+
+The best value for `threshold` is 1.
+
 ### VirusTotal (optional)
 It's possible add to results (for mail attachments and sender ip address) the VirusTotal report. You need a private API key.
 

README.rst

@@ -126,19 +126,28 @@ Clone repository
 
 ::
 
-    git clone https://github.com/SpamScope/spamscope.git
+    $ git clone https://github.com/SpamScope/spamscope.git
 
 then enter in SpamScope directory and install it:
 
 ::
 
-    python setup.py install
+    $ python setup.py install
 
 or
 
 ::
 
-    pip install SpamScope
+    $ pip install SpamScope
+
+If you want to install all optional packages:
+
+::
+
+    $ git clone https://github.com/SpamScope/spamscope.git
+    $ pip install -r requirements_optional
+
+Thug is not in requirements\_optional. To install it go in Thug section.
 
 Faup
 ~~~~
@@ -183,6 +192,15 @@ What is Thug? From README project:
 You can see a complete SpamScope report with Thug analysis
 `here <https://goo.gl/Y4kWCv>`__.
 
+Thug analysis can be very slow and you can have ``heartbeat timeout`` in
+Apache Storm. To avoid any issue set ``supervisor.worker.timeout.secs``:
+
+::
+
+    nr. user agents * timeout_thug < supervisor.worker.timeout.secs
+
+The best value for ``threshold`` is 1.
+
 VirusTotal (optional)
 ~~~~~~~~~~~~~~~~~~~~~