Merge branch 'release/2.7.0'

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is:
+ - which bolt/spout is in error
+ - Apache Storm error log
+ - ...
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ...
+
+Attach main configuration file of `SpamScope`.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Raw mail**
+The raw mail to reproduce the behavior.
+You can use a `gist` like [this](https://gist.github.com/fedelemantuano/5dd702004c25a46b2bd60de21e67458e).
+The issues without raw mail will be closed.
+
+**Environment:**
+ - OS: [e.g. Debian, Centos]
+ - Docker: [yes or no]
+ - `SpamScope` version [e.g. 3.6.0]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -6,10 +6,12 @@ _resources
 .env
 .idea/
 .ropeproject
+.tox/
 .vscode/
 *.pyc
 build/
 dist/
 logs/
+report/
 SpamScope.egg-info/
 venv/

.travis.yml

@@ -12,7 +12,6 @@ python:
 env:
     - TIKA_APP_JAR=/tmp/tika-app-${TIKA_VER}.jar
       FAUP_PATH=/tmp/faup
-      ZEMANA_PATH=/tmp/zemana
       DOCKER_ELASTICSEARCH_PATH=/tmp/docker-elasticsearch
 
 before_install:
@@ -48,7 +47,6 @@ install:
     - pip install --upgrade pip setuptools
     - python setup.py install
     - pip install -r requirements_optional.txt
-    - git clone https://$BITBUCKET_USER:[email protected]/$BITBUCKET_USER/zemana-api.git $ZEMANA_PATH && cd $ZEMANA_PATH && python setup.py install && cd -
     - src/cli/faup.sh
     - pip install coveralls
 

Makefile

@@ -0,0 +1,82 @@
+.PHONY: clean clean-test clean-pyc clean-build docs help
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+
+try:
+	from urllib import pathname2url
+except:
+	from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+
+define PRINT_HELP_PYSCRIPT
+import re, sys
+
+for line in sys.stdin:
+	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+	if match:
+		target, help = match.groups()
+		print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
+
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+help:
+	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+
+clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+
+clean-build: ## remove build artifacts
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: ## remove Python file artifacts
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## remove test and coverage artifacts
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -fr .pytest_cache
+
+lint: ## check style with flake8
+	flake8 src tests
+
+test: ## run tests quickly with the default Python
+	python -m unittest discover -s tests -f -v
+
+test-all: ## run tests on every Python version with tox
+	tox
+
+# docs: ## generate Sphinx HTML documentation, including API docs
+# 	rm -f docs/mailparser.rst
+# 	rm -f docs/modules.rst
+# 	sphinx-apidoc -o docs/ mailparser
+# 	$(MAKE) -C docs clean
+# 	$(MAKE) -C docs html
+# 	$(BROWSER) docs/_build/html/index.html
+
+# servedocs: docs ## compile the docs watching for changes
+# 	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
+
+release: dist ## package and upload a release
+	twine upload dist/*
+
+dist: clean ## builds source and wheel package
+	python setup.py sdist
+	python setup.py bdist_wheel
+	ls -l dist
+
+install: clean ## install the package to the active Python's site-packages
+	python setup.py install

ansible/requirements.txt

@@ -1 +1 @@
-ansible==2.5.0
+ansible

ansible/templates/spamscope.yml.j2

@@ -51,6 +51,9 @@ phishing:
 
 
 tokenizer:
+    # Persistent where store dumps of hashes.
+    persistent_path: /tmp
+
     # If true mails with same hash are filtered and not analyzed.
     # Only the body will not saved
     filter_mails: true
@@ -84,19 +87,37 @@ network:
         enabled: false
         api_key: xxxxxxxxxxxxxxxxxxxxxxxxxx
 
-
 # RawMail bolt configuration
 raw_mail:
     # SpamAssassin analysis: https://spamassassin.apache.org/
     spamassassin:
         enabled: false
 
-
+    # Dialect analysis: https://sissden.eu/blog/analysis-of-smtp-dialects
+    dialect:
+        enabled: false
+
+        # elasticsearch instance where are postfix logs
+        elasticsearch:
+            hosts:
+                - "node1:9200"
+                - "node2"
+
+            # Prefix with dash '-' of Postfix index in Elasticsearch
+            # The format of indices should be postfix-2018.12.30
+            index.prefix.postfix: postfix-
+
 # Attachments bolt configuration
 attachments:
-    # The lists of all components must be under lists keyword to load them
-    # automatically
     commons:
+        # enable or disable filter on size
+        size.filter.enabled: false
+
+        # max size to analyze in bytes
+        max.size: 3145728
+
+        # The lists of all components must be under lists keyword to load them
+        # automatically
         lists:
             blacklist_content_types:
                 # All content types to remove from results

conf/spamscope.example.yml

@@ -113,7 +113,7 @@ raw_mail:
     # Dialect analysis: https://sissden.eu/blog/analysis-of-smtp-dialects
     dialect:
         enabled: false
-        
+
         # elasticsearch instance where are postfix logs
         elasticsearch:
             hosts:
@@ -127,14 +127,15 @@ raw_mail:
 
 # Attachments bolt configuration
 attachments:
-    # The lists of all components must be under lists keyword to load them
-    # automatically
     commons:
         # enable or disable filter on size
         size.filter.enabled: false
+
         # max size to analyze in bytes
         max.size: 3145728
 
+        # The lists of all components must be under lists keyword to load them
+        # automatically
         lists:
             blacklist_content_types:
                 # All content types to remove from results

conf/templates/commons.json

@@ -4,6 +4,7 @@
   "settings": {
     "index.codec": "best_compression",
     "index.number_of_replicas": 0,
+    "index.number_of_shards": 1,
     "index.refresh_interval": "5s",
     "index.mapping.total_fields.limit": 100000,
     "index.mapping.ignore_malformed": true

conf/templates/spamscope_attachments.json

@@ -1,38 +1,44 @@
 {
   "order": 0,
-  "version": 2,
-  "index_patterns": "spamscope_attachments-*",
+  "version": 3,
+  "index_patterns": [
+    "spamscope_attachments-*"
+  ],
   "settings": {
-    "analysis": {
-      "analyzer": {
-        "header": {
-          "tokenizer": "uax_url_email",
-          "filter": [
-            "lowercase"
-          ]
-        },
-        "html_body": {
-          "char_filter": [
-            "html_strip"
-          ],
-          "tokenizer": "uax_url_email",
-          "filter": [
-            "lowercase"
-          ]
-        },
-        "path_pattern": {
-          "tokenizer": "path_hierarchy",
-          "filter": [
-            "lowercase"
-          ]
+    "index": {
+      "codec": "best_compression",
+      "mapping": {
+        "ignore_malformed": "true"
+      },
+      "refresh_interval": "5s",
+      "analysis": {
+        "analyzer": {
+          "header": {
+            "filter": [
+              "lowercase"
+            ],
+            "tokenizer": "uax_url_email"
+          },
+          "html_body": {
+            "filter": [
+              "lowercase"
+            ],
+            "char_filter": [
+              "html_strip"
+            ],
+            "tokenizer": "uax_url_email"
+          },
+          "path_pattern": {
+            "filter": [
+              "lowercase"
+            ],
+            "tokenizer": "path_hierarchy"
+          }
         }
-      }
-    },
-    "index.codec": "best_compression",
-    "index.number_of_shards": 1,
-    "index.number_of_replicas": 0,
-    "index.refresh_interval": "5s",
-    "index.mapping.ignore_malformed": true
+      },
+      "number_of_shards": "1",
+      "number_of_replicas": "0"
+    }
   },
   "mappings": {
     "_doc": {
@@ -86,5 +92,29 @@
         }
       }
     }
+  },
+  "aliases": {
+    "attachments": {},
+    "attachments_thug": {
+      "filter": {
+        "exists": {
+          "field": "thug"
+        }
+      }
+    },
+    "attachments_tika": {
+      "filter": {
+        "exists": {
+          "field": "tika"
+        }
+      }
+    },
+    "attachments_virustotal": {
+      "filter": {
+        "exists": {
+          "field": "virustotal"
+        }
+      }
+    }
   }
-}
+}