Merge pull request #6 from VultureProject/dev

Release 1.77

gui/models/modlog_settings.py

@@ -76,9 +76,9 @@ def get_format(self):
         if self.repository_type=='data':
             repo = BaseAbstractRepository.search_repository(self.data_repository)
             if isinstance(repo, MongoDBRepository):
-                return "@cee:{\\\"app_name\\\":\\\"%{app_name}e\\\",\\\"src_ip\\\":\\\"%a\\\",\\\"user\\\":\\\"%u\\\",\\\"time\\\":\\\"%{%Y-%m-%dT%H:%M:%S:%Z}t\\\",\\\"http_method\\\":\\\"%m\\\",\\\"requested_uri\\\":\\\"%U%q\\\",\\\"http_code\\\":%>s,\\\"incoming_protocol\\\":\\\"%H\\\",\\\"referer\\\":\\\"%{Referer}i\\\",\\\"user_agent\\\":\\\"%{User-agent}i\\\",\\\"size\\\":%B,\\\"bytes_received\\\":%I,\\\"bytes_sent\\\":%O,\\\"time_elapsed\\\":%D,\\\"country\\\":\\\"%{COUNTRY_CODE}e\\\",\\\"city\\\":\\\"%{CITY}e\\\",\\\"lat\\\":\\\"%{LATITUDE}e\\\",\\\"lon\\\":\\\"%{LONGITUDE}e\\\",\\\"reputation\\\":\\\"%{REPUTATION0}e,%{REPUTATION1}e,%{REPUTATION2}e,%{REPUTATION3}e,%{REPUTATION4}e\\\",\\\"owasp_top10\\\":\\\"%{owasp_top10}e\\\",\\\"reasons\\\":\\\"%{reasons}e\\\",\\\"threshold\\\":\\\"%{threshold}e\\\",\\\"score\\\":\\\"%{score}e\\\"}"
+                return "@cee:{\\\"app_name\\\":\\\"%{app_name}e\\\",\\\"src_ip\\\":\\\"%a\\\",\\\"user\\\":\\\"%u\\\",\\\"time\\\":\\\"%{%Y-%m-%dT%H:%M:%S:%Z}t\\\",\\\"http_method\\\":\\\"%m\\\",\\\"requested_uri\\\":\\\"%U%q\\\",\\\"http_code\\\":%>s,\\\"incoming_protocol\\\":\\\"%H\\\",\\\"referer\\\":\\\"%{Referer}i\\\",\\\"user_agent\\\":\\\"%{User-agent}i\\\",\\\"size\\\":%B,\\\"bytes_received\\\":%I,\\\"bytes_sent\\\":%O,\\\"time_elapsed\\\":%D,\\\"country\\\":\\\"%{COUNTRY_CODE}e\\\",\\\"city\\\":\\\"%{CITY}e\\\",\\\"lat\\\":\\\"%{LATITUDE}e\\\",\\\"lon\\\":\\\"%{LONGITUDE}e\\\",\\\"reputation\\\":\\\"%{REPUTATION0}e,%{REPUTATION1}e,%{REPUTATION2}e,%{REPUTATION3}e,%{REPUTATION4}e\\\",\\\"owasp_top10\\\":\\\"%{owasp_top10}e\\\",\\\"reasons\\\":\\\"%{reasons}e\\\",\\\"threshold\\\":\\\"%{threshold}e\\\",\\\"score\\\":\\\"%{score}e\\\",\\\"authentication\\\":\\\"%{X-Authentication}n\\\",\\\"authentication_result\\\":\\\"%{X-Authentication-Result}n\\\"}"
             elif isinstance(repo, ElasticSearchRepository):
-                return "@cee:{\\\"app_name\\\":\\\"%{app_name}e\\\",\\\"src_ip\\\":\\\"%a\\\",\\\"user\\\":\\\"%u\\\",\\\"time\\\":\\\"%{%Y-%m-%dT%H:%M:%S%z}t\\\",\\\"http_method\\\":\\\"%m\\\",\\\"requested_uri\\\":\\\"%U%q\\\",\\\"http_code\\\":%>s,\\\"incoming_protocol\\\":\\\"%H\\\",\\\"referer\\\":\\\"%{Referer}i\\\",\\\"user_agent\\\":\\\"%{User-agent}i\\\",\\\"size\\\":%B,\\\"bytes_received\\\":%I,\\\"bytes_sent\\\":%O,\\\"time_elapsed\\\":%D,\\\"country\\\":\\\"%{COUNTRY_CODE}e\\\",\\\"city\\\":\\\"%{CITY}e\\\",\\\"lat\\\":\\\"%{LATITUDE}e\\\",\\\"lon\\\":\\\"%{LONGITUDE}e\\\",\\\"reputation\\\":\\\"%{REPUTATION0}e,%{REPUTATION1}e,%{REPUTATION2}e,%{REPUTATION3}e,%{REPUTATION4}e\\\",\\\"owasp_top10\\\":\\\"%{owasp_top10}e\\\",\\\"reasons\\\":\\\"%{reasons}e\\\",\\\"threshold\\\":\\\"%{threshold}e\\\",\\\"score\\\":\\\"%{score}e\\\"}"
+                return "@cee:{\\\"app_name\\\":\\\"%{app_name}e\\\",\\\"src_ip\\\":\\\"%a\\\",\\\"user\\\":\\\"%u\\\",\\\"time\\\":\\\"%{%Y-%m-%dT%H:%M:%S%z}t\\\",\\\"http_method\\\":\\\"%m\\\",\\\"requested_uri\\\":\\\"%U%q\\\",\\\"http_code\\\":%>s,\\\"incoming_protocol\\\":\\\"%H\\\",\\\"referer\\\":\\\"%{Referer}i\\\",\\\"user_agent\\\":\\\"%{User-agent}i\\\",\\\"size\\\":%B,\\\"bytes_received\\\":%I,\\\"bytes_sent\\\":%O,\\\"time_elapsed\\\":%D,\\\"country\\\":\\\"%{COUNTRY_CODE}e\\\",\\\"city\\\":\\\"%{CITY}e\\\",\\\"lat\\\":\\\"%{LATITUDE}e\\\",\\\"lon\\\":\\\"%{LONGITUDE}e\\\",\\\"reputation\\\":\\\"%{REPUTATION0}e,%{REPUTATION1}e,%{REPUTATION2}e,%{REPUTATION3}e,%{REPUTATION4}e\\\",\\\"owasp_top10\\\":\\\"%{owasp_top10}e\\\",\\\"reasons\\\":\\\"%{reasons}e\\\",\\\"threshold\\\":\\\"%{threshold}e\\\",\\\"score\\\":\\\"%{score}e\\\"}\\\"authentication\\\":\\\"%{X-Authentication}n\\\",\\\"authentication_result\\\":\\\"%{X-Authentication-Result}n\\\"}"
 
         separators = {
             'space': ' ',

portal/views/oauth2_portal.py

@@ -34,6 +34,7 @@
 from gui.models.system_settings      import Cluster
 from portal.system.authentications   import OAUTH2Authentication
 from portal.system.redis_sessions    import REDISBase
+from portal.views.responses          import response_success, response_failure
 
 # Required exceptions imports
 from bson.errors                     import InvalidId
@@ -80,7 +81,7 @@ def log_in(request):
         logger.info("OAUTH2::log_in: Authentication succeed for user '{}'".format(authentication.credentials[0]))
         response               = authentication.generate_response(authentication_results)
         logger.info("OAUTH2::log_in: Response successfully generated for user '{}' : {}".format(authentication.credentials[0], response))
-        return response
+        return response_success(response, "oauth2")
 
     # Redis connection error
     except RedisConnectionError as e:
@@ -109,7 +110,7 @@ def log_in(request):
         logger.error("OAUTH2::log_in: Error while trying to authentication user '{}' : ".format(request.POST.get('username',None)))
         logger.exception(e)
 
-    return HttpResponseForbidden()
+    return response_failure(HttpResponseForbidden(), "oauth2")
 
 
 
@@ -162,6 +163,7 @@ def is_valid_token(request):
             body = {"active": "false"}
 
         logger.debug("OAuth2Portal::is_valid_token: Returning '{}'".format(body))
+        response_funcs = {"true"}
         return JsonResponse(body)
 
     else:

portal/views/responses.py

@@ -41,6 +41,22 @@
 # Global variables
 BASE_DIR = dirname(dirname(__file__))
 
+AUTH_TYPE_HEADER = "X-Authentication"
+AUTH_RESULT_HEADER = "X-Authentication-Result"
+
+
+def response_success(response, action_type):
+    if action_type:
+        response[AUTH_TYPE_HEADER] = action_type
+        response[AUTH_RESULT_HEADER] = "success"
+    return response
+
+def response_failure(response, action_type):
+    if action_type:
+        response[AUTH_TYPE_HEADER] = action_type
+        response[AUTH_RESULT_HEADER] = "failure"
+    return response
+
 
 def split_domain(url):
     """ Split an url and return the 2 last domains

portal/views/self.py

@@ -36,6 +36,7 @@
 # Django project imports
 from portal.system.self_actions import SELFService, SELFServiceChange, SELFServiceLogout, SELFServiceLost
 from vulture_toolkit.auth.exceptions import AuthenticationError, ChangePasswordError
+from portal.views.responses import response_failure, response_success
 
 # Required exceptions imports
 from django.utils.datastructures     import MultiValueDictKeyError
@@ -81,51 +82,51 @@ def self(request, token_name=None, proxy_app_id=None, action=None):
     except RedisConnectionError as e:
         # Redis connection error
         logger.error("PORTAL::log_in: Unable to connect to Redis server : {}".format(str(e)))
-        return HttpResponseServerError()
+        return response_failure(HttpResponseServerError(),action)
 
     # If assertionError : Forbidden
     except AssertionError as e:
         logger.error("PORTAL::log_in: AssertionError while trying to create Authentication : ".format(e))
-        return HttpResponseForbidden()
+        return response_failure(HttpResponseForbidden(), action)
 
     except Exception as e:
         logger.error("Unknown error occured while retrieving user informations :")
         logger.exception(e)
-        return HttpResponseForbidden()
+        return response_failure(HttpResponseForbidden(), action)
 
 
     try:
         credential = Action.retrieve_credentials(request)
         if not action:
             result = Action.perform_action()
             logger.info("SELF::main: List of apps successfully retrieven")
-            return Action.main_response(request, result)
+            return response_success(Action.main_response(request, result), "list_apps")
         else:
-            return Action.message_response(Action.perform_action(request, credential))
+            return response_success(Action.message_response(Action.perform_action(request, credential)), action)
 
     # Redis connection error
     except RedisConnectionError as e:
         logger.error("PORTAL::log_in: Unable to connect to Redis server : {}".format(str(e)))
-        return HttpResponseServerError()
+        return response_failure(HttpResponseServerError(), action)
 
     # If assertionError : Forbidden
     except AssertionError as e:
         logger.error("PORTAL::log_in: AssertionError while trying to create Authentication : '{}'".format(e))
-        return HttpResponseForbidden(e)
+        return response_failure(HttpResponseForbidden(e), action)
 
     except (DBAPIError, LDAPError, PyMongoError) as e:
         logger.error("SELF::self: Failed to update password :".format(e))
         logger.exception(e)
-        return Action.ask_credentials_response(request, action, "<b> Database error </b> <br> "
-                                                                "Please contact your administrator")
+        return response_failure(Action.ask_credentials_response(request, action, "<b> Database error </b> <br> "
+                                                                "Please contact your administrator"), action)
 
     except PasswordMatchError as e:
         logger.error("SELF::self: Validation form error: '{}'".format(e))
-        return Action.ask_credentials_response(request, action, e)
+        return response_failure(Action.ask_credentials_response(request, action, e), action)
 
     except (ChangePasswordError, AuthenticationError) as e:
         logger.error("SELF::self: Authentication or credentials error : '{}'".format(e))
-        return Action.ask_credentials_response(request, action, "Authentication failure.")
+        return response_failure(Action.ask_credentials_response(request, action, "Authentication failure."), action)
 
     except MultiValueDictKeyError as e:
         if request.method == "GET":
@@ -135,13 +136,13 @@ def self(request, token_name=None, proxy_app_id=None, action=None):
             return Action.ask_credentials_response(request, action, "Field missing : "+str(e))
 
     except SMTPException as e:
-        return Action.ask_credentials_response(request, action, str(e))
+        return response_failure(Action.ask_credentials_response(request, action, str(e)), action)
 
     except KeyError as e:
         logger.exception(e)
-        return HttpResponseForbidden()
+        return response_failure(HttpResponseForbidden(), action)
 
     except Exception as e:
         logger.error(type(e))
         logger.exception(e)
-        return Action.message_response("An unknown error occured <br><b> Please contact your admninistrator</b>")
+        return response_failure(Action.message_response("An unknown error occured <br><b> Please contact your admninistrator</b>"), action)

vulture_toolkit/templates/rsyslog.conf

@@ -34,6 +34,8 @@ template(name="tpl_mongo" type="list") {
   property(name="$!reasons")
   property(name="$!score")
   property(name="$!threshold")
+  property(name="$!authentication")
+  property(name="$!authentication_result")
 }
 
 ###### Vulture access log Template for Elastic ######

vulture_toolkit/templates/vulture_httpd.conf

@@ -583,6 +583,13 @@ SSLProxyEngine {{ conf.server_ssl_proxyengine }}
         RequestHeader set Referer "http://127.0.0.1:9000/portal"
         RequestHeader set X-Forwarded-For "%{REMOTE_ADDR}s"
 
+        # Retrieve authentication statuses from headers stock them in notes
+        Header note X-Authentication X-Authentication
+        Header note X-Authentication-Result X-Authentication-Result
+        # And remove them to prevent sending to user
+        Header unset X-Authentication
+        Header unset X-Authentication-Result
+
         SetEnv portal 1
         {% set first = True %}
         {% for app in apps %}

vulture_toolkit/update_api/update_scripts/GUI-1.77/maj_portal-logs.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+# This migration script install newest Vulture-LIBS package
+#
+#
+
+. /etc/rc.conf
+
+if [ "$http_proxy" != "" ]
+then
+    export https_proxy="http://$http_proxy"
+    export http_proxy="http://$http_proxy"
+fi
+
+dir="/var/log/Vulture/portal"
+
+for file in "debug.log" \
+"portal_authentication.log" \
+"redis_events.log" ; do
+    /usr/bin/touch "$dir/$file"
+    /bin/chmod 644 "$dir/$file"
+    /usr/sbin/chown vlt-portal:vlt-web "$dir/$file"
+done

vulture_toolkit/update_api/update_scripts/GUI-1.77/maj_vulture-LIBS.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# This migration script install newest Vulture-LIBS package
+#
+#
+
+. /etc/rc.conf
+
+if [ "$http_proxy" != "" ]
+then
+    export https_proxy="http://$http_proxy"
+    export http_proxy="http://$http_proxy"
+fi
+
+cd /tmp
+
+/bin/echo "[+] Updating Vulture-LIBS from branch \"$1\"..."
+/bin/rm -f /tmp/Vulture-LIBS.tar.gz
+
+bsd_version=$(/usr/bin/uname -r | /usr/bin/cut -d '-' -f 1)
+url="https://dl.vultureproject.org/$bsd_version$1/Vulture-LIBS.tar.gz"
+
+/bin/echo -n "[+] Downloading from '$url' ..."
+/usr/local/bin/wget --no-check-certificate $url >>/tmp/installation.log 2>&1
+/bin/echo "DONE"
+
+cd /home/vlt-gui
+/bin/rm -rf ./env
+/usr/bin/tar xf /tmp/Vulture-LIBS.tar.gz
+/usr/sbin/chown -R vlt-gui:vlt-gui /home/vlt-gui/
+
+/bin/echo -n "[+] Installing Vulture libraries ..."
+/bin/sh "/home/vlt-gui/lib-$bsd_version/install.sh"
+/bin/echo "OK"
+
+/bin/echo "[+] Installing required libraries for Images ..."
+/usr/sbin/pkg install -y libxcb && echo "OK" || (echo "KO" && echo "[!] Please install following packages manually : libxcb")
+
+/bin/echo "[*] Update of Vulture-LIBS ended"
+

vulture_toolkit/update_api/update_scripts/GUI-1.77/reload_apps_conf.py

@@ -0,0 +1,43 @@
+#!/home/vlt-gui/env/bin/python
+# coding:utf-8
+
+"""This file is part of Vulture 3.
+
+Vulture 3 is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Vulture 3 is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Vulture 3.  If not, see http://www.gnu.org/licenses/.
+"""
+__author__ = "Kevin Guillemot"
+__credits__ = []
+__license__ = "GPLv3"
+__version__ = "3.0.0"
+__maintainer__ = "Vulture Project"
+__email__ = "[email protected]"
+__doc__ = """This migration script reload Rsyslog and Logrotate configurations, and restart services if needed """
+
+import os
+import sys
+
+sys.path.append('/home/vlt-gui/vulture')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'vulture.settings')
+
+import django
+django.setup()
+
+from gui.models.system_settings import Cluster
+from gui.signals.gui_signals import config_modified
+
+if __name__ == '__main__':
+
+    config_modified.send(sender=Cluster, id=None)
+
+    print("Applications configuration reloaded.")

vulture_toolkit/update_api/update_scripts/GUI-1.77/reload_rsyslogd_conf.py

@@ -0,0 +1,42 @@
+#!/home/vlt-gui/env/bin/python
+# coding:utf-8
+
+"""This file is part of Vulture 3.
+
+Vulture 3 is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Vulture 3 is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Vulture 3.  If not, see http://www.gnu.org/licenses/.
+"""
+__author__ = "Kevin Guillemot"
+__credits__ = []
+__license__ = "GPLv3"
+__version__ = "3.0.0"
+__maintainer__ = "Vulture Project"
+__email__ = "[email protected]"
+__doc__ = """This migration script reload Rsyslog configuration"""
+
+import os
+import sys
+
+sys.path.append('/home/vlt-gui/vulture')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'vulture.settings')
+
+import django
+django.setup()
+
+from gui.signals.gui_signals import log_modified
+
+if __name__ == '__main__':
+
+    log_modified.send(sender=None)
+
+    print("Rsyslogd configuration reloaded.")